feat: Major refactor, implement web, caching, better tests and build files

* Update golang version to 1.24
* Update multiarch Dockerfile to be more ISA agnostic
* Refactor existing code and properly structure project into modules
* Get rid of global variables except where necessary (go:embed)
* Add default values to Config
* Add webserver with templates to finally correctly serve videos and gifs
* Add tiny caching library to decrease api load and improve latency
* Improve Webhook data preparation by filtering out redundant links
  from the tweet text and properly attaching videos and gifs in separate
  webhook request by utilising new webserver
* Improve tests for filter function
* Improve bake definition for easier CI integration

.dockerignore

@@ -3,3 +3,4 @@
 !**/go.mod
 !**/go.sum
 !**/*.go
+!**/*.html

.gitignore

@@ -1,4 +1,4 @@
 /config.toml
 /compose.yml
 /discord-tweeter
-/db
+/data

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22 AS build
+FROM golang:1.24 AS build
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends ca-certificates
@@ -17,4 +17,4 @@ COPY --from=build /tmp/app/discord-tweeter .
 #COPY ./config.example.toml ./config.toml
 
 ENTRYPOINT [ "./discord-tweeter" ]
-# CMD [ "./config.toml" ]
+#CMD [ "./config.toml" ]

Dockerfile.multiarch

@@ -1,8 +1,8 @@
-FROM --platform=linux/amd64 golang:1.22 AS build
+FROM --platform=$BUILDPLATFORM golang:1.24 AS build
 
 ARG TARGETOS TARGETARCH TARGETVARIANT
-ENV GOOS $TARGETOS
+ARG GOOS=$TARGETOS
-ENV GOARCH $TARGETARCH
+ARG GOARCH=$TARGETARCH
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends ca-certificates && \
@@ -10,8 +10,8 @@ RUN apt-get update && \
         "arm64")   apt-get install -y gcc-aarch64-linux-gnu ;; \
         "armv7")   apt-get install -y gcc-arm-linux-gnueabihf ;; \
         "riscv64") apt-get install -y gcc-riscv64-linux-gnu ;; \
-        "amd64")   apt-get install -y gcc ;; \
+        "amd64")   apt-get install -y gcc-x86-64-linux-gnu ;; \
-        *)         echo "Arch not supported" && exit 1 ;; \
+        *)         echo "Arch '$TARGETARCH$TARGETVARIANT' not supported" && exit 1 ;; \
     esac
 
 WORKDIR /tmp/app
@@ -21,14 +21,14 @@ RUN go mod download && \
         "arm64")   export CC="aarch64-linux-gnu-gcc" PIE=true ;; \
         "armv7")   export CC="arm-linux-gnueabihf-gcc" GOARM="7" ;; \
         "riscv64") export CC="riscv64-linux-gnu-gcc" ;; \
-        "amd64")   export CC="gcc" PIE=true ;; \
+        "amd64")   export CC="x86_64-linux-gnu-gcc" PIE=true ;; \
-        *)         echo "Arch not supported" && exit 1 ;; \
+        *)         echo "Arch '$TARGETARCH$TARGETVARIANT' not supported" && exit 1 ;; \
     esac && \
-    export CGO_ENABLED=1 && \
+    export CGO_ENABLED=1 LDFLAGS="-s -w -linkmode 'external'" && \
-    if [ $PIE = true ]; then \
+    if [ "$PIE" = "true" ]; then \
-        go build -a -buildmode=pie -trimpath -ldflags "-s -w -linkmode 'external' -extldflags '-static-pie' -X main.prefix=" -o discord-tweeter; \
+        go build -a -buildmode=pie -trimpath -ldflags "$LDFLAGS -extldflags '-static-pie' -X main.prefix=" -o discord-tweeter; \
     else \
-        go build -a -trimpath -ldflags "-s -w -linkmode 'external' -extldflags '-static' -X main.prefix=" -o discord-tweeter; \
+        go build -a -trimpath -ldflags "$LDFLAGS -extldflags '-static' -X main.prefix=" -o discord-tweeter; \
     fi
 
 FROM scratch
@@ -40,4 +40,4 @@ COPY --from=build /tmp/app/discord-tweeter .
 #COPY ./config.example.toml ./config.toml
 
 ENTRYPOINT [ "./discord-tweeter" ]
-# CMD [ "./config.toml" ]
+#CMD [ "./config.toml" ]

cmd/config.go

@@ -1,28 +0,0 @@
-package cmd
-
-import (
-	"github.com/BurntSushi/toml"
-	"os"
-)
-
-type Config struct {
-	Username   string
-	Password   string
-	ProxyAddr  string
-	Channels   []string
-	Filter     []uint8
-	Webhook    string
-	DbPath     string
-	CookiePath string
-}
-
-func ConfigFromFile(filePath string) (conf *Config, err error) {
-	tomlData, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = toml.Decode(string(tomlData), &conf)
-
-	return
-}

cmd/tweeter/embed.go

@@ -1,4 +1,4 @@
-package cmd
+package tweeter
 
 import (
 	"errors"
@@ -79,7 +79,7 @@ func (e *Embed) SetColor(color string) (*Embed, error) {
 	color = strings.Replace(color, "#", "", -1)
 	colorInt, err := strconv.ParseInt(color, 16, 64)
 	if err != nil {
-		return nil, errors.New("Invalid hex code passed")
+		return nil, errors.New("invalid hex code passed")
 	}
 	e.Color = colorInt
 	return e, nil

cmd/tweeter/tweeter.go

@@ -1,9 +1,8 @@
-package cmd
+package tweeter
 
 import (
 	"context"
 	"encoding/json"
-	ts "github.com/imperatrona/twitter-scraper"
 	"log"
 	"math/rand"
 	"net/http"
@@ -11,23 +10,23 @@ import (
 	"os/signal"
 	"syscall"
 	"time"
-)
 
-var (
+	"git.snrd.eu/sunred/discord-tweeter/pkg/config"
-	configPath = "./config.toml"
+	"git.snrd.eu/sunred/discord-tweeter/pkg/db"
-	dbPath     = "./db/tweets.db"
+	"git.snrd.eu/sunred/discord-tweeter/pkg/web"
-	cookiePath = "./db/cookies.json"
+	ts "github.com/imperatrona/twitter-scraper"
 )
 
 const (
 	ScrapeInterval int   = 3  // How often to check for new tweets (in minutes)
 	ScrapeDelay    int64 = 0  // How long to wait between api requests (in seconds)
 	ScrapeStep     int   = 10 // How many tweets to get at a time
+	DefaultConfig        = "./config.toml"
 )
 
 type App struct {
-	config  *Config
+	config  *config.Config
-	db      *Database
+	db      db.Database
 	scraper *ts.Scraper
 }
 
@@ -38,6 +37,7 @@ func Run() {
 		log.Fatalln("Too many arguments")
 	}
 
+	configPath := DefaultConfig
 	if len(args) == 1 {
 		if args[0] == "" {
 			log.Fatalln("No config path given")
@@ -45,7 +45,7 @@ func Run() {
 		configPath = args[0]
 	}
 
-	config, err := ConfigFromFile(configPath)
+	config, err := config.ConfigFromFile(configPath)
 	if err != nil {
 		log.Fatalf("There has been an error parsing config file '%s': %s\n", configPath, err.Error())
 	}
@@ -62,15 +62,11 @@ func Run() {
 		log.Fatalln("Webhook address cannot be empty")
 	}
 
-	if config.DbPath != "" {
+	if config.UseWebServer && config.HostURL == "" {
-		dbPath = config.DbPath
+		log.Fatalln("HostURL cannot be empty")
 	}
 
-	if config.CookiePath != "" {
+	db, dberr := db.New("sqlite3", config.DbPath)
-		cookiePath = config.CookiePath
-	}
-
-	db, dberr := NewDatabase("sqlite3", dbPath)
 	if dberr != nil {
 		log.Fatalf("An error occurred while creating database connection: %s\n", dberr.Error())
 	}
@@ -86,7 +82,7 @@ func Run() {
 	}
 
 	{
-		f, err := os.Open(cookiePath)
+		f, err := os.Open(config.CookiePath)
 		if err != nil {
 			log.Println("Cookie file does not yet exist")
 		} else {
@@ -105,14 +101,14 @@ func Run() {
 			if err != nil {
 				log.Fatalf("An error occurred during scraper login: %s\n", err.Error())
 			} else {
-				log.Printf("New Login - Saving cookies to %s\n", cookiePath)
+				log.Printf("New Login - Saving cookies to %s\n", config.CookiePath)
 				js, jsonErr := json.Marshal(scraper.GetCookies())
 				if jsonErr != nil {
 					log.Fatalf("An error occurred during cookie serialization: %s\n", jsonErr.Error())
 				}
-				f, fErr := os.Create(cookiePath)
+				f, fErr := os.Create(config.CookiePath)
 				if fErr != nil {
-					log.Fatalf("Failed to create cookie file at %s with the following error: %s\n", cookiePath, fErr.Error())
+					log.Fatalf("Failed to create cookie file at %s with the following error: %s\n", config.CookiePath, fErr.Error())
 				}
 				f.Write(js)
 				writeErr := f.Close()
@@ -132,10 +128,19 @@ func Run() {
 
 	scraper.WithDelay(ScrapeDelay)
 
-	app := App{config, db, scraper}
+	if config.UseWebServer {
+		log.Printf("Starting webserver on port %d", config.WebPort)
+		ws, err := web.New(config, scraper)
+		if err != nil {
+			log.Fatalf("An error occurred while starting webserver: %s\n", err.Error())
+		}
+		go ws.Server.ListenAndServe()
+	}
 
+	log.Printf("Starting main app with %d workers", len(config.Channels))
+	app := App{config, db, scraper}
 	for i, c := range config.Channels {
-		go app.queryX(i, c)
+		go app.queryLoop(i, c)
 	}
 
 	sigs := make(chan os.Signal, 1)
@@ -152,7 +157,7 @@ func Run() {
 	log.Println("Exiting...")
 }
 
-func (app *App) queryX(id int, channel string) {
+func (app App) queryLoop(id int, channel string) {
 	log.Printf("Starting worker %d for channel %s", id, channel)
 	// Sleep to stagger api queries of workers
 	time.Sleep(time.Duration(id) * time.Minute)
@@ -237,7 +242,7 @@ ScrapeLoop:
 			tweetsToPost = append(tweetsToPost, tweet)
 		}
 
-		sendToWebhook(app.config.Webhook, tweetsToPost)
+		app.SendToWebhook(tweetsToPost)
 		err := db.PruneOldestTweets(channel)
 		if err != nil {
 			log.Printf("Error while pruning old tweets for channel %s: %s", channel, err.Error())
@@ -246,6 +251,10 @@ ScrapeLoop:
 }
 
 func filterTweet(filter uint8, tweet *ts.Tweet) bool {
+	return filterByte(filter, tweet) > 0
+}
+
+func filterByte(filter uint8, tweet *ts.Tweet) uint8 {
 	var tweetFilter uint8 = 0
 	filterMap := []bool{tweet.IsSelfThread, tweet.IsRetweet, tweet.IsReply, tweet.IsPin, tweet.IsQuoted}
 	for _, f := range filterMap {
@@ -255,5 +264,5 @@ func filterTweet(filter uint8, tweet *ts.Tweet) bool {
 		}
 	}
 
-	return filter&tweetFilter > 0
+	return filter & tweetFilter
 }

cmd/tweeter/tweeter_test.go

@@ -0,0 +1,43 @@
+package tweeter
+
+import (
+	"testing"
+
+	ts "github.com/imperatrona/twitter-scraper"
+)
+
+func TestFilter(t *testing.T) {
+	filter1 := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b10111, 0b10101, 0b01101, 0b10101}
+	filter2 := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b01000, 0b01010, 0b01000, 0b11011}
+	filterR := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b00000, 0b00000, 0b01000, 0b10001}
+	filterB := []bool{true, true, false, true, true, true, false, false, true, true}
+
+	for i, filterBool := range filterB {
+		filterResult := filterR[i]
+
+		filterCheck := filter2[i]
+		filterQuoted := (filterCheck & 1) != 0      // first bit
+		filterPin := (filterCheck & 2) != 0         // second bit
+		filterReply := (filterCheck & 4) != 0       // third bit
+		filterRetweet := (filterCheck & 8) != 0     // fourth bit
+		filterSelfThread := (filterCheck & 16) != 0 // fifth bit
+
+		tweet := ts.Tweet{
+			IsSelfThread: filterSelfThread,
+			IsRetweet:    filterRetweet,
+			IsReply:      filterReply,
+			IsPin:        filterPin,
+			IsQuoted:     filterQuoted,
+		}
+
+		resultBool := filterTweet(filter1[i], &tweet)
+		if resultBool != filterBool {
+			t.Errorf("%b AND %b > 0 = %t; got %t\n", filter1[i], filter2[i], filterBool, resultBool)
+		}
+
+		resultByte := filterByte(filter1[i], &tweet)
+		if resultByte != filterResult {
+			t.Errorf("%b AND %b = %b; got %b\n", filter1[i], filter2[i], filterResult, resultByte)
+		}
+	}
+}

cmd/tweeter/webhook.go

@@ -1,17 +1,17 @@
-package cmd
+package tweeter
 
 import (
 	"bytes"
 	"encoding/json"
-	ts "github.com/imperatrona/twitter-scraper"
 	"log"
 	"net/http"
 	"strings"
+
+	ts "github.com/imperatrona/twitter-scraper"
 	//"strconv"
 )
 
 const (
-	BaseURL  = "https://twitter.com/"
 	BaseIcon = "https://abs.twimg.com/icons/apple-touch-icon-192x192.png"
 )
 
@@ -30,62 +30,66 @@ type Mention struct {
 	RepliedUser bool     `json:"replied_user,omitempty"`
 }
 
-func sendToWebhook(webhookURL string, tweets []*ts.Tweet) {
+func (app App) SendToWebhook(tweets []*ts.Tweet) {
 	for _, tweet := range tweets {
+		webhooksToSend := []*Webhook{}
 		data := Webhook{
-			Content: "<" + tweet.PermanentURL + ">",
 			Mention: &Mention{Parse: []string{"roles"}},
 		}
+		webhooksToSend = append(webhooksToSend, &data)
 
-		urlsToAppend := []string{}
+		if len(tweet.Videos) > 0 {
-		userUrl := BaseURL + tweet.Username
+			webhooksToSend = append(webhooksToSend, &Webhook{
+				Content: "[\u2800](" + app.config.HostURL + "/video/" + tweet.ID + ")",
+				Mention: &Mention{Parse: []string{"roles"}},
+			})
+		}
 
 		mainEmbed := data.NewEmbedWithURL(tweet.PermanentURL)
-		mainEmbed.SetAuthor(tweet.Name+" (@"+tweet.Username+")", userUrl, "https://unavatar.io/twitter/"+tweet.Username)
+		mainEmbed.SetAuthor(tweet.Name+" (@"+tweet.Username+")", app.config.HostURL+"/tweet/"+tweet.ID, app.config.HostURL+"/avatar/"+tweet.Username)
-		mainEmbed.SetText(tweet.Text)
 		mainEmbed.SetColor("#26a7de")
-		mainEmbed.SetFooter("Twitter", BaseIcon)
 		mainEmbed.SetTimestamp(tweet.TimeParsed)
+		//mainEmbed.SetFooter("Twitter", BaseIcon)
 		//mainEmbed.SetFooter("Twitter • <t:" + strconv.FormatInt((tweet.Timestamp), 10) + ":R>", BaseIcon)
 
+		tweetText := tweet.Text
 		for i, photo := range tweet.Photos {
 			embed := mainEmbed
 			if i > 0 {
 				embed = data.NewEmbedWithURL(tweet.PermanentURL)
 			}
 			embed.SetImage(photo.URL)
+			tweetText = strings.ReplaceAll(tweetText, photo.URL, "")
 		}
-		for i, gif := range tweet.GIFs {
+		for _, gif := range tweet.GIFs {
-			embed := mainEmbed
+			//embed := mainEmbed
-			if i > 0 {
+			//if i > 0 {
-				embed = data.NewEmbedWithURL(tweet.PermanentURL)
+			//	embed = data.NewEmbedWithURL(tweet.PermanentURL)
-			}
+			//}
-			embed.SetImage(gif.Preview)
+			//embed.SetImage(gif.Preview)
+			tweetText = strings.ReplaceAll(tweetText, gif.Preview, "")
+			tweetText = strings.ReplaceAll(tweetText, gif.URL, "")
 		}
-		for i, video := range tweet.Videos {
+		for _, video := range tweet.Videos {
-			embed := mainEmbed
+			//embed := mainEmbed
-			if i > 0 {
+			//if i > 0 {
-				embed = data.NewEmbedWithURL(tweet.PermanentURL)
+			//	embed = data.NewEmbedWithURL(tweet.PermanentURL)
-			}
+			//}
-			// Video embeds are not supported right now
+			//embed.SetImage(video.Preview)
-			embed.SetImage(video.Preview)
+			tweetText = strings.ReplaceAll(tweetText, video.Preview, "")
-			embed.SetVideo(video.URL) // This has sadly no effect
+			tweetText = strings.ReplaceAll(tweetText, video.URL, "")
-			urlsToAppend = append(urlsToAppend, strings.Replace(tweet.PermanentURL, "twitter", "fxtwitter", 1))
 		}
 
-		err := sendRequest(webhookURL, &data)
+		mainEmbed.SetText(strings.TrimSpace(tweetText))
-		if err != nil {
-			log.Println("Error while sending webhook for tweet %s: %s", tweet.ID, err.Error())
-			continue
-		}
 
-		for _, url := range urlsToAppend {
+		for _, data := range webhooksToSend {
-			err := sendRequest(webhookURL, &Webhook{Content: url})
+			err := sendRequest(app.config.Webhook, data)
 			if err != nil {
-				log.Println("Error while sending webhook for tweet %s: %s", tweet.ID, err.Error())
+				log.Printf("Error while sending webhook for tweet %s: %s", tweet.ID, err.Error())
 				continue
 			}
 		}
+
 	}
 }
 

cmd/tweeter_test.go

@@ -1,37 +0,0 @@
-package cmd
-
-import (
-	"testing"
-)
-
-func TestFilter(t *testing.T) {
-	filter1 := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b10111, 0b10101, 0b01101, 0b10101}
-	filter2 := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b01000, 0b01010, 0b01000, 0b11011}
-	filterR := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b00000, 0b00000, 0b01000, 0b10001}
-
-	for i, filterResult := range filterR {
-		resultByte := compareBytes(filter1[i], filter2[i])
-		if resultByte != filterResult {
-			t.Errorf("%b AND %b = %b; got %b\n", filter1[i], filter2[i], filterResult, resultByte)
-		}
-	}
-}
-
-func compareBytes(filter uint8, filterCheck uint8) uint8 {
-	filterQuoted := (filterCheck & 1) != 0      // first bit
-	filterPin := (filterCheck & 2) != 0         // second bit
-	filterReply := (filterCheck & 4) != 0       // third bit
-	filterRetweet := (filterCheck & 8) != 0     // fourth bit
-	filterSelfThread := (filterCheck & 16) != 0 // fifth bit
-
-	var tweetFilter uint8 = 0
-	filterMap := []bool{filterSelfThread, filterRetweet, filterReply, filterPin, filterQuoted}
-	for _, f := range filterMap {
-		tweetFilter <<= 1
-		if f {
-			tweetFilter |= 1
-		}
-	}
-
-	return filter & tweetFilter
-}

compose.example.yml

@@ -6,7 +6,10 @@ services:
     image: git.snrd.eu/sunred/discord-tweeter:latest
     container_name: discord-tweeter
     network_mode: bridge
+    ports:
+      - "127.0.0.1:8080:8080"
+      - "[::1]:8080:8080"
     volumes:
       - ./config.toml:/app/config.toml:ro
-      - ./db:/app/db:rw
+      - ./data:/app/data:rw
     restart: unless-stopped

config.example.toml

@@ -1,15 +1,16 @@
 username = ""
 password = "asd123"
 #proxyaddr = "socks5://localhost:5555"
+hostURL = "https://my.domain.tld"
 channels = [
-	"NinEverything",
+  "NinEverything",
-	"NintendoEurope",
+  "NintendoEurope",
-	"NintendoAmerica",
+  "NintendoAmerica",
 ]
 # Binary representation for efficient filtering
 # Bit from left to right (most to least significant bit): IsSelfThread, IsRetweet, IsReply, IsPin, IsQuoted
 filter = [
-	0b11111,
+  0b11111,
-	0b11111,
+  0b11111,
-	0b11111,
+  0b11111,
 ]

docker-bake.hcl

@@ -1,34 +1,44 @@
-variable "REG" {
+variable "IMAGE" {
-  default = "git.snrd.eu"
+  default = "git.snrd.eu/sunred/discord-tweeter"
-}
-variable "REPO" {
-  default = "sunred/discord-tweeter"
 }
 variable "TAG" {
   default = "latest"
 }
 
+function "generate_tags" {
+  params = [images, versions]
+  result = distinct(flatten(
+    [for i in split(",", images) :
+      [for v in split(",", versions) :
+        "${i}:${v}"
+      ]
+    ]))
+}
+
 group "default" {
   targets = ["prod"]
 }
 
 target "default" {
-  #platforms = [ "linux/amd64", "linux/arm64", "linux/arm/v7", "linux/riscv64" ]
+  tags = generate_tags(IMAGE, TAG)
-  platforms = [ "linux/amd64", "linux/arm64" ]
-  tags = ["${REG}/${REPO}:latest", "${REG}/${REPO}:${TAG}"]
-  attest = [
-    "type=provenance,disabled=true",
-    "type=sbom,disabled=true"
-  ]
 }
 target "prod" {
   inherits = ["default"]
-  #dockerfile = "Dockerfile"
+  //platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/riscv64"]
+  platforms = ["linux/amd64", "linux/arm64"]
   dockerfile = "Dockerfile.multiarch"
   output = ["type=registry"]
+  attest = [
+    { type = "provenance", mode = "max" },
+    { type = "sbom" }
+  ]
 }
 target "dev" {
   inherits = ["default"]
-  dockerfile = "Dockerfile.multiarch"
+  dockerfile = "Dockerfile"
-  output = ["type=image"]
+  output = ["type=docker"]
+  attest = [
+    { type = "provenance", disabled = "true" },
+    { type = "sbom", disabled = "true" }
+  ]
 }

go.mod

@@ -1,15 +1,17 @@
 module git.snrd.eu/sunred/discord-tweeter
 
-go 1.21
+go 1.23.0
+
+toolchain go1.24.1
 
 require (
-	github.com/BurntSushi/toml v1.3.2
+	github.com/BurntSushi/toml v1.5.0
-	github.com/imperatrona/twitter-scraper v0.0.0-20240425221339-715658ada4d9
+	github.com/imperatrona/twitter-scraper v0.0.16
 	github.com/jmoiron/sqlx v1.4.0
-	github.com/mattn/go-sqlite3 v1.14.22
+	github.com/mattn/go-sqlite3 v1.14.24
 )
 
 require (
 	github.com/AlexEidt/Vidio v1.5.1 // indirect
-	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/net v0.37.0 // indirect
 )

go.sum

@@ -1,48 +1,52 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/AlexEidt/Vidio v1.5.1 h1:tovwvtgQagUz1vifiL9OeWkg1fP/XUzFazFKh7tFtaE=
 github.com/AlexEidt/Vidio v1.5.1/go.mod h1:djhIMnWMqPrC3X6nB6ymGX6uWWlgw+VayYGKE1bNwmI=
-github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
-github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
-github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/imperatrona/twitter-scraper v0.0.0-20240425221339-715658ada4d9 h1:X/TQaFNGVNQXn33JnpQLBwQ+2Nd2P6x8EUEqMUI+xU4=
+github.com/imperatrona/twitter-scraper v0.0.16 h1:XrJZDGqr0/A7C3qRyoud9Ue9k6eUP10VkfRij3ewkSA=
-github.com/imperatrona/twitter-scraper v0.0.0-20240425221339-715658ada4d9/go.mod h1:+Z1pca7Faf2tzpHVRnRcratFxy/PuDKj2iaygdgZRLM=
+github.com/imperatrona/twitter-scraper v0.0.16/go.mod h1:38MY3g/h4V7Xl4HbW9lnkL8S3YiFZenBFv86hN57RG8=
-github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
-github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
 github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
 github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
-github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
+github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
+golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -50,22 +54,32 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"git.snrd.eu/sunred/discord-tweeter/cmd"
 	"log"
+
+	"git.snrd.eu/sunred/discord-tweeter/cmd/tweeter"
 )
 
 var (
@@ -11,5 +12,5 @@ var (
 
 func main() {
 	log.SetPrefix(prefix)
-	cmd.Run()
+	tweeter.Run()
 }

pkg/cache/cache.go

@@ -0,0 +1,52 @@
+package cache
+
+import (
+	"sync"
+	"time"
+)
+
+type Cache[K comparable, V any] struct {
+	data map[K]entry[K, V]
+	lock sync.RWMutex
+}
+
+type entry[K comparable, V any] struct {
+	value      V
+	expiration time.Time
+}
+
+func New[K comparable, V any]() *Cache[K, V] {
+	return &Cache[K, V]{
+		data: make(map[K]entry[K, V]),
+	}
+}
+
+func (c *Cache[K, V]) Set(key K, value V, ttl time.Duration) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.data[key] = entry[K, V]{
+		value:      value,
+		expiration: time.Now().Add(ttl),
+	}
+}
+
+func (c *Cache[K, V]) Get(key K) (V, bool) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	entry, ok := c.data[key]
+	if !ok || time.Now().After(entry.expiration) {
+		delete(c.data, key)
+		var zero V
+		return zero, false
+	}
+
+	return entry.value, true
+}
+
+func (c *Cache[K, V]) Delete(key K) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	delete(c.data, key)
+}

pkg/cache/cache_test.go

@@ -0,0 +1,145 @@
+package cache
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestCacheBasicOperations(t *testing.T) {
+	cache := New[string, int]()
+
+	// Test Set and Get
+	cache.Set("key1", 42, 10*time.Second)
+	value, exists := cache.Get("key1")
+	if !exists {
+		t.Errorf("expected key to exist")
+	}
+	if value != 42 {
+		t.Errorf("expected value 42, got %v", value)
+	}
+
+	// Test non-existent key
+	value, exists = cache.Get("nonexistent")
+	if exists {
+		t.Errorf("expected key to not exist")
+	}
+	if value != 0 {
+		t.Errorf("expected zero value, got %v", value)
+	}
+}
+
+func TestCacheExpiration(t *testing.T) {
+	cache := New[string, int]()
+
+	// Set value with very short TTL
+	cache.Set("short-lived", 42, 100*time.Millisecond)
+
+	// Verify initial existence
+	value, exists := cache.Get("short-lived")
+	if !exists {
+		t.Errorf("expected key to exist initially")
+	}
+	if value != 42 {
+		t.Errorf("expected value 42, got %v", value)
+	}
+
+	// Wait for expiration
+	time.Sleep(150 * time.Millisecond)
+
+	// Verify expiration
+	value, exists = cache.Get("short-lived")
+	if exists {
+		t.Errorf("expected key to have expired")
+	}
+	if value != 0 {
+		t.Errorf("expected zero value, got %v", value)
+	}
+}
+
+func TestCacheConcurrentAccess(t *testing.T) {
+	cache := New[string, int]()
+
+	// Start multiple writers
+	var wg sync.WaitGroup
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			cache.Set(fmt.Sprintf("key%d", i), i*2, 10*time.Second)
+		}(i)
+	}
+
+	// Start multiple readers
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			cache.Get(fmt.Sprintf("key%d", i))
+		}(i)
+	}
+
+	wg.Wait()
+
+	// Verify all values were written correctly
+	for i := 0; i < 10; i++ {
+		value, exists := cache.Get(fmt.Sprintf("key%d", i))
+		if !exists {
+			t.Errorf("expected key%d to exist", i)
+		}
+		if value != (i * 2) {
+			t.Errorf("expected value %d for key%d, got %d", i*2, i, value)
+		}
+	}
+}
+
+func TestCacheGenericTypes(t *testing.T) {
+	// Test with string values
+	strCache := New[int, string]()
+	strCache.Set(1, "hello", 10*time.Second)
+	value, exists := strCache.Get(1)
+	if !exists {
+		t.Errorf("expected key to exist")
+	}
+	if value != "hello" {
+		t.Errorf("expected value hello, got %v", value)
+	}
+
+	// Test with struct values
+	type Person struct {
+		Name string
+		Age  int
+	}
+	personCache := New[string, Person]()
+	personCache.Set("john", Person{"John", 30}, 10*time.Second)
+	person, exists := personCache.Get("john")
+	if !exists {
+		t.Errorf("expected key to exist")
+	}
+	expected := Person{"John", 30}
+	if !reflect.DeepEqual(person, expected) {
+		t.Errorf("expected %+v, got %+v", expected, person)
+	}
+}
+
+func TestCacheDelete(t *testing.T) {
+	cache := New[string, int]()
+
+	// Set and verify
+	cache.Set("key1", 42, 10*time.Second)
+	_, exists := cache.Get("key1")
+	if !exists {
+		t.Errorf("expected key to exist")
+	}
+
+	// Delete
+	cache.Delete("key1")
+
+	// Verify deletion
+	_, exists = cache.Get("key1")
+	if exists {
+		t.Errorf("expected key to not exist after deletion")
+	}
+}

pkg/config/config.go

@@ -0,0 +1,44 @@
+package config
+
+import (
+	"os"
+
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+	Username     string
+	Password     string
+	ProxyAddr    string
+	Channels     []string
+	Filter       []uint8
+	Webhook      string
+	DbPath       string
+	CookiePath   string
+	UseWebServer bool
+	HostURL      string
+	WebPort      uint16
+	UserAgents   []string
+	NitterBase   string
+}
+
+func ConfigFromFile(filePath string) (*Config, error) {
+	conf := &Config{
+		// Default values
+		DbPath:       "./data/tweets.db",
+		CookiePath:   "./data/cookies.json",
+		UseWebServer: true,
+		WebPort:      8080,
+		UserAgents:   []string{"discordbot", "curl", "httpie", "lwp-request", "wget", "python-requests", "openbsd ftp", "powershell"},
+		NitterBase:   "https://xcancel.com",
+	}
+
+	tomlData, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = toml.Decode(string(tomlData), conf)
+
+	return conf, err
+}

pkg/db/database.go

@@ -1,12 +1,12 @@
-package cmd
+package db
 
 import (
-	"errors"
 	"fmt"
+	"strconv"
+
+	ts "github.com/imperatrona/twitter-scraper"
 	"github.com/jmoiron/sqlx"
 	_ "github.com/mattn/go-sqlite3"
-	ts "github.com/imperatrona/twitter-scraper"
-	"strconv"
 )
 
 const (
@@ -32,7 +32,7 @@ type Database struct {
 	*sqlx.DB
 }
 
-func NewDatabase(driver string, connectString string) (*Database, error) {
+func New(driver string, connectString string) (Database, error) {
 	var connection *sqlx.DB
 	var err error
 
@@ -40,22 +40,22 @@ func NewDatabase(driver string, connectString string) (*Database, error) {
 	case "sqlite3":
 		connection, err = sqlx.Connect(driver, "file:"+connectString+"?cache=shared")
 		if err != nil {
-			return nil, err
+			return Database{}, err
 		}
 		connection.SetMaxOpenConns(1)
 
 		if _, err = connection.Exec(SqliteSchema); err != nil {
-			return nil, err
+			return Database{}, err
 		}
 
 	default:
-		return nil, errors.New(fmt.Sprintf("Database driver %s not supported right now!", driver))
+		return Database{}, fmt.Errorf("database driver %s not supported right now", driver)
 	}
 
-	return &Database{connection}, err
+	return Database{connection}, err
 }
 
-func (db *Database) GetNewestTweet(channel string) (*Tweet, error) {
+func (db Database) GetNewestTweet(channel string) (*Tweet, error) {
 	tweet := Tweet{}
 	err := db.Get(&tweet, "SELECT * FROM tweet WHERE channel=$1 ORDER BY timestamp DESC, snowflake DESC LIMIT 1", channel)
 	if err != nil {
@@ -64,7 +64,7 @@ func (db *Database) GetNewestTweet(channel string) (*Tweet, error) {
 	return &tweet, nil
 }
 
-func (db *Database) GetTweets(channel string) ([]*Tweet, error) {
+func (db Database) GetTweets(channel string) ([]*Tweet, error) {
 	tweet := []*Tweet{}
 	err := db.Select(&tweet, "SELECT * FROM tweet WHERE channel=$1 ORDER BY timestamp DESC, snowflake DESC", channel)
 	if err != nil {
@@ -73,7 +73,7 @@ func (db *Database) GetTweets(channel string) ([]*Tweet, error) {
 	return tweet, nil
 }
 
-func (db *Database) ContainsTweet(channel string, tweet *ts.Tweet) (bool, error) {
+func (db Database) ContainsTweet(channel string, tweet *ts.Tweet) (bool, error) {
 	snowflake, err := strconv.ParseUint(tweet.ID, 10, 64)
 	if err != nil {
 		return false, err
@@ -97,7 +97,7 @@ func (db *Database) ContainsTweet(channel string, tweet *ts.Tweet) (bool, error)
 	return false, nil
 }
 
-func (db *Database) InsertTweet(channel string, tweet *ts.Tweet) error {
+func (db Database) InsertTweet(channel string, tweet *ts.Tweet) error {
 	snowflake, err := strconv.ParseUint(tweet.ID, 10, 64)
 	if err != nil {
 		return err
@@ -111,7 +111,7 @@ func (db *Database) InsertTweet(channel string, tweet *ts.Tweet) error {
 	return nil
 }
 
-func (db *Database) PruneOldestTweets(channel string) error {
+func (db Database) PruneOldestTweets(channel string) error {
 	var count int
 	err := db.Get(&count, "SELECT COUNT(*) FROM tweet WHERE channel=$1", channel)
 	if err != nil {
@@ -157,7 +157,7 @@ func FromTweet(channel string, tweet *ts.Tweet) (*Tweet, error) {
 	return &Tweet{0, snowflake, channel, tweet.Timestamp}, nil
 }
 
-func (t *Tweet) EqualsTweet(tweet *ts.Tweet) bool {
+func (t Tweet) EqualsTweet(tweet *ts.Tweet) bool {
 	snowflake, err := strconv.ParseUint(tweet.ID, 10, 64)
 	if err != nil {
 		return false
@@ -166,6 +166,6 @@ func (t *Tweet) EqualsTweet(tweet *ts.Tweet) bool {
 	return t.Snowflake == snowflake
 }
 
-func (t *Tweet) Equals(tweet *Tweet) bool {
+func (t Tweet) Equals(tweet *Tweet) bool {
 	return t.Snowflake == tweet.Snowflake
 }

pkg/db/database_test.go

@@ -1,13 +1,14 @@
-package cmd
+package db
 
 import (
+	"testing"
+
 	"github.com/jmoiron/sqlx"
 	_ "github.com/mattn/go-sqlite3"
-	"testing"
 )
 
 const (
-	testDbPath = "../db/testdb.db"
+	testDbPath = "../data/testdb.db"
 )
 
 var (

pkg/web/templates/tweet.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="UTF-8">
+	<meta name="theme-color" content="#26a7de">
+	<link rel="canonical" href="{{ .URL }}">
+
+	<meta property="twitter:site" content="{{ .Username }}">
+	<meta property="twitter:creator" content="{{ .Username }}">
+	<meta property="twitter:title" content="{{ .Title }}">
+{{- range $idx, $e := .Images }}
+	<meta property="twitter:image" content="{{ $e }}">
+	<meta property="og:image" content="{{ $e }}">
+{{- end }}
+{{- range $idx, $e := .Videos }}
+	<meta property="twitter:player:stream" content="{{ $e }}">
+	<meta property="og:video" content="{{ $e }}">
+{{- end }}
+{{- range $idx, $e := .Previews }}
+	<meta property="twitter:image" content="0">
+	<meta property="og:image" content="{{ $e }}">
+{{- end }}
+	<meta property="twitter:card" content="{{ .Format }}">
+	<meta property="og:type" content="article">
+	<meta property="og:article:published_time" content="{{ .Timestamp }}">
+	<meta property="og:url" content="{{ .Link }}">
+	<meta property="og:title" content="{{ .Title }}">
+	<meta property="og:description" content="{{ .Text }}">
+
+	<style>
+		body, html { margin: 0; padding: 0; height: 100%; width: 100%; }
+		body { display: flex; }
+	</style>
+
+<body>

pkg/web/templates/video.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="UTF-8">
+	<meta name="theme-color" content="#26a7de">
+	<link rel="canonical" href="{{ .URL }}">
+
+{{- if .Videos }}
+	<meta property="twitter:site" content="{{ .Username }}">
+	<meta property="twitter:creator" content="{{ .Username }}">
+{{- range $idx, $e := .Videos }}
+	<meta property="twitter:player:stream" content="{{ $e }}">
+	<meta property="og:video" content="{{ $e }}">
+{{- end }}
+{{- range $idx, $e := .Previews }}
+	<meta property="twitter:image" content="0">
+	<meta property="og:image" content="{{ $e }}">
+{{- end }}
+	<meta property="twitter:card" content="{{ .Format }}">
+	<meta property="og:site_name" content="Video attached due to embed limitations">
+	<meta property="og:type" content="article">
+	<meta property="og:article:published_time" content="{{ .Timestamp }}">
+	<meta property="og:url" content="{{ .Link }}">
+	<meta property="og:title" content="&#x200B;">
+{{- end }}
+
+	<style>
+		body, html { margin: 0; padding: 0; height: 100%; width: 100%; }
+		body { display: flex; }
+	</style>
+
+<body>

pkg/web/web.go

@@ -0,0 +1,211 @@
+package web
+
+import (
+	"bytes"
+	"embed"
+	"fmt"
+	"html/template"
+	"net/http"
+	"slices"
+	"strings"
+	"time"
+
+	"git.snrd.eu/sunred/discord-tweeter/pkg/cache"
+	"git.snrd.eu/sunred/discord-tweeter/pkg/config"
+	ts "github.com/imperatrona/twitter-scraper"
+)
+
+const (
+	AvatarCacheTime = 60 * time.Minute // How long to serve same avatar from cache before doing a fresh scrape
+	TweetCacheTime  = 10 * time.Minute // How long to serve same tweet from cache before doing a fresh scrape
+)
+
+//go:embed templates/*.html
+var templateFiles embed.FS
+
+type Reponse struct {
+	StatusCode  int
+	ContentType string
+	Content     string
+}
+
+type WebServer struct {
+	config        *config.Config
+	scraper       *ts.Scraper
+	templates     *template.Template
+	avatarCache   *cache.Cache[string, string]
+	responseCache *cache.Cache[string, Reponse]
+
+	Server *http.Server
+}
+
+type Tweet struct {
+	URL       string
+	Link      string
+	Title     string
+	Text      string
+	Username  string
+	Timestamp string
+	Format    string
+	Images    []string
+	Videos    []string
+	Previews  []string
+}
+
+func New(config *config.Config, scraper *ts.Scraper) (*WebServer, error) {
+	sm := http.NewServeMux()
+	tmpl, err := template.ParseFS(templateFiles, "templates/*.html")
+	if err != nil {
+		return nil, err
+	}
+
+	ws := &WebServer{
+		config,
+		scraper,
+		tmpl,
+		cache.New[string, string](),
+		cache.New[string, Reponse](),
+		&http.Server{
+			Handler:      sm,
+			Addr:         fmt.Sprintf(":%d", config.WebPort),
+			ReadTimeout:  30 * time.Second,
+			WriteTimeout: 30 * time.Second,
+		},
+	}
+	sm.HandleFunc("GET /avatar/{username}", ws.handleAvatar)
+	sm.HandleFunc("GET /tweet/{id}", ws.handleTweet)
+	sm.HandleFunc("GET /video/{id}", ws.handleVideo)
+	return ws, nil
+}
+
+func (ws WebServer) handleAvatar(w http.ResponseWriter, r *http.Request) {
+	username := r.PathValue("username")
+
+	if !slices.Contains(ws.config.Channels, username) {
+		badRequest(w)
+		return
+	}
+
+	url, cached := ws.avatarCache.Get("avatar-" + username)
+	if cached {
+		http.Redirect(w, r, url, http.StatusPermanentRedirect)
+		return
+	}
+
+	profile, err := ws.scraper.GetProfile(username)
+	if err != nil {
+		serverError(w)
+		return
+	}
+
+	ws.avatarCache.Set("avatar-"+username, profile.Avatar, AvatarCacheTime)
+
+	http.Redirect(w, r, profile.Avatar, http.StatusPermanentRedirect)
+}
+
+func (ws WebServer) handleTweet(w http.ResponseWriter, r *http.Request) {
+	ws.handleTemplate(w, r, r.PathValue("id"), "tweet")
+}
+
+func (ws WebServer) handleVideo(w http.ResponseWriter, r *http.Request) {
+	ws.handleTemplate(w, r, r.PathValue("id"), "video")
+}
+
+func (ws WebServer) handleTemplate(w http.ResponseWriter, r *http.Request, id string, template string) {
+	if !validUserAgent(r.UserAgent(), ws.config.UserAgents) {
+		http.Redirect(w, r, ws.config.NitterBase+"/i/status/"+id, http.StatusPermanentRedirect)
+		return
+	}
+
+	entry, cached := ws.responseCache.Get(template + "-" + id)
+	if cached {
+		response(w, entry)
+		return
+	}
+
+	tweet, err := ws.scraper.GetTweet(id)
+	if err != nil {
+		serverError(w)
+		return
+	}
+
+	if !slices.Contains(ws.config.Channels, tweet.Username) {
+		res := Reponse{http.StatusBadRequest, "text/plain", "Bad Request"}
+		ws.responseCache.Set(template+"-"+id, res, TweetCacheTime)
+		response(w, res)
+		return
+	}
+
+	tweetText := tweet.Text
+	var images []string
+	var videos []string
+	var previews []string
+
+	for _, photo := range tweet.Photos {
+		images = append(images, photo.URL)
+		tweetText = strings.ReplaceAll(tweetText, photo.URL, "")
+	}
+	for _, gif := range tweet.GIFs {
+		videos = append(images, gif.URL)
+		previews = append(images, gif.Preview)
+		tweetText = strings.ReplaceAll(tweetText, gif.URL, "")
+		tweetText = strings.ReplaceAll(tweetText, gif.Preview, "")
+	}
+	for _, video := range tweet.Videos {
+		videos = append(videos, video.URL)
+		previews = append(previews, video.Preview)
+		tweetText = strings.ReplaceAll(tweetText, video.URL, "")
+		tweetText = strings.ReplaceAll(tweetText, video.Preview, "")
+	}
+
+	format := "summary_large_image"
+	if len(videos) > 0 {
+		format = "player"
+	}
+
+	data := Tweet{
+		URL:       tweet.PermanentURL,
+		Link:      ws.config.NitterBase + "/" + tweet.Username + "/status/" + id,
+		Title:     tweet.Name + " (@" + tweet.Username + ")",
+		Text:      strings.TrimSpace(tweetText),
+		Username:  tweet.Username,
+		Timestamp: tweet.TimeParsed.Format(time.RFC3339),
+		Format:    format,
+		Images:    images,
+		Videos:    videos,
+		Previews:  previews,
+	}
+
+	var tpl bytes.Buffer
+	if err := ws.templates.ExecuteTemplate(&tpl, template+".html", data); err != nil {
+		serverError(w)
+		return
+	}
+
+	res := Reponse{http.StatusOK, "text/html", tpl.String()}
+	ws.responseCache.Set(template+"-"+id, res, TweetCacheTime)
+	response(w, res)
+}
+
+func validUserAgent(ua string, uas []string) bool {
+	for _, s := range uas {
+		if s == "*" || strings.Contains(strings.ToLower(ua), s) {
+			return true
+		}
+	}
+	return false
+}
+
+func badRequest(w http.ResponseWriter) {
+	response(w, Reponse{http.StatusBadRequest, "text/plain", "Bad Request"})
+}
+
+func serverError(w http.ResponseWriter) {
+	response(w, Reponse{http.StatusInternalServerError, "text/plain", "Internal Server Error"})
+}
+
+func response(w http.ResponseWriter, res Reponse) {
+	w.WriteHeader(res.StatusCode)
+	w.Header().Set("Content-Type", res.ContentType)
+	fmt.Fprint(w, res.Content)
+}