diff --git a/.dockerignore b/.dockerignore index 38c978b..b9b2066 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,4 @@ !**/go.mod !**/go.sum !**/*.go +!**/*.html diff --git a/.gitignore b/.gitignore index 9899a81..b396970 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ /config.toml /compose.yml /discord-tweeter -/db +/data diff --git a/Dockerfile b/Dockerfile index 16f0ac7..894046f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22 AS build +FROM golang:1.24 AS build RUN apt-get update && \ apt-get install -y --no-install-recommends ca-certificates @@ -17,4 +17,4 @@ COPY --from=build /tmp/app/discord-tweeter . #COPY ./config.example.toml ./config.toml ENTRYPOINT [ "./discord-tweeter" ] -# CMD [ "./config.toml" ] +#CMD [ "./config.toml" ] diff --git a/Dockerfile.multiarch b/Dockerfile.multiarch index 6c9a364..4925a2b 100644 --- a/Dockerfile.multiarch +++ b/Dockerfile.multiarch @@ -1,8 +1,8 @@ -FROM --platform=linux/amd64 golang:1.22 AS build +FROM --platform=$BUILDPLATFORM golang:1.24 AS build ARG TARGETOS TARGETARCH TARGETVARIANT -ENV GOOS $TARGETOS -ENV GOARCH $TARGETARCH +ARG GOOS=$TARGETOS +ARG GOARCH=$TARGETARCH RUN apt-get update && \ apt-get install -y --no-install-recommends ca-certificates && \ @@ -10,8 +10,8 @@ RUN apt-get update && \ "arm64") apt-get install -y gcc-aarch64-linux-gnu ;; \ "armv7") apt-get install -y gcc-arm-linux-gnueabihf ;; \ "riscv64") apt-get install -y gcc-riscv64-linux-gnu ;; \ - "amd64") apt-get install -y gcc ;; \ - *) echo "Arch not supported" && exit 1 ;; \ + "amd64") apt-get install -y gcc-x86-64-linux-gnu ;; \ + *) echo "Arch '$TARGETARCH$TARGETVARIANT' not supported" && exit 1 ;; \ esac WORKDIR /tmp/app @@ -21,14 +21,14 @@ RUN go mod download && \ "arm64") export CC="aarch64-linux-gnu-gcc" PIE=true ;; \ "armv7") export CC="arm-linux-gnueabihf-gcc" GOARM="7" ;; \ "riscv64") export CC="riscv64-linux-gnu-gcc" ;; \ - "amd64") export CC="gcc" PIE=true ;; \ - *) echo "Arch not supported" && exit 1 ;; \ + "amd64") export CC="x86_64-linux-gnu-gcc" PIE=true ;; \ + *) echo "Arch '$TARGETARCH$TARGETVARIANT' not supported" && exit 1 ;; \ esac && \ - export CGO_ENABLED=1 && \ - if [ $PIE = true ]; then \ - go build -a -buildmode=pie -trimpath -ldflags "-s -w -linkmode 'external' -extldflags '-static-pie' -X main.prefix=" -o discord-tweeter; \ + export CGO_ENABLED=1 LDFLAGS="-s -w -linkmode 'external'" && \ + if [ "$PIE" = "true" ]; then \ + go build -a -buildmode=pie -trimpath -ldflags "$LDFLAGS -extldflags '-static-pie' -X main.prefix=" -o discord-tweeter; \ else \ - go build -a -trimpath -ldflags "-s -w -linkmode 'external' -extldflags '-static' -X main.prefix=" -o discord-tweeter; \ + go build -a -trimpath -ldflags "$LDFLAGS -extldflags '-static' -X main.prefix=" -o discord-tweeter; \ fi FROM scratch @@ -40,4 +40,4 @@ COPY --from=build /tmp/app/discord-tweeter . #COPY ./config.example.toml ./config.toml ENTRYPOINT [ "./discord-tweeter" ] -# CMD [ "./config.toml" ] +#CMD [ "./config.toml" ] diff --git a/cmd/config.go b/cmd/config.go deleted file mode 100644 index 276f686..0000000 --- a/cmd/config.go +++ /dev/null @@ -1,28 +0,0 @@ -package cmd - -import ( - "github.com/BurntSushi/toml" - "os" -) - -type Config struct { - Username string - Password string - ProxyAddr string - Channels []string - Filter []uint8 - Webhook string - DbPath string - CookiePath string -} - -func ConfigFromFile(filePath string) (conf *Config, err error) { - tomlData, err := os.ReadFile(filePath) - if err != nil { - return nil, err - } - - _, err = toml.Decode(string(tomlData), &conf) - - return -} diff --git a/cmd/embed.go b/cmd/tweeter/embed.go similarity index 97% rename from cmd/embed.go rename to cmd/tweeter/embed.go index 7740004..82cfa28 100644 --- a/cmd/embed.go +++ b/cmd/tweeter/embed.go @@ -1,4 +1,4 @@ -package cmd +package tweeter import ( "errors" @@ -79,7 +79,7 @@ func (e *Embed) SetColor(color string) (*Embed, error) { color = strings.Replace(color, "#", "", -1) colorInt, err := strconv.ParseInt(color, 16, 64) if err != nil { - return nil, errors.New("Invalid hex code passed") + return nil, errors.New("invalid hex code passed") } e.Color = colorInt return e, nil diff --git a/cmd/tweeter.go b/cmd/tweeter/tweeter.go similarity index 81% rename from cmd/tweeter.go rename to cmd/tweeter/tweeter.go index aedcc9c..e69f885 100644 --- a/cmd/tweeter.go +++ b/cmd/tweeter/tweeter.go @@ -1,9 +1,8 @@ -package cmd +package tweeter import ( "context" "encoding/json" - ts "github.com/imperatrona/twitter-scraper" "log" "math/rand" "net/http" @@ -11,23 +10,23 @@ import ( "os/signal" "syscall" "time" -) -var ( - configPath = "./config.toml" - dbPath = "./db/tweets.db" - cookiePath = "./db/cookies.json" + "git.snrd.eu/sunred/discord-tweeter/pkg/config" + "git.snrd.eu/sunred/discord-tweeter/pkg/db" + "git.snrd.eu/sunred/discord-tweeter/pkg/web" + ts "github.com/imperatrona/twitter-scraper" ) const ( ScrapeInterval int = 3 // How often to check for new tweets (in minutes) ScrapeDelay int64 = 0 // How long to wait between api requests (in seconds) ScrapeStep int = 10 // How many tweets to get at a time + DefaultConfig = "./config.toml" ) type App struct { - config *Config - db *Database + config *config.Config + db db.Database scraper *ts.Scraper } @@ -38,6 +37,7 @@ func Run() { log.Fatalln("Too many arguments") } + configPath := DefaultConfig if len(args) == 1 { if args[0] == "" { log.Fatalln("No config path given") @@ -45,7 +45,7 @@ func Run() { configPath = args[0] } - config, err := ConfigFromFile(configPath) + config, err := config.ConfigFromFile(configPath) if err != nil { log.Fatalf("There has been an error parsing config file '%s': %s\n", configPath, err.Error()) } @@ -62,15 +62,11 @@ func Run() { log.Fatalln("Webhook address cannot be empty") } - if config.DbPath != "" { - dbPath = config.DbPath + if config.UseWebServer && config.HostURL == "" { + log.Fatalln("HostURL cannot be empty") } - if config.CookiePath != "" { - cookiePath = config.CookiePath - } - - db, dberr := NewDatabase("sqlite3", dbPath) + db, dberr := db.New("sqlite3", config.DbPath) if dberr != nil { log.Fatalf("An error occurred while creating database connection: %s\n", dberr.Error()) } @@ -86,7 +82,7 @@ func Run() { } { - f, err := os.Open(cookiePath) + f, err := os.Open(config.CookiePath) if err != nil { log.Println("Cookie file does not yet exist") } else { @@ -105,14 +101,14 @@ func Run() { if err != nil { log.Fatalf("An error occurred during scraper login: %s\n", err.Error()) } else { - log.Printf("New Login - Saving cookies to %s\n", cookiePath) + log.Printf("New Login - Saving cookies to %s\n", config.CookiePath) js, jsonErr := json.Marshal(scraper.GetCookies()) if jsonErr != nil { log.Fatalf("An error occurred during cookie serialization: %s\n", jsonErr.Error()) } - f, fErr := os.Create(cookiePath) + f, fErr := os.Create(config.CookiePath) if fErr != nil { - log.Fatalf("Failed to create cookie file at %s with the following error: %s\n", cookiePath, fErr.Error()) + log.Fatalf("Failed to create cookie file at %s with the following error: %s\n", config.CookiePath, fErr.Error()) } f.Write(js) writeErr := f.Close() @@ -132,10 +128,19 @@ func Run() { scraper.WithDelay(ScrapeDelay) - app := App{config, db, scraper} + if config.UseWebServer { + log.Printf("Starting webserver on port %d", config.WebPort) + ws, err := web.New(config, scraper) + if err != nil { + log.Fatalf("An error occurred while starting webserver: %s\n", err.Error()) + } + go ws.Server.ListenAndServe() + } + log.Printf("Starting main app with %d workers", len(config.Channels)) + app := App{config, db, scraper} for i, c := range config.Channels { - go app.queryX(i, c) + go app.queryLoop(i, c) } sigs := make(chan os.Signal, 1) @@ -152,7 +157,7 @@ func Run() { log.Println("Exiting...") } -func (app *App) queryX(id int, channel string) { +func (app App) queryLoop(id int, channel string) { log.Printf("Starting worker %d for channel %s", id, channel) // Sleep to stagger api queries of workers time.Sleep(time.Duration(id) * time.Minute) @@ -237,7 +242,7 @@ ScrapeLoop: tweetsToPost = append(tweetsToPost, tweet) } - sendToWebhook(app.config.Webhook, tweetsToPost) + app.SendToWebhook(tweetsToPost) err := db.PruneOldestTweets(channel) if err != nil { log.Printf("Error while pruning old tweets for channel %s: %s", channel, err.Error()) @@ -246,6 +251,10 @@ ScrapeLoop: } func filterTweet(filter uint8, tweet *ts.Tweet) bool { + return filterByte(filter, tweet) > 0 +} + +func filterByte(filter uint8, tweet *ts.Tweet) uint8 { var tweetFilter uint8 = 0 filterMap := []bool{tweet.IsSelfThread, tweet.IsRetweet, tweet.IsReply, tweet.IsPin, tweet.IsQuoted} for _, f := range filterMap { @@ -255,5 +264,5 @@ func filterTweet(filter uint8, tweet *ts.Tweet) bool { } } - return filter&tweetFilter > 0 + return filter & tweetFilter } diff --git a/cmd/tweeter/tweeter_test.go b/cmd/tweeter/tweeter_test.go new file mode 100644 index 0000000..ccd5fe5 --- /dev/null +++ b/cmd/tweeter/tweeter_test.go @@ -0,0 +1,43 @@ +package tweeter + +import ( + "testing" + + ts "github.com/imperatrona/twitter-scraper" +) + +func TestFilter(t *testing.T) { + filter1 := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b10111, 0b10101, 0b01101, 0b10101} + filter2 := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b01000, 0b01010, 0b01000, 0b11011} + filterR := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b00000, 0b00000, 0b01000, 0b10001} + filterB := []bool{true, true, false, true, true, true, false, false, true, true} + + for i, filterBool := range filterB { + filterResult := filterR[i] + + filterCheck := filter2[i] + filterQuoted := (filterCheck & 1) != 0 // first bit + filterPin := (filterCheck & 2) != 0 // second bit + filterReply := (filterCheck & 4) != 0 // third bit + filterRetweet := (filterCheck & 8) != 0 // fourth bit + filterSelfThread := (filterCheck & 16) != 0 // fifth bit + + tweet := ts.Tweet{ + IsSelfThread: filterSelfThread, + IsRetweet: filterRetweet, + IsReply: filterReply, + IsPin: filterPin, + IsQuoted: filterQuoted, + } + + resultBool := filterTweet(filter1[i], &tweet) + if resultBool != filterBool { + t.Errorf("%b AND %b > 0 = %t; got %t\n", filter1[i], filter2[i], filterBool, resultBool) + } + + resultByte := filterByte(filter1[i], &tweet) + if resultByte != filterResult { + t.Errorf("%b AND %b = %b; got %b\n", filter1[i], filter2[i], filterResult, resultByte) + } + } +} diff --git a/cmd/webhook.go b/cmd/tweeter/webhook.go similarity index 51% rename from cmd/webhook.go rename to cmd/tweeter/webhook.go index 3fddab0..0c69601 100644 --- a/cmd/webhook.go +++ b/cmd/tweeter/webhook.go @@ -1,17 +1,17 @@ -package cmd +package tweeter import ( "bytes" "encoding/json" - ts "github.com/imperatrona/twitter-scraper" "log" "net/http" "strings" + + ts "github.com/imperatrona/twitter-scraper" //"strconv" ) const ( - BaseURL = "https://twitter.com/" BaseIcon = "https://abs.twimg.com/icons/apple-touch-icon-192x192.png" ) @@ -30,62 +30,66 @@ type Mention struct { RepliedUser bool `json:"replied_user,omitempty"` } -func sendToWebhook(webhookURL string, tweets []*ts.Tweet) { +func (app App) SendToWebhook(tweets []*ts.Tweet) { for _, tweet := range tweets { + webhooksToSend := []*Webhook{} data := Webhook{ - Content: "<" + tweet.PermanentURL + ">", Mention: &Mention{Parse: []string{"roles"}}, } + webhooksToSend = append(webhooksToSend, &data) - urlsToAppend := []string{} - userUrl := BaseURL + tweet.Username + if len(tweet.Videos) > 0 { + webhooksToSend = append(webhooksToSend, &Webhook{ + Content: "[\u2800](" + app.config.HostURL + "/video/" + tweet.ID + ")", + Mention: &Mention{Parse: []string{"roles"}}, + }) + } mainEmbed := data.NewEmbedWithURL(tweet.PermanentURL) - mainEmbed.SetAuthor(tweet.Name+" (@"+tweet.Username+")", userUrl, "https://unavatar.io/twitter/"+tweet.Username) - mainEmbed.SetText(tweet.Text) + mainEmbed.SetAuthor(tweet.Name+" (@"+tweet.Username+")", app.config.HostURL+"/tweet/"+tweet.ID, app.config.HostURL+"/avatar/"+tweet.Username) mainEmbed.SetColor("#26a7de") - mainEmbed.SetFooter("Twitter", BaseIcon) mainEmbed.SetTimestamp(tweet.TimeParsed) + //mainEmbed.SetFooter("Twitter", BaseIcon) //mainEmbed.SetFooter("Twitter • <t:" + strconv.FormatInt((tweet.Timestamp), 10) + ":R>", BaseIcon) + tweetText := tweet.Text for i, photo := range tweet.Photos { embed := mainEmbed if i > 0 { embed = data.NewEmbedWithURL(tweet.PermanentURL) } embed.SetImage(photo.URL) + tweetText = strings.ReplaceAll(tweetText, photo.URL, "") } - for i, gif := range tweet.GIFs { - embed := mainEmbed - if i > 0 { - embed = data.NewEmbedWithURL(tweet.PermanentURL) - } - embed.SetImage(gif.Preview) + for _, gif := range tweet.GIFs { + //embed := mainEmbed + //if i > 0 { + // embed = data.NewEmbedWithURL(tweet.PermanentURL) + //} + //embed.SetImage(gif.Preview) + tweetText = strings.ReplaceAll(tweetText, gif.Preview, "") + tweetText = strings.ReplaceAll(tweetText, gif.URL, "") } - for i, video := range tweet.Videos { - embed := mainEmbed - if i > 0 { - embed = data.NewEmbedWithURL(tweet.PermanentURL) - } - // Video embeds are not supported right now - embed.SetImage(video.Preview) - embed.SetVideo(video.URL) // This has sadly no effect - urlsToAppend = append(urlsToAppend, strings.Replace(tweet.PermanentURL, "twitter", "fxtwitter", 1)) + for _, video := range tweet.Videos { + //embed := mainEmbed + //if i > 0 { + // embed = data.NewEmbedWithURL(tweet.PermanentURL) + //} + //embed.SetImage(video.Preview) + tweetText = strings.ReplaceAll(tweetText, video.Preview, "") + tweetText = strings.ReplaceAll(tweetText, video.URL, "") } - err := sendRequest(webhookURL, &data) - if err != nil { - log.Println("Error while sending webhook for tweet %s: %s", tweet.ID, err.Error()) - continue - } + mainEmbed.SetText(strings.TrimSpace(tweetText)) - for _, url := range urlsToAppend { - err := sendRequest(webhookURL, &Webhook{Content: url}) + for _, data := range webhooksToSend { + err := sendRequest(app.config.Webhook, data) if err != nil { - log.Println("Error while sending webhook for tweet %s: %s", tweet.ID, err.Error()) + log.Printf("Error while sending webhook for tweet %s: %s", tweet.ID, err.Error()) continue } } + } } diff --git a/cmd/tweeter_test.go b/cmd/tweeter_test.go deleted file mode 100644 index e8eacca..0000000 --- a/cmd/tweeter_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package cmd - -import ( - "testing" -) - -func TestFilter(t *testing.T) { - filter1 := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b10111, 0b10101, 0b01101, 0b10101} - filter2 := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b01000, 0b01010, 0b01000, 0b11011} - filterR := []uint8{0b10101, 0b11111, 0b00000, 0b00111, 0b00101, 0b00001, 0b00000, 0b00000, 0b01000, 0b10001} - - for i, filterResult := range filterR { - resultByte := compareBytes(filter1[i], filter2[i]) - if resultByte != filterResult { - t.Errorf("%b AND %b = %b; got %b\n", filter1[i], filter2[i], filterResult, resultByte) - } - } -} - -func compareBytes(filter uint8, filterCheck uint8) uint8 { - filterQuoted := (filterCheck & 1) != 0 // first bit - filterPin := (filterCheck & 2) != 0 // second bit - filterReply := (filterCheck & 4) != 0 // third bit - filterRetweet := (filterCheck & 8) != 0 // fourth bit - filterSelfThread := (filterCheck & 16) != 0 // fifth bit - - var tweetFilter uint8 = 0 - filterMap := []bool{filterSelfThread, filterRetweet, filterReply, filterPin, filterQuoted} - for _, f := range filterMap { - tweetFilter <<= 1 - if f { - tweetFilter |= 1 - } - } - - return filter & tweetFilter -} diff --git a/compose.example.yml b/compose.example.yml index d0af096..a2d1cff 100644 --- a/compose.example.yml +++ b/compose.example.yml @@ -6,7 +6,10 @@ services: image: git.snrd.eu/sunred/discord-tweeter:latest container_name: discord-tweeter network_mode: bridge + ports: + - "127.0.0.1:8080:8080" + - "[::1]:8080:8080" volumes: - ./config.toml:/app/config.toml:ro - - ./db:/app/db:rw + - ./data:/app/data:rw restart: unless-stopped diff --git a/config.example.toml b/config.example.toml index 8766ec4..b4da53d 100644 --- a/config.example.toml +++ b/config.example.toml @@ -1,15 +1,16 @@ username = "" password = "asd123" #proxyaddr = "socks5://localhost:5555" +hostURL = "https://my.domain.tld" channels = [ - "NinEverything", - "NintendoEurope", - "NintendoAmerica", + "NinEverything", + "NintendoEurope", + "NintendoAmerica", ] # Binary representation for efficient filtering # Bit from left to right (most to least significant bit): IsSelfThread, IsRetweet, IsReply, IsPin, IsQuoted filter = [ - 0b11111, - 0b11111, - 0b11111, + 0b11111, + 0b11111, + 0b11111, ] diff --git a/docker-bake.hcl b/docker-bake.hcl index 2c6c7cc..e18ac8c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,34 +1,44 @@ -variable "REG" { - default = "git.snrd.eu" -} -variable "REPO" { - default = "sunred/discord-tweeter" +variable "IMAGE" { + default = "git.snrd.eu/sunred/discord-tweeter" } variable "TAG" { default = "latest" } +function "generate_tags" { + params = [images, versions] + result = distinct(flatten( + [for i in split(",", images) : + [for v in split(",", versions) : + "${i}:${v}" + ] + ])) +} + group "default" { targets = ["prod"] } target "default" { - #platforms = [ "linux/amd64", "linux/arm64", "linux/arm/v7", "linux/riscv64" ] - platforms = [ "linux/amd64", "linux/arm64" ] - tags = ["${REG}/${REPO}:latest", "${REG}/${REPO}:${TAG}"] - attest = [ - "type=provenance,disabled=true", - "type=sbom,disabled=true" - ] + tags = generate_tags(IMAGE, TAG) } target "prod" { inherits = ["default"] - #dockerfile = "Dockerfile" + //platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/riscv64"] + platforms = ["linux/amd64", "linux/arm64"] dockerfile = "Dockerfile.multiarch" output = ["type=registry"] + attest = [ + { type = "provenance", mode = "max" }, + { type = "sbom" } + ] } target "dev" { inherits = ["default"] - dockerfile = "Dockerfile.multiarch" - output = ["type=image"] + dockerfile = "Dockerfile" + output = ["type=docker"] + attest = [ + { type = "provenance", disabled = "true" }, + { type = "sbom", disabled = "true" } + ] } diff --git a/go.mod b/go.mod index 1cf0d87..02cb105 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,17 @@ module git.snrd.eu/sunred/discord-tweeter -go 1.21 +go 1.23.0 + +toolchain go1.24.1 require ( - github.com/BurntSushi/toml v1.3.2 - github.com/imperatrona/twitter-scraper v0.0.0-20240425221339-715658ada4d9 + github.com/BurntSushi/toml v1.5.0 + github.com/imperatrona/twitter-scraper v0.0.16 github.com/jmoiron/sqlx v1.4.0 - github.com/mattn/go-sqlite3 v1.14.22 + github.com/mattn/go-sqlite3 v1.14.24 ) require ( github.com/AlexEidt/Vidio v1.5.1 // indirect - golang.org/x/net v0.24.0 // indirect + golang.org/x/net v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index 3823a76..a77bd27 100644 --- a/go.sum +++ b/go.sum @@ -1,48 +1,52 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/AlexEidt/Vidio v1.5.1 h1:tovwvtgQagUz1vifiL9OeWkg1fP/XUzFazFKh7tFtaE= github.com/AlexEidt/Vidio v1.5.1/go.mod h1:djhIMnWMqPrC3X6nB6ymGX6uWWlgw+VayYGKE1bNwmI= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/imperatrona/twitter-scraper v0.0.0-20240425221339-715658ada4d9 h1:X/TQaFNGVNQXn33JnpQLBwQ+2Nd2P6x8EUEqMUI+xU4= -github.com/imperatrona/twitter-scraper v0.0.0-20240425221339-715658ada4d9/go.mod h1:+Z1pca7Faf2tzpHVRnRcratFxy/PuDKj2iaygdgZRLM= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/imperatrona/twitter-scraper v0.0.16 h1:XrJZDGqr0/A7C3qRyoud9Ue9k6eUP10VkfRij3ewkSA= +github.com/imperatrona/twitter-scraper v0.0.16/go.mod h1:38MY3g/h4V7Xl4HbW9lnkL8S3YiFZenBFv86hN57RG8= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= -github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -50,22 +54,32 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index e3c8da7..1ff7ca0 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,9 @@ package main import ( - "git.snrd.eu/sunred/discord-tweeter/cmd" "log" + + "git.snrd.eu/sunred/discord-tweeter/cmd/tweeter" ) var ( @@ -11,5 +12,5 @@ var ( func main() { log.SetPrefix(prefix) - cmd.Run() + tweeter.Run() } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go new file mode 100644 index 0000000..bd4379f --- /dev/null +++ b/pkg/cache/cache.go @@ -0,0 +1,52 @@ +package cache + +import ( + "sync" + "time" +) + +type Cache[K comparable, V any] struct { + data map[K]entry[K, V] + lock sync.RWMutex +} + +type entry[K comparable, V any] struct { + value V + expiration time.Time +} + +func New[K comparable, V any]() *Cache[K, V] { + return &Cache[K, V]{ + data: make(map[K]entry[K, V]), + } +} + +func (c *Cache[K, V]) Set(key K, value V, ttl time.Duration) { + c.lock.Lock() + defer c.lock.Unlock() + + c.data[key] = entry[K, V]{ + value: value, + expiration: time.Now().Add(ttl), + } +} + +func (c *Cache[K, V]) Get(key K) (V, bool) { + c.lock.RLock() + defer c.lock.RUnlock() + + entry, ok := c.data[key] + if !ok || time.Now().After(entry.expiration) { + delete(c.data, key) + var zero V + return zero, false + } + + return entry.value, true +} + +func (c *Cache[K, V]) Delete(key K) { + c.lock.Lock() + defer c.lock.Unlock() + delete(c.data, key) +} diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go new file mode 100644 index 0000000..5076cdd --- /dev/null +++ b/pkg/cache/cache_test.go @@ -0,0 +1,145 @@ +package cache + +import ( + "fmt" + "reflect" + "sync" + "testing" + "time" +) + +func TestCacheBasicOperations(t *testing.T) { + cache := New[string, int]() + + // Test Set and Get + cache.Set("key1", 42, 10*time.Second) + value, exists := cache.Get("key1") + if !exists { + t.Errorf("expected key to exist") + } + if value != 42 { + t.Errorf("expected value 42, got %v", value) + } + + // Test non-existent key + value, exists = cache.Get("nonexistent") + if exists { + t.Errorf("expected key to not exist") + } + if value != 0 { + t.Errorf("expected zero value, got %v", value) + } +} + +func TestCacheExpiration(t *testing.T) { + cache := New[string, int]() + + // Set value with very short TTL + cache.Set("short-lived", 42, 100*time.Millisecond) + + // Verify initial existence + value, exists := cache.Get("short-lived") + if !exists { + t.Errorf("expected key to exist initially") + } + if value != 42 { + t.Errorf("expected value 42, got %v", value) + } + + // Wait for expiration + time.Sleep(150 * time.Millisecond) + + // Verify expiration + value, exists = cache.Get("short-lived") + if exists { + t.Errorf("expected key to have expired") + } + if value != 0 { + t.Errorf("expected zero value, got %v", value) + } +} + +func TestCacheConcurrentAccess(t *testing.T) { + cache := New[string, int]() + + // Start multiple writers + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + cache.Set(fmt.Sprintf("key%d", i), i*2, 10*time.Second) + }(i) + } + + // Start multiple readers + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + cache.Get(fmt.Sprintf("key%d", i)) + }(i) + } + + wg.Wait() + + // Verify all values were written correctly + for i := 0; i < 10; i++ { + value, exists := cache.Get(fmt.Sprintf("key%d", i)) + if !exists { + t.Errorf("expected key%d to exist", i) + } + if value != (i * 2) { + t.Errorf("expected value %d for key%d, got %d", i*2, i, value) + } + } +} + +func TestCacheGenericTypes(t *testing.T) { + // Test with string values + strCache := New[int, string]() + strCache.Set(1, "hello", 10*time.Second) + value, exists := strCache.Get(1) + if !exists { + t.Errorf("expected key to exist") + } + if value != "hello" { + t.Errorf("expected value hello, got %v", value) + } + + // Test with struct values + type Person struct { + Name string + Age int + } + personCache := New[string, Person]() + personCache.Set("john", Person{"John", 30}, 10*time.Second) + person, exists := personCache.Get("john") + if !exists { + t.Errorf("expected key to exist") + } + expected := Person{"John", 30} + if !reflect.DeepEqual(person, expected) { + t.Errorf("expected %+v, got %+v", expected, person) + } +} + +func TestCacheDelete(t *testing.T) { + cache := New[string, int]() + + // Set and verify + cache.Set("key1", 42, 10*time.Second) + _, exists := cache.Get("key1") + if !exists { + t.Errorf("expected key to exist") + } + + // Delete + cache.Delete("key1") + + // Verify deletion + _, exists = cache.Get("key1") + if exists { + t.Errorf("expected key to not exist after deletion") + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..dd540fb --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,44 @@ +package config + +import ( + "os" + + "github.com/BurntSushi/toml" +) + +type Config struct { + Username string + Password string + ProxyAddr string + Channels []string + Filter []uint8 + Webhook string + DbPath string + CookiePath string + UseWebServer bool + HostURL string + WebPort uint16 + UserAgents []string + NitterBase string +} + +func ConfigFromFile(filePath string) (*Config, error) { + conf := &Config{ + // Default values + DbPath: "./data/tweets.db", + CookiePath: "./data/cookies.json", + UseWebServer: true, + WebPort: 8080, + UserAgents: []string{"discordbot", "curl", "httpie", "lwp-request", "wget", "python-requests", "openbsd ftp", "powershell"}, + NitterBase: "https://xcancel.com", + } + + tomlData, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + + _, err = toml.Decode(string(tomlData), conf) + + return conf, err +} diff --git a/cmd/database.go b/pkg/db/database.go similarity index 81% rename from cmd/database.go rename to pkg/db/database.go index b9e1a6a..482a63f 100644 --- a/cmd/database.go +++ b/pkg/db/database.go @@ -1,12 +1,12 @@ -package cmd +package db import ( - "errors" "fmt" + "strconv" + + ts "github.com/imperatrona/twitter-scraper" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" - ts "github.com/imperatrona/twitter-scraper" - "strconv" ) const ( @@ -32,7 +32,7 @@ type Database struct { *sqlx.DB } -func NewDatabase(driver string, connectString string) (*Database, error) { +func New(driver string, connectString string) (Database, error) { var connection *sqlx.DB var err error @@ -40,22 +40,22 @@ func NewDatabase(driver string, connectString string) (*Database, error) { case "sqlite3": connection, err = sqlx.Connect(driver, "file:"+connectString+"?cache=shared") if err != nil { - return nil, err + return Database{}, err } connection.SetMaxOpenConns(1) if _, err = connection.Exec(SqliteSchema); err != nil { - return nil, err + return Database{}, err } default: - return nil, errors.New(fmt.Sprintf("Database driver %s not supported right now!", driver)) + return Database{}, fmt.Errorf("database driver %s not supported right now", driver) } - return &Database{connection}, err + return Database{connection}, err } -func (db *Database) GetNewestTweet(channel string) (*Tweet, error) { +func (db Database) GetNewestTweet(channel string) (*Tweet, error) { tweet := Tweet{} err := db.Get(&tweet, "SELECT * FROM tweet WHERE channel=$1 ORDER BY timestamp DESC, snowflake DESC LIMIT 1", channel) if err != nil { @@ -64,7 +64,7 @@ func (db *Database) GetNewestTweet(channel string) (*Tweet, error) { return &tweet, nil } -func (db *Database) GetTweets(channel string) ([]*Tweet, error) { +func (db Database) GetTweets(channel string) ([]*Tweet, error) { tweet := []*Tweet{} err := db.Select(&tweet, "SELECT * FROM tweet WHERE channel=$1 ORDER BY timestamp DESC, snowflake DESC", channel) if err != nil { @@ -73,7 +73,7 @@ func (db *Database) GetTweets(channel string) ([]*Tweet, error) { return tweet, nil } -func (db *Database) ContainsTweet(channel string, tweet *ts.Tweet) (bool, error) { +func (db Database) ContainsTweet(channel string, tweet *ts.Tweet) (bool, error) { snowflake, err := strconv.ParseUint(tweet.ID, 10, 64) if err != nil { return false, err @@ -97,7 +97,7 @@ func (db *Database) ContainsTweet(channel string, tweet *ts.Tweet) (bool, error) return false, nil } -func (db *Database) InsertTweet(channel string, tweet *ts.Tweet) error { +func (db Database) InsertTweet(channel string, tweet *ts.Tweet) error { snowflake, err := strconv.ParseUint(tweet.ID, 10, 64) if err != nil { return err @@ -111,7 +111,7 @@ func (db *Database) InsertTweet(channel string, tweet *ts.Tweet) error { return nil } -func (db *Database) PruneOldestTweets(channel string) error { +func (db Database) PruneOldestTweets(channel string) error { var count int err := db.Get(&count, "SELECT COUNT(*) FROM tweet WHERE channel=$1", channel) if err != nil { @@ -157,7 +157,7 @@ func FromTweet(channel string, tweet *ts.Tweet) (*Tweet, error) { return &Tweet{0, snowflake, channel, tweet.Timestamp}, nil } -func (t *Tweet) EqualsTweet(tweet *ts.Tweet) bool { +func (t Tweet) EqualsTweet(tweet *ts.Tweet) bool { snowflake, err := strconv.ParseUint(tweet.ID, 10, 64) if err != nil { return false @@ -166,6 +166,6 @@ func (t *Tweet) EqualsTweet(tweet *ts.Tweet) bool { return t.Snowflake == snowflake } -func (t *Tweet) Equals(tweet *Tweet) bool { +func (t Tweet) Equals(tweet *Tweet) bool { return t.Snowflake == tweet.Snowflake } diff --git a/cmd/database_test.go b/pkg/db/database_test.go similarity index 93% rename from cmd/database_test.go rename to pkg/db/database_test.go index a4b762f..b9607e2 100644 --- a/cmd/database_test.go +++ b/pkg/db/database_test.go @@ -1,13 +1,14 @@ -package cmd +package db import ( + "testing" + "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" - "testing" ) const ( - testDbPath = "../db/testdb.db" + testDbPath = "../data/testdb.db" ) var ( diff --git a/pkg/web/templates/tweet.html b/pkg/web/templates/tweet.html new file mode 100644 index 0000000..d019a1b --- /dev/null +++ b/pkg/web/templates/tweet.html @@ -0,0 +1,35 @@ +<!DOCTYPE html> +<html> +<head> + <meta charset="UTF-8"> + <meta name="theme-color" content="#26a7de"> + <link rel="canonical" href="{{ .URL }}"> + + <meta property="twitter:site" content="{{ .Username }}"> + <meta property="twitter:creator" content="{{ .Username }}"> + <meta property="twitter:title" content="{{ .Title }}"> +{{- range $idx, $e := .Images }} + <meta property="twitter:image" content="{{ $e }}"> + <meta property="og:image" content="{{ $e }}"> +{{- end }} +{{- range $idx, $e := .Videos }} + <meta property="twitter:player:stream" content="{{ $e }}"> + <meta property="og:video" content="{{ $e }}"> +{{- end }} +{{- range $idx, $e := .Previews }} + <meta property="twitter:image" content="0"> + <meta property="og:image" content="{{ $e }}"> +{{- end }} + <meta property="twitter:card" content="{{ .Format }}"> + <meta property="og:type" content="article"> + <meta property="og:article:published_time" content="{{ .Timestamp }}"> + <meta property="og:url" content="{{ .Link }}"> + <meta property="og:title" content="{{ .Title }}"> + <meta property="og:description" content="{{ .Text }}"> + + <style> + body, html { margin: 0; padding: 0; height: 100%; width: 100%; } + body { display: flex; } + </style> + +<body> diff --git a/pkg/web/templates/video.html b/pkg/web/templates/video.html new file mode 100644 index 0000000..c4dc13b --- /dev/null +++ b/pkg/web/templates/video.html @@ -0,0 +1,32 @@ +<!DOCTYPE html> +<html> +<head> + <meta charset="UTF-8"> + <meta name="theme-color" content="#26a7de"> + <link rel="canonical" href="{{ .URL }}"> + +{{- if .Videos }} + <meta property="twitter:site" content="{{ .Username }}"> + <meta property="twitter:creator" content="{{ .Username }}"> +{{- range $idx, $e := .Videos }} + <meta property="twitter:player:stream" content="{{ $e }}"> + <meta property="og:video" content="{{ $e }}"> +{{- end }} +{{- range $idx, $e := .Previews }} + <meta property="twitter:image" content="0"> + <meta property="og:image" content="{{ $e }}"> +{{- end }} + <meta property="twitter:card" content="{{ .Format }}"> + <meta property="og:site_name" content="Video attached due to embed limitations"> + <meta property="og:type" content="article"> + <meta property="og:article:published_time" content="{{ .Timestamp }}"> + <meta property="og:url" content="{{ .Link }}"> + <meta property="og:title" content="​"> +{{- end }} + + <style> + body, html { margin: 0; padding: 0; height: 100%; width: 100%; } + body { display: flex; } + </style> + +<body> diff --git a/pkg/web/web.go b/pkg/web/web.go new file mode 100644 index 0000000..161bcc0 --- /dev/null +++ b/pkg/web/web.go @@ -0,0 +1,211 @@ +package web + +import ( + "bytes" + "embed" + "fmt" + "html/template" + "net/http" + "slices" + "strings" + "time" + + "git.snrd.eu/sunred/discord-tweeter/pkg/cache" + "git.snrd.eu/sunred/discord-tweeter/pkg/config" + ts "github.com/imperatrona/twitter-scraper" +) + +const ( + AvatarCacheTime = 60 * time.Minute // How long to serve same avatar from cache before doing a fresh scrape + TweetCacheTime = 10 * time.Minute // How long to serve same tweet from cache before doing a fresh scrape +) + +//go:embed templates/*.html +var templateFiles embed.FS + +type Reponse struct { + StatusCode int + ContentType string + Content string +} + +type WebServer struct { + config *config.Config + scraper *ts.Scraper + templates *template.Template + avatarCache *cache.Cache[string, string] + responseCache *cache.Cache[string, Reponse] + + Server *http.Server +} + +type Tweet struct { + URL string + Link string + Title string + Text string + Username string + Timestamp string + Format string + Images []string + Videos []string + Previews []string +} + +func New(config *config.Config, scraper *ts.Scraper) (*WebServer, error) { + sm := http.NewServeMux() + tmpl, err := template.ParseFS(templateFiles, "templates/*.html") + if err != nil { + return nil, err + } + + ws := &WebServer{ + config, + scraper, + tmpl, + cache.New[string, string](), + cache.New[string, Reponse](), + &http.Server{ + Handler: sm, + Addr: fmt.Sprintf(":%d", config.WebPort), + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + }, + } + sm.HandleFunc("GET /avatar/{username}", ws.handleAvatar) + sm.HandleFunc("GET /tweet/{id}", ws.handleTweet) + sm.HandleFunc("GET /video/{id}", ws.handleVideo) + return ws, nil +} + +func (ws WebServer) handleAvatar(w http.ResponseWriter, r *http.Request) { + username := r.PathValue("username") + + if !slices.Contains(ws.config.Channels, username) { + badRequest(w) + return + } + + url, cached := ws.avatarCache.Get("avatar-" + username) + if cached { + http.Redirect(w, r, url, http.StatusPermanentRedirect) + return + } + + profile, err := ws.scraper.GetProfile(username) + if err != nil { + serverError(w) + return + } + + ws.avatarCache.Set("avatar-"+username, profile.Avatar, AvatarCacheTime) + + http.Redirect(w, r, profile.Avatar, http.StatusPermanentRedirect) +} + +func (ws WebServer) handleTweet(w http.ResponseWriter, r *http.Request) { + ws.handleTemplate(w, r, r.PathValue("id"), "tweet") +} + +func (ws WebServer) handleVideo(w http.ResponseWriter, r *http.Request) { + ws.handleTemplate(w, r, r.PathValue("id"), "video") +} + +func (ws WebServer) handleTemplate(w http.ResponseWriter, r *http.Request, id string, template string) { + if !validUserAgent(r.UserAgent(), ws.config.UserAgents) { + http.Redirect(w, r, ws.config.NitterBase+"/i/status/"+id, http.StatusPermanentRedirect) + return + } + + entry, cached := ws.responseCache.Get(template + "-" + id) + if cached { + response(w, entry) + return + } + + tweet, err := ws.scraper.GetTweet(id) + if err != nil { + serverError(w) + return + } + + if !slices.Contains(ws.config.Channels, tweet.Username) { + res := Reponse{http.StatusBadRequest, "text/plain", "Bad Request"} + ws.responseCache.Set(template+"-"+id, res, TweetCacheTime) + response(w, res) + return + } + + tweetText := tweet.Text + var images []string + var videos []string + var previews []string + + for _, photo := range tweet.Photos { + images = append(images, photo.URL) + tweetText = strings.ReplaceAll(tweetText, photo.URL, "") + } + for _, gif := range tweet.GIFs { + videos = append(images, gif.URL) + previews = append(images, gif.Preview) + tweetText = strings.ReplaceAll(tweetText, gif.URL, "") + tweetText = strings.ReplaceAll(tweetText, gif.Preview, "") + } + for _, video := range tweet.Videos { + videos = append(videos, video.URL) + previews = append(previews, video.Preview) + tweetText = strings.ReplaceAll(tweetText, video.URL, "") + tweetText = strings.ReplaceAll(tweetText, video.Preview, "") + } + + format := "summary_large_image" + if len(videos) > 0 { + format = "player" + } + + data := Tweet{ + URL: tweet.PermanentURL, + Link: ws.config.NitterBase + "/" + tweet.Username + "/status/" + id, + Title: tweet.Name + " (@" + tweet.Username + ")", + Text: strings.TrimSpace(tweetText), + Username: tweet.Username, + Timestamp: tweet.TimeParsed.Format(time.RFC3339), + Format: format, + Images: images, + Videos: videos, + Previews: previews, + } + + var tpl bytes.Buffer + if err := ws.templates.ExecuteTemplate(&tpl, template+".html", data); err != nil { + serverError(w) + return + } + + res := Reponse{http.StatusOK, "text/html", tpl.String()} + ws.responseCache.Set(template+"-"+id, res, TweetCacheTime) + response(w, res) +} + +func validUserAgent(ua string, uas []string) bool { + for _, s := range uas { + if s == "*" || strings.Contains(strings.ToLower(ua), s) { + return true + } + } + return false +} + +func badRequest(w http.ResponseWriter) { + response(w, Reponse{http.StatusBadRequest, "text/plain", "Bad Request"}) +} + +func serverError(w http.ResponseWriter) { + response(w, Reponse{http.StatusInternalServerError, "text/plain", "Internal Server Error"}) +} + +func response(w http.ResponseWriter, res Reponse) { + w.WriteHeader(res.StatusCode) + w.Header().Set("Content-Type", res.ContentType) + fmt.Fprint(w, res.Content) +}