From 56225d85e8c004f23e503d59e1863d6b42ced653 Mon Sep 17 00:00:00 2001 From: Philipp Date: Wed, 19 Mar 2025 18:26:33 +0100 Subject: [PATCH 1/9] add missing webhook entry to config.example.toml --- config.example.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/config.example.toml b/config.example.toml index b4da53d..36f09a9 100644 --- a/config.example.toml +++ b/config.example.toml @@ -2,6 +2,7 @@ username = "" password = "asd123" #proxyaddr = "socks5://localhost:5555" hostURL = "https://my.domain.tld" +webhook = "https://domain.tld/api/webhooks/" channels = [ "NinEverything", "NintendoEurope", From 8ff9e561826a94649c46d3d90ff1296ff779cbc8 Mon Sep 17 00:00:00 2001 From: Manuel Date: Wed, 19 Mar 2025 20:36:24 +0100 Subject: [PATCH 2/9] fix(web): Minor typo of struct and add @ in front of username --- pkg/web/templates/tweet.html | 4 ++-- pkg/web/templates/video.html | 4 ++-- pkg/web/web.go | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/web/templates/tweet.html b/pkg/web/templates/tweet.html index d019a1b..96165d5 100644 --- a/pkg/web/templates/tweet.html +++ b/pkg/web/templates/tweet.html @@ -5,8 +5,8 @@ - - + + {{- range $idx, $e := .Images }} diff --git a/pkg/web/templates/video.html b/pkg/web/templates/video.html index c4dc13b..2c02e15 100644 --- a/pkg/web/templates/video.html +++ b/pkg/web/templates/video.html @@ -6,8 +6,8 @@ {{- if .Videos }} - - + + {{- range $idx, $e := .Videos }} diff --git a/pkg/web/web.go b/pkg/web/web.go index 161bcc0..11b99f1 100644 --- a/pkg/web/web.go +++ b/pkg/web/web.go @@ -23,7 +23,7 @@ const ( //go:embed templates/*.html var templateFiles embed.FS -type Reponse struct { +type Response struct { StatusCode int ContentType string Content string @@ -34,7 +34,7 @@ type WebServer struct { scraper *ts.Scraper templates *template.Template avatarCache *cache.Cache[string, string] - responseCache *cache.Cache[string, Reponse] + responseCache *cache.Cache[string, Response] Server *http.Server } @@ -64,7 +64,7 @@ func New(config *config.Config, scraper *ts.Scraper) (*WebServer, error) { scraper, tmpl, cache.New[string, string](), - cache.New[string, Reponse](), + cache.New[string, Response](), &http.Server{ Handler: sm, Addr: fmt.Sprintf(":%d", config.WebPort), @@ -130,7 +130,7 @@ func (ws WebServer) handleTemplate(w http.ResponseWriter, r *http.Request, id st } if !slices.Contains(ws.config.Channels, tweet.Username) { - res := Reponse{http.StatusBadRequest, "text/plain", "Bad Request"} + res := Response{http.StatusBadRequest, "text/plain", "Bad Request"} ws.responseCache.Set(template+"-"+id, res, TweetCacheTime) response(w, res) return @@ -182,7 +182,7 @@ func (ws WebServer) handleTemplate(w http.ResponseWriter, r *http.Request, id st return } - res := Reponse{http.StatusOK, "text/html", tpl.String()} + res := Response{http.StatusOK, "text/html", tpl.String()} ws.responseCache.Set(template+"-"+id, res, TweetCacheTime) response(w, res) } @@ -197,14 +197,14 @@ func validUserAgent(ua string, uas []string) bool { } func badRequest(w http.ResponseWriter) { - response(w, Reponse{http.StatusBadRequest, "text/plain", "Bad Request"}) + response(w, Response{http.StatusBadRequest, "text/plain", "Bad Request"}) } func serverError(w http.ResponseWriter) { - response(w, Reponse{http.StatusInternalServerError, "text/plain", "Internal Server Error"}) + response(w, Response{http.StatusInternalServerError, "text/plain", "Internal Server Error"}) } -func response(w http.ResponseWriter, res Reponse) { +func response(w http.ResponseWriter, res Response) { w.WriteHeader(res.StatusCode) w.Header().Set("Content-Type", res.ContentType) fmt.Fprint(w, res.Content) From ddaf33b4f8a96a23bbf4d78d3b07e860e75c8a49 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 20 Mar 2025 00:44:55 +0100 Subject: [PATCH 3/9] feat(nix-flake): add nix flake and nix module --- nix/default.nix | 21 ++++++++++++ nix/flake.lock | 85 ++++++++++++++++++++++++++++++++++++++++++++++ nix/flake.nix | 24 +++++++++++++ nix/gomod2nix.toml | 21 ++++++++++++ nix/services.nix | 51 ++++++++++++++++++++++++++++ nix/shell.nix | 24 +++++++++++++ 6 files changed, 226 insertions(+) create mode 100644 nix/default.nix create mode 100644 nix/flake.lock create mode 100644 nix/flake.nix create mode 100644 nix/gomod2nix.toml create mode 100644 nix/services.nix create mode 100644 nix/shell.nix diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..b07d863 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,21 @@ +{ pkgs ? ( + let + inherit (builtins) fetchTree fromJSON readFile; + inherit ((fromJSON (readFile ./flake.lock)).nodes) nixpkgs gomod2nix; + in + import (fetchTree nixpkgs.locked) { + overlays = [ + (import "${fetchTree gomod2nix.locked}/overlay.nix") + ]; + } +) +, buildGoApplication ? pkgs.buildGoApplication +}: + +buildGoApplication { + pname = "discord-tweeter"; + version = "0.1"; + pwd = ./.; + src = ../.; + modules = ./gomod2nix.toml; +} diff --git a/nix/flake.lock b/nix/flake.lock new file mode 100644 index 0000000..a0e83ed --- /dev/null +++ b/nix/flake.lock @@ -0,0 +1,85 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gomod2nix": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1742209644, + "narHash": "sha256-jMy1XqXqD0/tJprEbUmKilTkvbDY/C0ZGSsJJH4TNCE=", + "owner": "nix-community", + "repo": "gomod2nix", + "rev": "8f3534eb8f6c5c3fce799376dc3b91bae6b11884", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "gomod2nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1742288794, + "narHash": "sha256-Txwa5uO+qpQXrNG4eumPSD+hHzzYi/CdaM80M9XRLCo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b6eaf97c6960d97350c584de1b6dcff03c9daf42", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "gomod2nix": "gomod2nix", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/flake.nix b/nix/flake.nix new file mode 100644 index 0000000..80ad039 --- /dev/null +++ b/nix/flake.nix @@ -0,0 +1,24 @@ +{ + description = "NixOS flake for discord-tweeter"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + inputs.gomod2nix.url = "github:nix-community/gomod2nix"; + inputs.gomod2nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.gomod2nix.inputs.flake-utils.follows = "flake-utils"; + + outputs = { self, nixpkgs, flake-utils, gomod2nix }: + (flake-utils.lib.eachDefaultSystem + (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + + callPackage = pkgs.darwin.apple_sdk_11_0.callPackage or pkgs.callPackage; + in + { + nixosModules.default = callPackage ./services.nix { inherit self; }; + packages.default = callPackage ./. { inherit (gomod2nix.legacyPackages.${system}) buildGoApplication; }; + devShells.default = callPackage ./shell.nix { inherit (gomod2nix.legacyPackages.${system}) mkGoEnv gomod2nix; }; + }) + ); +} diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml new file mode 100644 index 0000000..5edd8c3 --- /dev/null +++ b/nix/gomod2nix.toml @@ -0,0 +1,21 @@ +schema = 3 + +[mod] + [mod."github.com/AlexEidt/Vidio"] + version = "v1.5.1" + hash = "sha256-WdnCYjxbFqDmh/IVaVCCc1TzvklKdJtDMdIgn6m6NTM=" + [mod."github.com/BurntSushi/toml"] + version = "v1.5.0" + hash = "sha256-wX8bEVo7swuuAlm0awTIiV1KNCAXnm7Epzwl+wzyqhw=" + [mod."github.com/imperatrona/twitter-scraper"] + version = "v0.0.16" + hash = "sha256-nbv9fI6/3OxnC8NaLO82UfcwbJMz1jxvfNjZPOrILzM=" + [mod."github.com/jmoiron/sqlx"] + version = "v1.4.0" + hash = "sha256-0H132+A983nBr2zEyCKsJoBCZlC9pG+ylEcGysxKL4M=" + [mod."github.com/mattn/go-sqlite3"] + version = "v1.14.24" + hash = "sha256-taGKFZFQlR5++5b2oZ1dYS3RERKv6yh1gniNWhb4egg=" + [mod."golang.org/x/net"] + version = "v0.37.0" + hash = "sha256-sZKbJISVdBwyuYRQgrraTKxeIORWlzK5hScceQ2dE58=" diff --git a/nix/services.nix b/nix/services.nix new file mode 100644 index 0000000..6658611 --- /dev/null +++ b/nix/services.nix @@ -0,0 +1,51 @@ +{config, lib, pkgs, ...}: + +with lib; + +let + cfg = config.services.discord-tweeter; + + format = pkgs.formats.toml { }; + configFile = format.generate "config.toml" cfg.settings; +in +{ + options.services.discord-tweeter = { + enable = mkEnableOption (lib.mdDoc "discord-tweeter"); + + settings = mkOption { + type = format.type; + default = { + username = ""; + password = ""; + proxyaddr = ""; + webhook = ""; + DbPath = "/var/lib/discord-tweeter/"; + CookiePath = "/var/lib/discord-tweeter/"; + channels = [ ]; + filter = [ 0b11111 ]; + UseWebServer = true; + HostURL = ""; + WebPort = 8080; + UserAgents = [ "discordbot" "curl" "httpie" "lwp-request" "wget" "python-requests" "openbsd ftp" "powershell" ]; + NitterBase = "https://xcancel.com"; + }; + description = lib.mdDoc '' + ''; + }; + }; + + config = mkIf cfg.enable { + systemd.services.discord-tweeter = { + description = "Send tweets to Discord by scraping Twitter (now X)"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + DynamicUser = true; + ExecStart = "${self.packages.${pkgs.system}.default}/bin/discord-tweeter ${configFile}"; + Restart = "on-failure"; + }; + }; + }; +} + diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 0000000..8d806a2 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1,24 @@ +{ pkgs ? ( + let + inherit (builtins) fetchTree fromJSON readFile; + inherit ((fromJSON (readFile ./flake.lock)).nodes) nixpkgs gomod2nix; + in + import (fetchTree nixpkgs.locked) { + overlays = [ + (import "${fetchTree gomod2nix.locked}/overlay.nix") + ]; + } + ) +, mkGoEnv ? pkgs.mkGoEnv +, gomod2nix ? pkgs.gomod2nix +}: + +let + goEnv = mkGoEnv { pwd = ../.; }; +in +pkgs.mkShell { + packages = [ + goEnv + gomod2nix + ]; +} From d27348ca0b008206641aece8cb902b5a4d4d00d6 Mon Sep 17 00:00:00 2001 From: Manuel Date: Thu, 20 Mar 2025 07:30:00 +0100 Subject: [PATCH 4/9] fix: Unescape tweet text for webhook --- cmd/tweeter/webhook.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/tweeter/webhook.go b/cmd/tweeter/webhook.go index 0c69601..7995371 100644 --- a/cmd/tweeter/webhook.go +++ b/cmd/tweeter/webhook.go @@ -3,6 +3,7 @@ package tweeter import ( "bytes" "encoding/json" + "html" "log" "net/http" "strings" @@ -80,7 +81,7 @@ func (app App) SendToWebhook(tweets []*ts.Tweet) { tweetText = strings.ReplaceAll(tweetText, video.URL, "") } - mainEmbed.SetText(strings.TrimSpace(tweetText)) + mainEmbed.SetText(html.UnescapeString(strings.TrimSpace(tweetText))) for _, data := range webhooksToSend { err := sendRequest(app.config.Webhook, data) From 15bf56a1e742790d77f8b13bbf9771187d2b80dc Mon Sep 17 00:00:00 2001 From: Manuel Date: Thu, 20 Mar 2025 07:40:00 +0100 Subject: [PATCH 5/9] feat(web): Support redirecting to custom url for index page --- config.example.toml | 8 +++++++- pkg/config/config.go | 1 + pkg/web/web.go | 7 +++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/config.example.toml b/config.example.toml index 36f09a9..fdcd679 100644 --- a/config.example.toml +++ b/config.example.toml @@ -1,13 +1,19 @@ username = "" password = "asd123" #proxyaddr = "socks5://localhost:5555" -hostURL = "https://my.domain.tld" + +#usewebserver = false +indextarget = "https://discord.gg/Kw4MFGxYEj" # Optional +hostURL = "https://my.domain.tld" # Can be omitted if not using webserver + webhook = "https://domain.tld/api/webhooks/" + channels = [ "NinEverything", "NintendoEurope", "NintendoAmerica", ] + # Binary representation for efficient filtering # Bit from left to right (most to least significant bit): IsSelfThread, IsRetweet, IsReply, IsPin, IsQuoted filter = [ diff --git a/pkg/config/config.go b/pkg/config/config.go index dd540fb..499badb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -20,6 +20,7 @@ type Config struct { WebPort uint16 UserAgents []string NitterBase string + IndexTarget string } func ConfigFromFile(filePath string) (*Config, error) { diff --git a/pkg/web/web.go b/pkg/web/web.go index 11b99f1..a8b5a5a 100644 --- a/pkg/web/web.go +++ b/pkg/web/web.go @@ -72,12 +72,19 @@ func New(config *config.Config, scraper *ts.Scraper) (*WebServer, error) { WriteTimeout: 30 * time.Second, }, } + sm.HandleFunc("GET /", ws.handleIndex) sm.HandleFunc("GET /avatar/{username}", ws.handleAvatar) sm.HandleFunc("GET /tweet/{id}", ws.handleTweet) sm.HandleFunc("GET /video/{id}", ws.handleVideo) return ws, nil } +func (ws WebServer) handleIndex(w http.ResponseWriter, r *http.Request) { + if ws.config.IndexTarget != "" { + http.Redirect(w, r, ws.config.IndexTarget, http.StatusPermanentRedirect) + } +} + func (ws WebServer) handleAvatar(w http.ResponseWriter, r *http.Request) { username := r.PathValue("username") From da9a0bca3df6ef4383e2e8686e550526094e12ff Mon Sep 17 00:00:00 2001 From: Manuel Date: Thu, 20 Mar 2025 08:35:18 +0100 Subject: [PATCH 6/9] feat: Add actions workflow using Docker bake file --- .forgejo/workflows/bake.yml | 55 +++++++++++++++++++++++++++++++++++++ docker-bake.hcl | 4 +-- 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 .forgejo/workflows/bake.yml diff --git a/.forgejo/workflows/bake.yml b/.forgejo/workflows/bake.yml new file mode 100644 index 0000000..60611ba --- /dev/null +++ b/.forgejo/workflows/bake.yml @@ -0,0 +1,55 @@ +on: + push: + branches: + - 'ci-test' + tags: + - 'v*' + paths: + - '**.go' + - '**.html' + - 'Dockerfile' + - 'Dockerfile.*' + - 'docker-bake.hcl' + - '.forgejo/workflows/*.yml' + workflow_dispatch: + +jobs: + bake: + runs-on: docker + steps: + - name: Prepare Registry FQDN + id: registry + run: | + registry=${{ github.server_url }} + registry=${registry##http*://} + echo "registry=${registry}" >> "$GITHUB_OUTPUT" + + - name: Login to Container Registry + uses: https://code.forgejo.org/docker/login-action@v3 + with: + registry: ${{ steps.registry.outputs.registry }} + username: ${{ github.repository_owner }} + password: ${{ secrets.TOKEN }} + + - name: Checkout + uses: https://code.forgejo.org/actions/checkout@v4 + + - name: Set up Docker Buildx + uses: https://code.forgejo.org/docker/setup-buildx-action@v3 + + - name: Extract metadata + id: meta + uses: https://code.forgejo.org/docker/metadata-action@v5 + with: + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Build and push + uses: https://code.forgejo.org/docker/bake-action@v6 + with: + source: . + env: + TAG: ${{ steps.meta.outputs.tags }} diff --git a/docker-bake.hcl b/docker-bake.hcl index e18ac8c..0af347b 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -8,8 +8,8 @@ variable "TAG" { function "generate_tags" { params = [images, versions] result = distinct(flatten( - [for i in split(",", images) : - [for v in split(",", versions) : + [for i in split(",", replace(images, "\n", ",")) : + [for v in split(",", replace(versions, "\n", ",")) : "${i}:${v}" ] ])) From 327af5ae2e12b32e54dfc4236a9c7b5cade4930a Mon Sep 17 00:00:00 2001 From: Manuel Date: Thu, 20 Mar 2025 08:41:43 +0100 Subject: [PATCH 7/9] ci: Remove url prefix as code.forgejo.org is default anyway --- .forgejo/workflows/bake.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/bake.yml b/.forgejo/workflows/bake.yml index 60611ba..3f2cd63 100644 --- a/.forgejo/workflows/bake.yml +++ b/.forgejo/workflows/bake.yml @@ -10,7 +10,6 @@ on: - 'Dockerfile' - 'Dockerfile.*' - 'docker-bake.hcl' - - '.forgejo/workflows/*.yml' workflow_dispatch: jobs: @@ -25,21 +24,21 @@ jobs: echo "registry=${registry}" >> "$GITHUB_OUTPUT" - name: Login to Container Registry - uses: https://code.forgejo.org/docker/login-action@v3 + uses: docker/login-action@v3 with: registry: ${{ steps.registry.outputs.registry }} username: ${{ github.repository_owner }} password: ${{ secrets.TOKEN }} - name: Checkout - uses: https://code.forgejo.org/actions/checkout@v4 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: https://code.forgejo.org/docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3 - name: Extract metadata id: meta - uses: https://code.forgejo.org/docker/metadata-action@v5 + uses: docker/metadata-action@v5 with: tags: | type=ref,event=branch @@ -48,7 +47,7 @@ jobs: type=semver,pattern={{major}} - name: Build and push - uses: https://code.forgejo.org/docker/bake-action@v6 + uses: docker/bake-action@v6 with: source: . env: From ea66bb4285dd79197cc0cf32501641e84a44d220 Mon Sep 17 00:00:00 2001 From: Manuel Date: Thu, 20 Mar 2025 08:57:59 +0100 Subject: [PATCH 8/9] ci: Change branch to main --- .forgejo/workflows/bake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/bake.yml b/.forgejo/workflows/bake.yml index 3f2cd63..a28de2b 100644 --- a/.forgejo/workflows/bake.yml +++ b/.forgejo/workflows/bake.yml @@ -1,7 +1,7 @@ on: push: branches: - - 'ci-test' + - 'main' tags: - 'v*' paths: From f8116c5777fed0bddb4c91177ee24bfeb00438ec Mon Sep 17 00:00:00 2001 From: Manuel Date: Thu, 20 Mar 2025 09:03:06 +0100 Subject: [PATCH 9/9] ci: Build for more platforms --- docker-bake.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 0af347b..7cb61be 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -24,8 +24,8 @@ target "default" { } target "prod" { inherits = ["default"] - //platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/riscv64"] - platforms = ["linux/amd64", "linux/arm64"] + platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/riscv64"] + #platforms = ["linux/amd64", "linux/arm64"] dockerfile = "Dockerfile.multiarch" output = ["type=registry"] attest = [