forked from SunRed/discord-tweeter
feat: Major refactor, implement web, caching, better tests and build files
* Update golang version to 1.24 * Update multiarch Dockerfile to be more ISA agnostic * Refactor existing code and properly structure project into modules * Get rid of global variables except where necessary (go:embed) * Add default values to Config * Add webserver with templates to finally correctly serve videos and gifs * Add tiny caching library to decrease api load and improve latency * Improve Webhook data preparation by filtering out redundant links from the tweet text and properly attaching videos and gifs in separate webhook request by utilising new webserver * Improve tests for filter function * Improve bake definition for easier CI integration
This commit is contained in:
parent
7562b86894
commit
21d580d1a6
GPG key ID:
4085037435E1F07A
24 changed files with 752 additions and 209 deletions
35
pkg/web/templates/tweet.html
Normal file
35
pkg/web/templates/tweet.html
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="theme-color" content="#26a7de">
|
||||
<link rel="canonical" href="{{ .URL }}">
|
||||
|
||||
<meta property="twitter:site" content="{{ .Username }}">
|
||||
<meta property="twitter:creator" content="{{ .Username }}">
|
||||
<meta property="twitter:title" content="{{ .Title }}">
|
||||
{{- range $idx, $e := .Images }}
|
||||
<meta property="twitter:image" content="{{ $e }}">
|
||||
<meta property="og:image" content="{{ $e }}">
|
||||
{{- end }}
|
||||
{{- range $idx, $e := .Videos }}
|
||||
<meta property="twitter:player:stream" content="{{ $e }}">
|
||||
<meta property="og:video" content="{{ $e }}">
|
||||
{{- end }}
|
||||
{{- range $idx, $e := .Previews }}
|
||||
<meta property="twitter:image" content="0">
|
||||
<meta property="og:image" content="{{ $e }}">
|
||||
{{- end }}
|
||||
<meta property="twitter:card" content="{{ .Format }}">
|
||||
<meta property="og:type" content="article">
|
||||
<meta property="og:article:published_time" content="{{ .Timestamp }}">
|
||||
<meta property="og:url" content="{{ .Link }}">
|
||||
<meta property="og:title" content="{{ .Title }}">
|
||||
<meta property="og:description" content="{{ .Text }}">
|
||||
|
||||
<style>
|
||||
body, html { margin: 0; padding: 0; height: 100%; width: 100%; }
|
||||
body { display: flex; }
|
||||
</style>
|
||||
|
||||
<body>
|
||||
32
pkg/web/templates/video.html
Normal file
32
pkg/web/templates/video.html
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="theme-color" content="#26a7de">
|
||||
<link rel="canonical" href="{{ .URL }}">
|
||||
|
||||
{{- if .Videos }}
|
||||
<meta property="twitter:site" content="{{ .Username }}">
|
||||
<meta property="twitter:creator" content="{{ .Username }}">
|
||||
{{- range $idx, $e := .Videos }}
|
||||
<meta property="twitter:player:stream" content="{{ $e }}">
|
||||
<meta property="og:video" content="{{ $e }}">
|
||||
{{- end }}
|
||||
{{- range $idx, $e := .Previews }}
|
||||
<meta property="twitter:image" content="0">
|
||||
<meta property="og:image" content="{{ $e }}">
|
||||
{{- end }}
|
||||
<meta property="twitter:card" content="{{ .Format }}">
|
||||
<meta property="og:site_name" content="Video attached due to embed limitations">
|
||||
<meta property="og:type" content="article">
|
||||
<meta property="og:article:published_time" content="{{ .Timestamp }}">
|
||||
<meta property="og:url" content="{{ .Link }}">
|
||||
<meta property="og:title" content="​">
|
||||
{{- end }}
|
||||
|
||||
<style>
|
||||
body, html { margin: 0; padding: 0; height: 100%; width: 100%; }
|
||||
body { display: flex; }
|
||||
</style>
|
||||
|
||||
<body>
|
||||
211
pkg/web/web.go
Normal file
211
pkg/web/web.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"embed"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.snrd.eu/sunred/discord-tweeter/pkg/cache"
|
||||
"git.snrd.eu/sunred/discord-tweeter/pkg/config"
|
||||
ts "github.com/imperatrona/twitter-scraper"
|
||||
)
|
||||
|
||||
const (
|
||||
AvatarCacheTime = 60 * time.Minute // How long to serve same avatar from cache before doing a fresh scrape
|
||||
TweetCacheTime = 10 * time.Minute // How long to serve same tweet from cache before doing a fresh scrape
|
||||
)
|
||||
|
||||
//go:embed templates/*.html
|
||||
var templateFiles embed.FS
|
||||
|
||||
type Reponse struct {
|
||||
StatusCode int
|
||||
ContentType string
|
||||
Content string
|
||||
}
|
||||
|
||||
type WebServer struct {
|
||||
config *config.Config
|
||||
scraper *ts.Scraper
|
||||
templates *template.Template
|
||||
avatarCache *cache.Cache[string, string]
|
||||
responseCache *cache.Cache[string, Reponse]
|
||||
|
||||
Server *http.Server
|
||||
}
|
||||
|
||||
type Tweet struct {
|
||||
URL string
|
||||
Link string
|
||||
Title string
|
||||
Text string
|
||||
Username string
|
||||
Timestamp string
|
||||
Format string
|
||||
Images []string
|
||||
Videos []string
|
||||
Previews []string
|
||||
}
|
||||
|
||||
func New(config *config.Config, scraper *ts.Scraper) (*WebServer, error) {
|
||||
sm := http.NewServeMux()
|
||||
tmpl, err := template.ParseFS(templateFiles, "templates/*.html")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ws := &WebServer{
|
||||
config,
|
||||
scraper,
|
||||
tmpl,
|
||||
cache.New[string, string](),
|
||||
cache.New[string, Reponse](),
|
||||
&http.Server{
|
||||
Handler: sm,
|
||||
Addr: fmt.Sprintf(":%d", config.WebPort),
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
sm.HandleFunc("GET /avatar/{username}", ws.handleAvatar)
|
||||
sm.HandleFunc("GET /tweet/{id}", ws.handleTweet)
|
||||
sm.HandleFunc("GET /video/{id}", ws.handleVideo)
|
||||
return ws, nil
|
||||
}
|
||||
|
||||
func (ws WebServer) handleAvatar(w http.ResponseWriter, r *http.Request) {
|
||||
username := r.PathValue("username")
|
||||
|
||||
if !slices.Contains(ws.config.Channels, username) {
|
||||
badRequest(w)
|
||||
return
|
||||
}
|
||||
|
||||
url, cached := ws.avatarCache.Get("avatar-" + username)
|
||||
if cached {
|
||||
http.Redirect(w, r, url, http.StatusPermanentRedirect)
|
||||
return
|
||||
}
|
||||
|
||||
profile, err := ws.scraper.GetProfile(username)
|
||||
if err != nil {
|
||||
serverError(w)
|
||||
return
|
||||
}
|
||||
|
||||
ws.avatarCache.Set("avatar-"+username, profile.Avatar, AvatarCacheTime)
|
||||
|
||||
http.Redirect(w, r, profile.Avatar, http.StatusPermanentRedirect)
|
||||
}
|
||||
|
||||
func (ws WebServer) handleTweet(w http.ResponseWriter, r *http.Request) {
|
||||
ws.handleTemplate(w, r, r.PathValue("id"), "tweet")
|
||||
}
|
||||
|
||||
func (ws WebServer) handleVideo(w http.ResponseWriter, r *http.Request) {
|
||||
ws.handleTemplate(w, r, r.PathValue("id"), "video")
|
||||
}
|
||||
|
||||
func (ws WebServer) handleTemplate(w http.ResponseWriter, r *http.Request, id string, template string) {
|
||||
if !validUserAgent(r.UserAgent(), ws.config.UserAgents) {
|
||||
http.Redirect(w, r, ws.config.NitterBase+"/i/status/"+id, http.StatusPermanentRedirect)
|
||||
return
|
||||
}
|
||||
|
||||
entry, cached := ws.responseCache.Get(template + "-" + id)
|
||||
if cached {
|
||||
response(w, entry)
|
||||
return
|
||||
}
|
||||
|
||||
tweet, err := ws.scraper.GetTweet(id)
|
||||
if err != nil {
|
||||
serverError(w)
|
||||
return
|
||||
}
|
||||
|
||||
if !slices.Contains(ws.config.Channels, tweet.Username) {
|
||||
res := Reponse{http.StatusBadRequest, "text/plain", "Bad Request"}
|
||||
ws.responseCache.Set(template+"-"+id, res, TweetCacheTime)
|
||||
response(w, res)
|
||||
return
|
||||
}
|
||||
|
||||
tweetText := tweet.Text
|
||||
var images []string
|
||||
var videos []string
|
||||
var previews []string
|
||||
|
||||
for _, photo := range tweet.Photos {
|
||||
images = append(images, photo.URL)
|
||||
tweetText = strings.ReplaceAll(tweetText, photo.URL, "")
|
||||
}
|
||||
for _, gif := range tweet.GIFs {
|
||||
videos = append(images, gif.URL)
|
||||
previews = append(images, gif.Preview)
|
||||
tweetText = strings.ReplaceAll(tweetText, gif.URL, "")
|
||||
tweetText = strings.ReplaceAll(tweetText, gif.Preview, "")
|
||||
}
|
||||
for _, video := range tweet.Videos {
|
||||
videos = append(videos, video.URL)
|
||||
previews = append(previews, video.Preview)
|
||||
tweetText = strings.ReplaceAll(tweetText, video.URL, "")
|
||||
tweetText = strings.ReplaceAll(tweetText, video.Preview, "")
|
||||
}
|
||||
|
||||
format := "summary_large_image"
|
||||
if len(videos) > 0 {
|
||||
format = "player"
|
||||
}
|
||||
|
||||
data := Tweet{
|
||||
URL: tweet.PermanentURL,
|
||||
Link: ws.config.NitterBase + "/" + tweet.Username + "/status/" + id,
|
||||
Title: tweet.Name + " (@" + tweet.Username + ")",
|
||||
Text: strings.TrimSpace(tweetText),
|
||||
Username: tweet.Username,
|
||||
Timestamp: tweet.TimeParsed.Format(time.RFC3339),
|
||||
Format: format,
|
||||
Images: images,
|
||||
Videos: videos,
|
||||
Previews: previews,
|
||||
}
|
||||
|
||||
var tpl bytes.Buffer
|
||||
if err := ws.templates.ExecuteTemplate(&tpl, template+".html", data); err != nil {
|
||||
serverError(w)
|
||||
return
|
||||
}
|
||||
|
||||
res := Reponse{http.StatusOK, "text/html", tpl.String()}
|
||||
ws.responseCache.Set(template+"-"+id, res, TweetCacheTime)
|
||||
response(w, res)
|
||||
}
|
||||
|
||||
func validUserAgent(ua string, uas []string) bool {
|
||||
for _, s := range uas {
|
||||
if s == "*" || strings.Contains(strings.ToLower(ua), s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func badRequest(w http.ResponseWriter) {
|
||||
response(w, Reponse{http.StatusBadRequest, "text/plain", "Bad Request"})
|
||||
}
|
||||
|
||||
func serverError(w http.ResponseWriter) {
|
||||
response(w, Reponse{http.StatusInternalServerError, "text/plain", "Internal Server Error"})
|
||||
}
|
||||
|
||||
func response(w http.ResponseWriter, res Reponse) {
|
||||
w.WriteHeader(res.StatusCode)
|
||||
w.Header().Set("Content-Type", res.ContentType)
|
||||
fmt.Fprint(w, res.Content)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue