fix: use lightweight bot/scraper html responses. Fixes #3253

utils/utils.go

@@ -16,6 +16,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mssola/user_agent"
 	log "github.com/sirupsen/logrus"
 	"github.com/yuin/goldmark"
 	"github.com/yuin/goldmark/extension"
@@ -120,6 +121,34 @@ func IsUserAgentAPlayer(userAgent string) bool {
 	return false
 }
 
+// IsUserAgentABot returns if a web client user-agent is seen as a bot.
+func IsUserAgentABot(userAgent string) bool {
+	if userAgent == "" {
+		return false
+	}
+
+	botStrings := []string{
+		"mastodon",
+		"pleroma",
+		"applebot",
+		"whatsapp",
+		"matrix",
+		"synapse",
+		"element",
+		"rocket.chat",
+		"duckduckbot",
+	}
+
+	for _, botString := range botStrings {
+		if strings.Contains(strings.ToLower(userAgent), botString) {
+			return true
+		}
+	}
+
+	ua := user_agent.New(userAgent)
+	return ua.Bot()
+}
+
 // RenderSimpleMarkdown will return HTML without sanitization or specific formatting rules.
 func RenderSimpleMarkdown(raw string) string {
 	markdown := goldmark.New(