feat: add support for robots.txt disabling search indexing (#2929)

* feat: add support for robots.txt

Can toggle disabling search engine indexing. Closes #2684

* fix: unexport ts const

controllers/admin/config.go

@@ -751,6 +751,26 @@ func SetHideViewerCount(w http.ResponseWriter, r *http.Request) {
 	controllers.WriteSimpleResponse(w, true, "hide viewer count setting updated")
 }
 
+// SetDisableSearchIndexing will set search indexing support.
+func SetDisableSearchIndexing(w http.ResponseWriter, r *http.Request) {
+	if !requirePOST(w, r) {
+		return
+	}
+
+	configValue, success := getValueFromRequest(w, r)
+	if !success {
+		controllers.WriteSimpleResponse(w, false, "unable to update search indexing")
+		return
+	}
+
+	if err := data.SetDisableSearchIndexing(configValue.Value.(bool)); err != nil {
+		controllers.WriteSimpleResponse(w, false, err.Error())
+		return
+	}
+
+	controllers.WriteSimpleResponse(w, true, "search indexing support updated")
+}
+
 func requirePOST(w http.ResponseWriter, r *http.Request) bool {
 	if r.Method != controllers.POST {
 		controllers.WriteSimpleResponse(w, false, r.Method+" not supported")

controllers/admin/serverConfig.go

@@ -61,6 +61,7 @@ func GetServerConfig(w http.ResponseWriter, r *http.Request) {
 		SocketHostOverride:      data.GetWebsocketOverrideHost(),
 		ChatEstablishedUserMode: data.GetChatEstbalishedUsersOnlyMode(),
 		HideViewerCount:         data.GetHideViewerCount(),
+		DisableSearchIndexing:   data.GetDisableSearchIndexing(),
 		VideoSettings: videoSettings{
 			VideoQualityVariants: videoQualityVariants,
 			LatencyLevel:         data.GetStreamLatencyLevel().Level,
@@ -121,6 +122,7 @@ type serverConfigAdminResponse struct {
 	ChatEstablishedUserMode bool                        `json:"chatEstablishedUserMode"`
 	StreamKeyOverridden     bool                        `json:"streamKeyOverridden"`
 	HideViewerCount         bool                        `json:"hideViewerCount"`
+	DisableSearchIndexing   bool                        `json:"disableSearchIndexing"`
 }
 
 type videoSettings struct {

controllers/robots.go

@@ -0,0 +1,28 @@
+package controllers
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/owncast/owncast/core/data"
+)
+
+// GetRobotsDotTxt returns the contents of our robots.txt.
+func GetRobotsDotTxt(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	contents := []string{
+		"User-agent: *",
+		"Disallow: /admin",
+		"Disallow: /api",
+	}
+
+	if data.GetDisableSearchIndexing() {
+		contents = append(contents, "Disallow: /")
+	}
+
+	txt := []byte(strings.Join(contents, "\n"))
+
+	if _, err := w.Write(txt); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}