diff --git a/controllers/admin/config.go b/controllers/admin/config.go
index f7c5af058..be0192d20 100644
--- a/controllers/admin/config.go
+++ b/controllers/admin/config.go
@@ -751,6 +751,26 @@ func SetHideViewerCount(w http.ResponseWriter, r *http.Request) {
controllers.WriteSimpleResponse(w, true, "hide viewer count setting updated")
}
+// SetDisableSearchIndexing will set search indexing support.
+func SetDisableSearchIndexing(w http.ResponseWriter, r *http.Request) {
+ if !requirePOST(w, r) {
+ return
+ }
+
+ configValue, success := getValueFromRequest(w, r)
+ if !success {
+ controllers.WriteSimpleResponse(w, false, "unable to update search indexing")
+ return
+ }
+
+ if err := data.SetDisableSearchIndexing(configValue.Value.(bool)); err != nil {
+ controllers.WriteSimpleResponse(w, false, err.Error())
+ return
+ }
+
+ controllers.WriteSimpleResponse(w, true, "search indexing support updated")
+}
+
func requirePOST(w http.ResponseWriter, r *http.Request) bool {
if r.Method != controllers.POST {
controllers.WriteSimpleResponse(w, false, r.Method+" not supported")
diff --git a/controllers/admin/serverConfig.go b/controllers/admin/serverConfig.go
index f301f9fbc..d069a5479 100644
--- a/controllers/admin/serverConfig.go
+++ b/controllers/admin/serverConfig.go
@@ -61,6 +61,7 @@ func GetServerConfig(w http.ResponseWriter, r *http.Request) {
SocketHostOverride: data.GetWebsocketOverrideHost(),
ChatEstablishedUserMode: data.GetChatEstbalishedUsersOnlyMode(),
HideViewerCount: data.GetHideViewerCount(),
+ DisableSearchIndexing: data.GetDisableSearchIndexing(),
VideoSettings: videoSettings{
VideoQualityVariants: videoQualityVariants,
LatencyLevel: data.GetStreamLatencyLevel().Level,
@@ -121,6 +122,7 @@ type serverConfigAdminResponse struct {
ChatEstablishedUserMode bool `json:"chatEstablishedUserMode"`
StreamKeyOverridden bool `json:"streamKeyOverridden"`
HideViewerCount bool `json:"hideViewerCount"`
+ DisableSearchIndexing bool `json:"disableSearchIndexing"`
}
type videoSettings struct {
diff --git a/controllers/robots.go b/controllers/robots.go
new file mode 100644
index 000000000..4605d3da4
--- /dev/null
+++ b/controllers/robots.go
@@ -0,0 +1,28 @@
+package controllers
+
+import (
+ "net/http"
+ "strings"
+
+ "github.com/owncast/owncast/core/data"
+)
+
+// GetRobotsDotTxt returns the contents of our robots.txt.
+func GetRobotsDotTxt(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/plain")
+ contents := []string{
+ "User-agent: *",
+ "Disallow: /admin",
+ "Disallow: /api",
+ }
+
+ if data.GetDisableSearchIndexing() {
+ contents = append(contents, "Disallow: /")
+ }
+
+ txt := []byte(strings.Join(contents, "\n"))
+
+ if _, err := w.Write(txt); err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+}
diff --git a/core/data/config.go b/core/data/config.go
index 48e8feb5b..c7dd3c53b 100644
--- a/core/data/config.go
+++ b/core/data/config.go
@@ -69,6 +69,7 @@ const (
customOfflineMessageKey = "custom_offline_message"
customColorVariableValuesKey = "custom_color_variable_values"
streamKeysKey = "stream_keys"
+ disableSearchIndexingKey = "disable_search_indexing"
)
// GetExtraPageBodyContent will return the user-supplied body content.
@@ -959,3 +960,17 @@ func SetStreamKeys(actions []models.StreamKey) error {
configEntry := ConfigEntry{Key: streamKeysKey, Value: actions}
return _datastore.Save(configEntry)
}
+
+// SetDisableSearchIndexing will set if the web server should be indexable.
+func SetDisableSearchIndexing(disableSearchIndexing bool) error {
+ return _datastore.SetBool(disableSearchIndexingKey, disableSearchIndexing)
+}
+
+// GetDisableSearchIndexing will return if the web server should be indexable.
+func GetDisableSearchIndexing() bool {
+ disableSearchIndexing, err := _datastore.GetBool(disableSearchIndexingKey)
+ if err != nil {
+ return false
+ }
+ return disableSearchIndexing
+}
diff --git a/router/router.go b/router/router.go
index 3001cf43e..fd9583172 100644
--- a/router/router.go
+++ b/router/router.go
@@ -50,6 +50,9 @@ func Start() error {
// return a logo that's compatible with external social networks
http.HandleFunc("/logo/external", controllers.GetCompatibleLogo)
+ // robots.txt
+ http.HandleFunc("/robots.txt", controllers.GetRobotsDotTxt)
+
// status of the system
http.HandleFunc("/api/status", controllers.GetStatus)
@@ -327,6 +330,9 @@ func Start() error {
// Is the viewer count hidden from viewers
http.HandleFunc("/api/admin/config/hideviewercount", middleware.RequireAdminAuth(admin.SetHideViewerCount))
+ // set disabling of search indexing
+ http.HandleFunc("/api/admin/config/disablesearchindexing", middleware.RequireAdminAuth(admin.SetDisableSearchIndexing))
+
// Inline chat moderation actions
// Update chat message visibility
diff --git a/test/automated/api/configmanagement.test.js b/test/automated/api/configmanagement.test.js
index af41c1fb5..5bab5a318 100644
--- a/test/automated/api/configmanagement.test.js
+++ b/test/automated/api/configmanagement.test.js
@@ -37,6 +37,8 @@ const defaultFederationConfig = {
blockedDomains: [],
};
const defaultHideViewerCount = false;
+const defaultDisableSearchIndexing = false;
+
const defaultSocialHandles = [
{
icon: '/img/platformlogos/github.svg',
@@ -130,6 +132,7 @@ const newFederationConfig = {
};
const newHideViewerCount = !defaultHideViewerCount;
+const newDisableSearchIndexing = !defaultDisableSearchIndexing;
const overriddenWebsocketHost = 'ws://lolcalhost.biz';
const customCSS = randomString();
@@ -340,6 +343,14 @@ test('enable federation', async (done) => {
done();
});
+test('disable search indexing', async (done) => {
+ await sendAdminRequest(
+ 'config/disablesearchindexing',
+ newDisableSearchIndexing
+ );
+ done();
+});
+
test('change admin password', async (done) => {
const res = await sendAdminRequest('config/adminpass', newAdminPassword);
done();
@@ -472,3 +483,18 @@ test('verify frontend status', (done) => {
done();
});
});
+
+test('verify robots.txt is correct after disabling search indexing', (done) => {
+ const expected = `User-agent: *
+Disallow: /admin
+Disallow: /api
+Disallow: /`;
+
+ request
+ .get('/robots.txt')
+ .expect(200)
+ .then((res) => {
+ expect(res.text).toBe(expected);
+ done();
+ });
+});
diff --git a/web/components/admin/config/general/EditInstanceDetails.tsx b/web/components/admin/config/general/EditInstanceDetails.tsx
index 9c9ec7535..243a2497b 100644
--- a/web/components/admin/config/general/EditInstanceDetails.tsx
+++ b/web/components/admin/config/general/EditInstanceDetails.tsx
@@ -21,6 +21,7 @@ import {
FIELD_PROPS_NSFW,
FIELD_PROPS_HIDE_VIEWER_COUNT,
API_SERVER_OFFLINE_MESSAGE,
+ FIELD_PROPS_DISABLE_SEARCH_INDEXING,
} from '../../../../utils/config-constants';
import { UpdateArgs } from '../../../../types/config-section';
import { ToggleSwitch } from '../../ToggleSwitch';
@@ -36,7 +37,7 @@ export default function EditInstanceDetails() {
const serverStatusData = useContext(ServerStatusContext);
const { serverConfig } = serverStatusData || {};
- const { instanceDetails, yp, hideViewerCount } = serverConfig;
+ const { instanceDetails, yp, hideViewerCount, disableSearchIndexing } = serverConfig;
const { instanceUrl } = yp;
const [offlineMessageSaveStatus, setOfflineMessageSaveStatus] = useState(null);
@@ -46,6 +47,7 @@ export default function EditInstanceDetails() {
...instanceDetails,
...yp,
hideViewerCount,
+ disableSearchIndexing,
});
}, [instanceDetails, yp]);
@@ -87,6 +89,10 @@ export default function EditInstanceDetails() {
handleFieldChange({ fieldName: 'hideViewerCount', value: enabled });
}
+ function handleDisableSearchEngineIndexingChange(enabled: boolean) {
+ handleFieldChange({ fieldName: 'disableSearchIndexing', value: enabled });
+ }
+
const hasInstanceUrl = instanceUrl !== '';
return (
@@ -171,6 +177,14 @@ export default function EditInstanceDetails() {
onChange={handleHideViewerCountChange}
/>
+
+
Increase your audience by appearing in the{' '}
diff --git a/web/types/config-section.ts b/web/types/config-section.ts
index 0f0fece19..b70ee8bed 100644
--- a/web/types/config-section.ts
+++ b/web/types/config-section.ts
@@ -156,4 +156,5 @@ export interface ConfigDetails {
chatJoinMessagesEnabled: boolean;
chatEstablishedUserMode: boolean;
hideViewerCount: boolean;
+ disableSearchIndexing: boolean;
}
diff --git a/web/utils/config-constants.tsx b/web/utils/config-constants.tsx
index e0ef3dc23..0ec5e56cb 100644
--- a/web/utils/config-constants.tsx
+++ b/web/utils/config-constants.tsx
@@ -38,7 +38,7 @@ const API_HIDE_VIEWER_COUNT = '/hideviewercount';
const API_CHAT_DISABLE = '/chat/disable';
const API_CHAT_JOIN_MESSAGES_ENABLED = '/chat/joinmessagesenabled';
const API_CHAT_ESTABLISHED_MODE = '/chat/establishedusermode';
-
+const API_DISABLE_SEARCH_INDEXING = '/disablesearchindexing';
const API_SOCKET_HOST_OVERRIDE = '/sockethostoverride';
// Federation
@@ -212,6 +212,13 @@ export const FIELD_PROPS_HIDE_VIEWER_COUNT = {
tip: 'Turn this ON to hide the viewer count on the web page.',
};
+export const FIELD_PROPS_DISABLE_SEARCH_INDEXING = {
+ apiPath: API_DISABLE_SEARCH_INDEXING,
+ configPath: '',
+ label: 'Disable search engine indexing',
+ tip: 'Turn this ON to to tell search engines not to index this site.',
+};
+
export const DEFAULT_VARIANT_STATE: VideoVariant = {
framerate: 24,
videoPassthrough: false,
diff --git a/web/utils/server-status-context.tsx b/web/utils/server-status-context.tsx
index 70328f919..de3304bc9 100644
--- a/web/utils/server-status-context.tsx
+++ b/web/utils/server-status-context.tsx
@@ -71,6 +71,7 @@ const initialServerConfigState: ConfigDetails = {
chatJoinMessagesEnabled: true,
chatEstablishedUserMode: false,
hideViewerCount: false,
+ disableSearchIndexing: false,
};
const initialServerStatusState = {