feat: add support for robots.txt disabling search indexing (#2929)

* feat: add support for robots.txt

Can toggle disabling search engine indexing. Closes #2684

* fix: unexport ts const

web/components/admin/config/general/EditInstanceDetails.tsx

@@ -21,6 +21,7 @@ import {
   FIELD_PROPS_NSFW,
   FIELD_PROPS_HIDE_VIEWER_COUNT,
   API_SERVER_OFFLINE_MESSAGE,
+  FIELD_PROPS_DISABLE_SEARCH_INDEXING,
 } from '../../../../utils/config-constants';
 import { UpdateArgs } from '../../../../types/config-section';
 import { ToggleSwitch } from '../../ToggleSwitch';
@@ -36,7 +37,7 @@ export default function EditInstanceDetails() {
   const serverStatusData = useContext(ServerStatusContext);
   const { serverConfig } = serverStatusData || {};
 
-  const { instanceDetails, yp, hideViewerCount } = serverConfig;
+  const { instanceDetails, yp, hideViewerCount, disableSearchIndexing } = serverConfig;
   const { instanceUrl } = yp;
 
   const [offlineMessageSaveStatus, setOfflineMessageSaveStatus] = useState(null);
@@ -46,6 +47,7 @@ export default function EditInstanceDetails() {
       ...instanceDetails,
       ...yp,
       hideViewerCount,
+      disableSearchIndexing,
     });
   }, [instanceDetails, yp]);
 
@@ -87,6 +89,10 @@ export default function EditInstanceDetails() {
     handleFieldChange({ fieldName: 'hideViewerCount', value: enabled });
   }
 
+  function handleDisableSearchEngineIndexingChange(enabled: boolean) {
+    handleFieldChange({ fieldName: 'disableSearchIndexing', value: enabled });
+  }
+
   const hasInstanceUrl = instanceUrl !== '';
 
   return (
@@ -171,6 +177,14 @@ export default function EditInstanceDetails() {
         onChange={handleHideViewerCountChange}
       />
 
+      <ToggleSwitch
+        fieldName="disableSearchIndexing"
+        useSubmit
+        {...FIELD_PROPS_DISABLE_SEARCH_INDEXING}
+        checked={formDataValues.disableSearchIndexing}
+        onChange={handleDisableSearchEngineIndexingChange}
+      />
+
       <br />
       <p className="description">
         Increase your audience by appearing in the{' '}

web/types/config-section.ts

@@ -156,4 +156,5 @@ export interface ConfigDetails {
   chatJoinMessagesEnabled: boolean;
   chatEstablishedUserMode: boolean;
   hideViewerCount: boolean;
+  disableSearchIndexing: boolean;
 }

web/utils/config-constants.tsx

@@ -38,7 +38,7 @@ const API_HIDE_VIEWER_COUNT = '/hideviewercount';
 const API_CHAT_DISABLE = '/chat/disable';
 const API_CHAT_JOIN_MESSAGES_ENABLED = '/chat/joinmessagesenabled';
 const API_CHAT_ESTABLISHED_MODE = '/chat/establishedusermode';
-
+const API_DISABLE_SEARCH_INDEXING = '/disablesearchindexing';
 const API_SOCKET_HOST_OVERRIDE = '/sockethostoverride';
 
 // Federation
@@ -212,6 +212,13 @@ export const FIELD_PROPS_HIDE_VIEWER_COUNT = {
   tip: 'Turn this ON to hide the viewer count on the web page.',
 };
 
+export const FIELD_PROPS_DISABLE_SEARCH_INDEXING = {
+  apiPath: API_DISABLE_SEARCH_INDEXING,
+  configPath: '',
+  label: 'Disable search engine indexing',
+  tip: 'Turn this ON to to tell search engines not to index this site.',
+};
+
 export const DEFAULT_VARIANT_STATE: VideoVariant = {
   framerate: 24,
   videoPassthrough: false,

web/utils/server-status-context.tsx

@@ -71,6 +71,7 @@ const initialServerConfigState: ConfigDetails = {
   chatJoinMessagesEnabled: true,
   chatEstablishedUserMode: false,
   hideViewerCount: false,
+  disableSearchIndexing: false,
 };
 
 const initialServerStatusState = {