feat: add support for robots.txt disabling search indexing (#2929)

* feat: add support for robots.txt

Can toggle disabling search engine indexing. Closes #2684

* fix: unexport ts const

web/components/admin/config/general/EditInstanceDetails.tsx

@@ -21,6 +21,7 @@ import {
   FIELD_PROPS_NSFW,
   FIELD_PROPS_HIDE_VIEWER_COUNT,
   API_SERVER_OFFLINE_MESSAGE,
+  FIELD_PROPS_DISABLE_SEARCH_INDEXING,
 } from '../../../../utils/config-constants';
 import { UpdateArgs } from '../../../../types/config-section';
 import { ToggleSwitch } from '../../ToggleSwitch';
@@ -36,7 +37,7 @@ export default function EditInstanceDetails() {
   const serverStatusData = useContext(ServerStatusContext);
   const { serverConfig } = serverStatusData || {};
 
-  const { instanceDetails, yp, hideViewerCount } = serverConfig;
+  const { instanceDetails, yp, hideViewerCount, disableSearchIndexing } = serverConfig;
   const { instanceUrl } = yp;
 
   const [offlineMessageSaveStatus, setOfflineMessageSaveStatus] = useState(null);
@@ -46,6 +47,7 @@ export default function EditInstanceDetails() {
       ...instanceDetails,
       ...yp,
       hideViewerCount,
+      disableSearchIndexing,
     });
   }, [instanceDetails, yp]);
 
@@ -87,6 +89,10 @@ export default function EditInstanceDetails() {
     handleFieldChange({ fieldName: 'hideViewerCount', value: enabled });
   }
 
+  function handleDisableSearchEngineIndexingChange(enabled: boolean) {
+    handleFieldChange({ fieldName: 'disableSearchIndexing', value: enabled });
+  }
+
   const hasInstanceUrl = instanceUrl !== '';
 
   return (
@@ -171,6 +177,14 @@ export default function EditInstanceDetails() {
         onChange={handleHideViewerCountChange}
       />
 
+      <ToggleSwitch
+        fieldName="disableSearchIndexing"
+        useSubmit
+        {...FIELD_PROPS_DISABLE_SEARCH_INDEXING}
+        checked={formDataValues.disableSearchIndexing}
+        onChange={handleDisableSearchEngineIndexingChange}
+      />
+
       <br />
       <p className="description">
         Increase your audience by appearing in the{' '}