feat: allow force enable search index even if noIndex: true is set

Closes #385

Author: weareoutman

README.md

@@ -97,6 +97,7 @@ module.exports = {
 | searchContextByPaths              | `(string \| { label: string \| Record<string, string>; path: string; } )[]` | `[]`      | Provide an list of sub-paths as separate search context, E.g.: `["docs", "community", "legacy/resources"]`. It will create multiple search indexes by these paths.                                                                                                                                                                                          |
 | hideSearchBarWithNoSearchContext  | boolean                                                                     | `false`   | Whether to hide the search bar when no search context was matched. By default, if `searchContextByPaths` is set, pages which are not matched with it will be considered as with a search context of ROOT. By setting `hideSearchBarWithNoSearchContext: true`, these pages will be considered as with NO search context, and the search bar will be hidden. |
 | useAllContextsWithNoSearchContext | boolean                                                                     | `false`   | Whether to show results from all the contexts if no context is provided. This option should not be used with `hideSearchBarWithNoSearchContext: true` as this would show results when there is no search context. This will duplicate indexes and might have a performance cost depending on the index sizes.                                               |
+| `forceIgnoreNoIndex`              | boolean                                                                     | `false`   | Force enable search index even if `noIndex: true` is set, this also affects unlisted articles.                                                                                                                                                                                                                                                              |
 
 ### I18N
 

docusaurus-search-local/src/declarations.ts

@@ -26,6 +26,7 @@ declare module "*/generated.js" {
   )[];
   export const hideSearchBarWithNoSearchContext: boolean;
   export const useAllContextsWithNoSearchContext: boolean;
+  export const forceIgnoreNoIndex: boolean;
   // These below are for mocking only.
   export const __setLanguage: (value: string[]) => void;
   export const __setRemoveDefaultStopWordFilter: (value: boolean) => void;

docusaurus-search-local/src/index.ts

@@ -192,4 +192,11 @@ export interface PluginOptions {
    * @default false
    */
   useAllContextsWithNoSearchContext?: boolean;
+
+  /**
+   * Force enable search index even if noIndex: true is set, this also affects unlisted articles.
+   *
+   * @default false
+   */
+  forceIgnoreNoIndex?: boolean;
 }

docusaurus-search-local/src/server/utils/parse.ts

@@ -10,12 +10,12 @@ export function parse(
   html: string,
   type: "docs" | "blog" | "page",
   url: string,
-  { ignoreCssSelectors }: ProcessedPluginOptions
+  { ignoreCssSelectors, forceIgnoreNoIndex }: ProcessedPluginOptions
 ): ParsedDocument | null {
   const $ = cheerio.load(html);
 
   const robotsMeta = $('meta[name="robots"]');
-  if (robotsMeta.attr("content")?.includes("noindex")) {
+  if (!forceIgnoreNoIndex && robotsMeta.attr("content")?.includes("noindex")) {
     // Unlisted content
     return null;
   }

docusaurus-search-local/src/server/utils/validateOptions.spec.ts

@@ -56,6 +56,7 @@ describe("validateOptions", () => {
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
+        forceIgnoreNoIndex: false,
       },
     ],
     [
@@ -83,6 +84,7 @@ describe("validateOptions", () => {
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
+        forceIgnoreNoIndex: false,
       },
     ],
     [
@@ -110,6 +112,7 @@ describe("validateOptions", () => {
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
+        forceIgnoreNoIndex: false,
       },
     ],
     [
@@ -137,6 +140,7 @@ describe("validateOptions", () => {
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
+        forceIgnoreNoIndex: false,
       },
     ],
     [
@@ -151,6 +155,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 30,
         searchBarShortcut: false,
+        forceIgnoreNoIndex: true,
       },
       {
         blogRouteBasePath: ["blog"],
@@ -175,6 +180,7 @@ describe("validateOptions", () => {
         searchBarShortcut: false,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
+        forceIgnoreNoIndex: true,
       },
     ],
     [
@@ -207,6 +213,7 @@ describe("validateOptions", () => {
         searchBarShortcut: true,
         searchBarShortcutHint: false,
         searchBarPosition: "auto",
+        forceIgnoreNoIndex: false,
       },
     ],
     [
@@ -245,6 +252,7 @@ describe("validateOptions", () => {
         searchBarPosition: "left",
         docsPluginIdForPreferredVersion: "product",
         searchContextByPaths: ["docs", "community"],
+        forceIgnoreNoIndex: false,
       },
     ],
     [
@@ -282,6 +290,7 @@ describe("validateOptions", () => {
         searchBarPosition: "left",
         docsPluginIdForPreferredVersion: "product",
         searchContextByPaths: ["docs", "community"],
+        forceIgnoreNoIndex: false,
       },
     ],
   ])("validateOptions(...) should work", (options, config) => {

docusaurus-search-local/src/server/utils/validateOptions.ts

@@ -60,6 +60,7 @@ const schema = Joi.object<PluginOptions>({
   ),
   hideSearchBarWithNoSearchContext: Joi.boolean().default(false),
   useAllContextsWithNoSearchContext: Joi.boolean().default(false),
+  forceIgnoreNoIndex: Joi.boolean().default(false),
 });
 
 export function validateOptions({