From 589f1cf312bfe66badd75d2d28c0d28e01de56e2 Mon Sep 17 00:00:00 2001 From: Jonathan Poltak Samosir Date: Sun, 9 Jun 2024 11:19:40 +0700 Subject: [PATCH 1/3] Consolidate annotations typedef used for search - It was still flip-flopping between a really old src/annotations/type definition and the newer core-data-types def. Removed the old one --- src/dashboard-refactor/logic.test.util.ts | 4 ++-- src/dashboard-refactor/logic.ts | 4 ++-- src/dashboard-refactor/search-results/types.ts | 2 +- src/dashboard-refactor/search-results/util.ts | 12 ++++++------ src/search/background/types.ts | 5 ++--- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/dashboard-refactor/logic.test.util.ts b/src/dashboard-refactor/logic.test.util.ts index 10f9f0de4a..3fbf189b89 100644 --- a/src/dashboard-refactor/logic.test.util.ts +++ b/src/dashboard-refactor/logic.test.util.ts @@ -54,7 +54,7 @@ export const setPageSearchResult: DataSeederCreator = ( await storageManager.collection('annotations').createObject({ ...annot, }) - if (annot.isShared) { + if (annot['isShared']) { await storageManager .collection('sharedAnnotationMetadata') .createObject({ @@ -68,7 +68,7 @@ export const setPageSearchResult: DataSeederCreator = ( id: idCounter++, annotation: annot.url, createdWhen: new Date(), - privacyLevel: annot.isBulkShareProtected + privacyLevel: annot['isBulkShareProtected'] ? AnnotationPrivacyLevels.SHARED_PROTECTED : AnnotationPrivacyLevels.SHARED, }) diff --git a/src/dashboard-refactor/logic.ts b/src/dashboard-refactor/logic.ts index b60bce670d..5b01fe6f10 100644 --- a/src/dashboard-refactor/logic.ts +++ b/src/dashboard-refactor/logic.ts @@ -1562,7 +1562,7 @@ export class DashboardLogic extends UILogic { byId: { [event.noteId]: { comment: { $set: existing.comment }, - color: { $set: event.color as RGBAColor }, + color: { $set: event.color as string }, }, }, }, @@ -3586,7 +3586,7 @@ export class DashboardLogic extends UILogic { lists: { $set: nextLists }, color: { $set: (event.color ?? - existing.color) as RGBAColor, + existing.color) as string, }, }, }, diff --git a/src/dashboard-refactor/search-results/types.ts b/src/dashboard-refactor/search-results/types.ts index 0181f8a90c..f1baa88aaa 100644 --- a/src/dashboard-refactor/search-results/types.ts +++ b/src/dashboard-refactor/search-results/types.ts @@ -142,7 +142,7 @@ export interface NoteData { selector?: Anchor isShared?: boolean isBulkShareProtected?: boolean - color?: RGBAColor + color?: string isInFocus?: boolean } diff --git a/src/dashboard-refactor/search-results/util.ts b/src/dashboard-refactor/search-results/util.ts index d28c0eef7c..49e01a326b 100644 --- a/src/dashboard-refactor/search-results/util.ts +++ b/src/dashboard-refactor/search-results/util.ts @@ -177,12 +177,12 @@ const annotationToNoteData = ( return { url: annotation.url, pageUrl: annotation.pageUrl, - highlight: annotation.body, + highlight: annotation['body'], comment: annotation.comment, - tags: annotation.tags ?? [], + tags: [], lists, - selector: annotation.selector, - color: annotation.color as RGBAColor, + selector: annotation['selector'], + color: annotation['color'], createdWhen: annotation.createdWhen, displayTime: new Date( annotation.lastEdited ?? annotation.createdWhen, @@ -193,8 +193,8 @@ const annotationToNoteData = ( ...getInitialNoteResultState(), editNoteForm: { inputValue: annotation.comment ?? '', - bodyInputValue: annotation.body ?? '', - tags: annotation.tags ?? [], + bodyInputValue: annotation['body'] ?? '', + tags: [], lists, isTagPickerShown: false, isListPickerShown: false, diff --git a/src/search/background/types.ts b/src/search/background/types.ts index 981666670f..bc1beee91c 100644 --- a/src/search/background/types.ts +++ b/src/search/background/types.ts @@ -1,8 +1,7 @@ import type { User } from 'src/social-integration/types' import type SearchStorage from './storage' -import type { Annotation } from 'src/annotations/types' import type { PageIndexingBackground } from 'src/page-indexing/background' -import type { Annotation as _Annotation } from '@worldbrain/memex-common/lib/types/core-data-types/client' +import type { Annotation } from '@worldbrain/memex-common/lib/types/core-data-types/client' import type { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types' export type SearchResultAnnotation = Annotation & { @@ -141,7 +140,7 @@ export type UnifiedTermsSearchParams = UnifiedSearchParams & queryAnnotations: ( terms: string[], phrases?: string[], - ) => Promise<_Annotation[]> + ) => Promise } /** From 9ae7c1e76a29297025de8175cc5264cd6e99a11c Mon Sep 17 00:00:00 2001 From: Jonathan Poltak Samosir Date: Sun, 9 Jun 2024 12:57:25 +0700 Subject: [PATCH 2/3] Move all terms search logic to memex-common --- external/@worldbrain/memex-common | 2 +- .../AnnotationSearchCopyPaster.tsx | 2 +- src/copy-paster/PageSearchCopyPaster.tsx | 2 +- src/copy-paster/background/types.ts | 2 +- .../components/search-copy-paster.tsx | 2 +- src/dashboard-refactor/search-results/util.ts | 2 +- src/dashboard-refactor/util.ts | 8 +- src/search/background/index.ts | 67 +++------------- src/search/background/types.ts | 79 ++----------------- src/search/background/unified-search.test.ts | 5 +- src/search/background/utils.ts | 37 +-------- 11 files changed, 31 insertions(+), 177 deletions(-) diff --git a/external/@worldbrain/memex-common b/external/@worldbrain/memex-common index daaa08a8eb..dca9cbcdfd 160000 --- a/external/@worldbrain/memex-common +++ b/external/@worldbrain/memex-common @@ -1 +1 @@ -Subproject commit daaa08a8eb68ecc74460648968e5119891b050b5 +Subproject commit dca9cbcdfd0e3f6dce2c414cabc68193eeb8727b diff --git a/src/copy-paster/AnnotationSearchCopyPaster.tsx b/src/copy-paster/AnnotationSearchCopyPaster.tsx index 15963bcbea..5a1e4a4d80 100644 --- a/src/copy-paster/AnnotationSearchCopyPaster.tsx +++ b/src/copy-paster/AnnotationSearchCopyPaster.tsx @@ -1,7 +1,7 @@ import React from 'react' import CopyPaster, { Props as CopyPasterProps } from './CopyPaster' -import type { UnifiedSearchParams } from 'src/search/background/types' import type { Template } from './types' +import type { UnifiedSearchParams } from '@worldbrain/memex-common/lib/search/types' export interface Props extends Omit { diff --git a/src/copy-paster/PageSearchCopyPaster.tsx b/src/copy-paster/PageSearchCopyPaster.tsx index ba7e27c948..fb46326d56 100644 --- a/src/copy-paster/PageSearchCopyPaster.tsx +++ b/src/copy-paster/PageSearchCopyPaster.tsx @@ -1,7 +1,7 @@ import React from 'react' import CopyPaster, { Props as CopyPasterProps } from './CopyPaster' -import type { UnifiedSearchParams } from 'src/search/background/types' import type { Template } from './types' +import type { UnifiedSearchParams } from '@worldbrain/memex-common/lib/search/types' export interface Props extends Omit { diff --git a/src/copy-paster/background/types.ts b/src/copy-paster/background/types.ts index 4600cb7b48..bd273b2203 100644 --- a/src/copy-paster/background/types.ts +++ b/src/copy-paster/background/types.ts @@ -1,5 +1,5 @@ import type { Template } from '../types' -import type { UnifiedSearchParams } from 'src/search/background/types' +import type { UnifiedSearchParams } from '@worldbrain/memex-common/lib/search/types' export interface RemoteCopyPasterInterface { createTemplate(args: Omit): Promise diff --git a/src/dashboard-refactor/search-results/components/search-copy-paster.tsx b/src/dashboard-refactor/search-results/components/search-copy-paster.tsx index 61349e48de..eb56130b39 100644 --- a/src/dashboard-refactor/search-results/components/search-copy-paster.tsx +++ b/src/dashboard-refactor/search-results/components/search-copy-paster.tsx @@ -6,7 +6,7 @@ import { PageSearchCopyPaster, } from 'src/copy-paster' import { SearchType } from '../types' -import { UnifiedSearchParams } from 'src/search/background/types' +import type { UnifiedSearchParams } from '@worldbrain/memex-common/lib/search/types' import Icon from '@worldbrain/memex-common/lib/common-ui/components/icon' import { PopoutBox } from '@worldbrain/memex-common/lib/common-ui/components/popout-box' import { TooltipBox } from '@worldbrain/memex-common/lib/common-ui/components/tooltip-box' diff --git a/src/dashboard-refactor/search-results/util.ts b/src/dashboard-refactor/search-results/util.ts index 49e01a326b..2768185d80 100644 --- a/src/dashboard-refactor/search-results/util.ts +++ b/src/dashboard-refactor/search-results/util.ts @@ -1,9 +1,9 @@ import type { StandardSearchResponse, SearchResultPage, - UnifiedSearchPaginationParams, SearchResultAnnotation, } from 'src/search/background/types' +import type { UnifiedSearchPaginationParams } from '@worldbrain/memex-common/lib/search/types' import type { PageData, PageResult, diff --git a/src/dashboard-refactor/util.ts b/src/dashboard-refactor/util.ts index 9c44fdc3b1..8fd58af46b 100644 --- a/src/dashboard-refactor/util.ts +++ b/src/dashboard-refactor/util.ts @@ -3,13 +3,7 @@ import type { TermsSearchOpts, UnifiedSearchPaginationParams, UnifiedSearchParams, -} from 'src/search/background/types' -import type { PageResult } from './search-results/types' -import { - initNormalizedState, - NormalizedState, -} from '@worldbrain/memex-common/lib/common-ui/utils/normalized-state' -import { EVENT_PROVIDER_DOMAINS } from '@worldbrain/memex-common/lib/constants' +} from '@worldbrain/memex-common/lib/search/types' import type { PageAnnotationsCacheInterface, UnifiedList, diff --git a/src/search/background/index.ts b/src/search/background/index.ts index 1b10b5629d..873aeaf21b 100644 --- a/src/search/background/index.ts +++ b/src/search/background/index.ts @@ -7,14 +7,20 @@ import { makeRemotelyCallable } from 'src/util/webextensionRPC' import type { RemoteSearchInterface, SearchResultPage, - UnifiedSearchParams, UnifiedBlankSearchParams, - UnifiedTermsSearchParams, + UnifiedSearchLookupData, +} from './types' +import type { ResultDataByPage, + UnifiedSearchParams, IntermediarySearchResult, UnifiedSearchPaginationParams, - UnifiedSearchLookupData, -} from './types' +} from '@worldbrain/memex-common/lib/search/types' +import { + needToFilterSearchByUrl, + sortSearchResult, +} from '@worldbrain/memex-common/lib/search/utils' +import { unifiedTermsSearch } from '@worldbrain/memex-common/lib/search/terms-search' import { SearchError, BadTermError } from './errors' import type { PageIndexingBackground } from 'src/page-indexing/background' import { @@ -38,12 +44,9 @@ import type { FavIcon, } from '@worldbrain/memex-common/lib/types/core-data-types/client' import { - sortSearchResult, queryAnnotationsByTerms, queryPagesByTerms, splitQueryIntoTerms, - sliceSearchResult, - needToFilterSearchByUrl, } from './utils' import { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types' import { SPECIAL_LIST_IDS } from '@worldbrain/memex-common/lib/storage/modules/lists/constants' @@ -678,53 +681,6 @@ export default class SearchBackground { } } - private async unifiedTermsSearch( - params: UnifiedTermsSearchParams, - ): Promise { - const resultDataByPage: ResultDataByPage = new Map() - - const [pages, annotations] = await Promise.all([ - params.queryPages(params.terms, params.phrases), - params.queryAnnotations(params.terms, params.phrases), - ]) - - // Add in all the annotations to the results - const annotsByPage = groupBy(annotations, (a) => a.pageUrl) - for (const [pageId, annots] of Object.entries(annotsByPage)) { - const sortedAnnots = annots.sort( - (a, b) => b.lastEdited.valueOf() - a.lastEdited.valueOf(), - ) - resultDataByPage.set(pageId, { - annotations: sortedAnnots, - // This gets overwritten in the next loop by the latest visit/bookmark time (if exist in this results "page") - latestPageTimestamp: sortedAnnots[0].lastEdited.valueOf(), - oldestTimestamp: 0, - }) - } - - // Add in all the pages to the results - for (const { id, latestTimestamp } of pages) { - const annotations = resultDataByPage.get(id)?.annotations ?? [] - resultDataByPage.set(id, { - annotations, - latestPageTimestamp: latestTimestamp, - oldestTimestamp: 0, - }) - } - - await this.filterUnifiedSearchResultsByFilters(resultDataByPage, params) - - // Paginate! - const sortedResultPages = sortSearchResult(resultDataByPage) - const paginatedResults = sliceSearchResult(sortedResultPages, params) - - return { - oldestResultTimestamp: null, - resultDataByPage: paginatedResults, - resultsExhausted: paginatedResults.size < params.limit, - } - } - private async unifiedIntermediarySearch( params: UnifiedSearchParams & UnifiedSearchPaginationParams, ): Promise { @@ -781,8 +737,9 @@ export default class SearchBackground { params.phrases = phrases params.terms = terms params.matchTermsFuzzyStartsWith = matchTermsFuzzyStartsWith - return this.unifiedTermsSearch({ + return unifiedTermsSearch({ ...params, + storageManager: this.options.storageManager, queryPages: queryPagesByTerms(this.options.storageManager, params), queryAnnotations: queryAnnotationsByTerms( this.options.storageManager, diff --git a/src/search/background/types.ts b/src/search/background/types.ts index bc1beee91c..79d22926da 100644 --- a/src/search/background/types.ts +++ b/src/search/background/types.ts @@ -1,8 +1,11 @@ -import type { User } from 'src/social-integration/types' import type SearchStorage from './storage' import type { PageIndexingBackground } from 'src/page-indexing/background' import type { Annotation } from '@worldbrain/memex-common/lib/types/core-data-types/client' import type { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types' +import type { + UnifiedSearchParams, + UnifiedSearchPaginationParams, +} from '@worldbrain/memex-common/lib/search/types' export type SearchResultAnnotation = Annotation & { lists: number[] @@ -24,6 +27,9 @@ export interface SearchResultPage { text: string } +/** + * @deprecated + */ export interface AnnotSearchParams { /** Main text to search against annot (pre-processed). */ query?: string @@ -72,13 +78,6 @@ export interface AnnotsByPageUrl { [pageUrl: string]: Annotation[] } -export interface SocialSearchParams extends AnnotSearchParams { - usersInc?: User[] - usersExc?: User[] - hashtagsInc?: string[] - hashtagsExc?: string[] -} - export interface StandardSearchResponse { resultsExhausted: boolean totalCount?: number @@ -102,47 +101,6 @@ export interface RemoteSearchInterface { }> } -export type UnifiedSearchParams = TermsSearchOpts & { - query: string - terms?: string[] - phrases?: string[] - fromWhen?: number - untilWhen?: number - filterByDomains: string[] - filterByListIds: number[] - filterByPDFs?: boolean - filterByVideos?: boolean - filterByTweets?: boolean - filterByEvents?: boolean - omitPagesWithoutAnnotations?: boolean -} - -export interface TermsSearchOpts { - /** Set to enable fuzzy startsWith terms matching, instead of exact match. */ - matchTermsFuzzyStartsWith?: boolean - matchPageText?: boolean - matchPageTitleUrl?: boolean - matchHighlights?: boolean - matchNotes?: boolean -} - -export interface UnifiedSearchPaginationParams { - skip: number - limit: number -} - -export type UnifiedTermsSearchParams = UnifiedSearchParams & - UnifiedSearchPaginationParams & { - queryPages: ( - terms: string[], - phrases?: string[], - ) => Promise> - queryAnnotations: ( - terms: string[], - phrases?: string[], - ) => Promise - } - /** * Note that, unlike terms search, blank search does not use the traditional pagination params. * Instead it expects the caller to keep a state of the oldest search result so far @@ -163,29 +121,6 @@ export type UnifiedSearchResult = { oldestResultTimestamp: number } -export type IntermediarySearchResult = { - resultsExhausted: boolean - resultDataByPage: ResultDataByPage - /** Always null for terms search. */ - oldestResultTimestamp: number | null -} - -export type ResultDataByPage = Map - -export type UnifiedSearchPageResultData = { - annotations: Annotation[] - /** - * Contains the latest timestamp associated with the page bookmark or visits. Not annotation. - * Used primarily for sorting and display in UI. - */ - latestPageTimestamp: number - /** - * Contains the oldest timestamp associated with the page, including bookmarks, visits, or annotations. - * Used for paging back through blank search results (oldest timestamp of batch N is the upper bound for batch N+1) - */ - oldestTimestamp: number -} - export type UnifiedSearchLookupData = { pages: Map> annotations: Map diff --git a/src/search/background/unified-search.test.ts b/src/search/background/unified-search.test.ts index ce9c85d2b8..7a385a55b9 100644 --- a/src/search/background/unified-search.test.ts +++ b/src/search/background/unified-search.test.ts @@ -2,14 +2,15 @@ import omit from 'lodash/omit' import type Storex from '@worldbrain/storex' import * as DATA from './unified-search.test.data' import { setupBackgroundIntegrationTest } from 'src/tests/background-integration-tests' -import { sortSearchResult, splitQueryIntoTerms } from './utils' +import { splitQueryIntoTerms } from './utils' import type { UnifiedSearchPaginationParams, IntermediarySearchResult, UnifiedSearchParams, -} from './types' +} from '@worldbrain/memex-common/lib/search/types' import type { BackgroundModules } from 'src/background-script/setup' import { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types' +import { sortSearchResult } from '@worldbrain/memex-common/lib/search/utils' async function insertTestData(storageManager: Storex) { for (const doc of Object.values(DATA.LISTS)) { diff --git a/src/search/background/utils.ts b/src/search/background/utils.ts index 8cd917bc8f..e521f882e1 100644 --- a/src/search/background/utils.ts +++ b/src/search/background/utils.ts @@ -1,12 +1,8 @@ import type Storex from '@worldbrain/storex' import type { - ResultDataByPage, - TermsSearchOpts, - UnifiedSearchPageResultData, - UnifiedSearchPaginationParams, - UnifiedSearchParams, UnifiedTermsSearchParams, -} from './types' + TermsSearchOpts, +} from '@worldbrain/memex-common/lib/search/types' import type { SearchParams as OldSearchParams } from '../types' import type { Page, @@ -66,27 +62,6 @@ export const reshapePageForDisplay = (page) => ({ annotsCount: page.annotsCount, }) -export const sortSearchResult = (resultDataByPage: ResultDataByPage) => - [...resultDataByPage].sort( - ([, a], [, b]) => - Math.max( - b.latestPageTimestamp, - b.annotations[0]?.lastEdited.valueOf() ?? 0, - ) - - Math.max( - a.latestPageTimestamp, - a.annotations[0]?.lastEdited.valueOf() ?? 0, - ), - ) - -export const sliceSearchResult = ( - resultDataByPage: Array<[string, UnifiedSearchPageResultData]>, - { limit, skip }: UnifiedSearchPaginationParams, -): ResultDataByPage => { - // NOTE: Current implementation ignores annotation count, and simply paginates by the number of pages in the results - return new Map(resultDataByPage.slice(skip, skip + limit)) -} - /** Given separate result sets of the same type, gets the intersection of them / ANDs them together by ID */ const intersectResults = (results: string[][]): string[] => !results.length @@ -304,11 +279,3 @@ export const splitQueryIntoTerms = ( matchTermsFuzzyStartsWith, } } - -export const needToFilterSearchByUrl = (params: UnifiedSearchParams): boolean => - params.filterByDomains.length > 0 || - params.filterByPDFs || - params.filterByVideos || - params.filterByTweets || - params.filterByEvents || - params.omitPagesWithoutAnnotations From 5c26489552b9a2d909caa5bcba73bdde16145bdc Mon Sep 17 00:00:00 2001 From: Jonathan Poltak Samosir Date: Wed, 12 Jun 2024 10:22:47 +0700 Subject: [PATCH 3/3] Move query terms+phrases split logic to memex-common --- external/@worldbrain/memex-common | 2 +- external/@worldbrain/memex-stemmer | 2 +- .../search-results/index.tsx | 2 +- src/search/background/index.ts | 11 ++- src/search/background/unified-search.test.ts | 2 +- src/search/background/utils.ts | 81 ------------------- 6 files changed, 9 insertions(+), 91 deletions(-) diff --git a/external/@worldbrain/memex-common b/external/@worldbrain/memex-common index dca9cbcdfd..5ee5c68f8b 160000 --- a/external/@worldbrain/memex-common +++ b/external/@worldbrain/memex-common @@ -1 +1 @@ -Subproject commit dca9cbcdfd0e3f6dce2c414cabc68193eeb8727b +Subproject commit 5ee5c68f8b53a225a29d50a96ad0fbaa17d34b76 diff --git a/external/@worldbrain/memex-stemmer b/external/@worldbrain/memex-stemmer index 5e0f2108cd..f227df770b 160000 --- a/external/@worldbrain/memex-stemmer +++ b/external/@worldbrain/memex-stemmer @@ -1 +1 @@ -Subproject commit 5e0f2108cde4b7060165de462110b4f5f0f2c196 +Subproject commit f227df770bad47fb5090df06407020e10eb936e1 diff --git a/src/dashboard-refactor/search-results/index.tsx b/src/dashboard-refactor/search-results/index.tsx index a33668466a..db3e941f48 100644 --- a/src/dashboard-refactor/search-results/index.tsx +++ b/src/dashboard-refactor/search-results/index.tsx @@ -65,7 +65,7 @@ import { RemoteSyncSettingsInterface } from 'src/sync-settings/background/types' import TutorialBox from '@worldbrain/memex-common/lib/common-ui/components/tutorial-box' import { SpaceSearchSuggestion } from '@worldbrain/memex-common/lib/editor' import type { HighlightColor } from '@worldbrain/memex-common/lib/common-ui/components/highlightColorPicker/types' -import { splitQueryIntoTerms } from 'src/search/background/utils' +import { splitQueryIntoTerms } from '@worldbrain/memex-common/lib/search/terms-search' const timestampToString = (timestamp: number) => timestamp === -1 ? undefined : formatDayGroupTime(timestamp) diff --git a/src/search/background/index.ts b/src/search/background/index.ts index 873aeaf21b..d7c720a359 100644 --- a/src/search/background/index.ts +++ b/src/search/background/index.ts @@ -20,7 +20,10 @@ import { needToFilterSearchByUrl, sortSearchResult, } from '@worldbrain/memex-common/lib/search/utils' -import { unifiedTermsSearch } from '@worldbrain/memex-common/lib/search/terms-search' +import { + splitQueryIntoTerms, + unifiedTermsSearch, +} from '@worldbrain/memex-common/lib/search/terms-search' import { SearchError, BadTermError } from './errors' import type { PageIndexingBackground } from 'src/page-indexing/background' import { @@ -43,11 +46,7 @@ import type { Visit, FavIcon, } from '@worldbrain/memex-common/lib/types/core-data-types/client' -import { - queryAnnotationsByTerms, - queryPagesByTerms, - splitQueryIntoTerms, -} from './utils' +import { queryAnnotationsByTerms, queryPagesByTerms } from './utils' import { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types' import { SPECIAL_LIST_IDS } from '@worldbrain/memex-common/lib/storage/modules/lists/constants' import { diff --git a/src/search/background/unified-search.test.ts b/src/search/background/unified-search.test.ts index 7a385a55b9..0b918cad68 100644 --- a/src/search/background/unified-search.test.ts +++ b/src/search/background/unified-search.test.ts @@ -2,7 +2,6 @@ import omit from 'lodash/omit' import type Storex from '@worldbrain/storex' import * as DATA from './unified-search.test.data' import { setupBackgroundIntegrationTest } from 'src/tests/background-integration-tests' -import { splitQueryIntoTerms } from './utils' import type { UnifiedSearchPaginationParams, IntermediarySearchResult, @@ -11,6 +10,7 @@ import type { import type { BackgroundModules } from 'src/background-script/setup' import { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types' import { sortSearchResult } from '@worldbrain/memex-common/lib/search/utils' +import { splitQueryIntoTerms } from '@worldbrain/memex-common/lib/search/terms-search' async function insertTestData(storageManager: Storex) { for (const doc of Object.values(DATA.LISTS)) { diff --git a/src/search/background/utils.ts b/src/search/background/utils.ts index e521f882e1..d581cdca3b 100644 --- a/src/search/background/utils.ts +++ b/src/search/background/utils.ts @@ -198,84 +198,3 @@ export const queryPagesByTerms = ( latestTimestamp: latestTimestampByPageUrl.get(id) ?? 0, })) } - -export const splitQueryIntoTerms = ( - query: string, -): { - terms: string[] - phrases: string[] - inTitle: boolean - inContent: boolean - inHighlight: boolean - inComment: boolean - matchTermsFuzzyStartsWith: boolean -} => { - const discreteTerms = new Set() - const phrases = new Set() - let queryString = query.trim().toLocaleLowerCase() - let inTitle = true - let inContent = true - let inHighlight = true - let inComment = true - let matchTermsFuzzyStartsWith = false - - if (queryString.includes('*')) { - matchTermsFuzzyStartsWith = true - queryString = queryString.replace(/\*/g, '') - } - const matches = query.match(/"[^"]+"/g) - matches?.forEach((match) => { - phrases.add(match.slice(1, -1)) - queryString = queryString.replace(match, ' ') // Remove the matched phrase from queryString - }) - queryString = queryString.replace(/\s+/g, ' ').trim() - - // Also process out chinese characters - - queryString = processCJKCharacters(queryString) - - // First split by double quotes, then by spaces on non-double quoted phrases - const terms = queryString.split(' ').filter(Boolean) - for (const term of terms) { - const subTerms = term.split(/\s+/).filter(Boolean) - subTerms.forEach((subTerm) => { - switch (subTerm) { - case 'intitle': - inTitle = true - inContent = false - inHighlight = false - inComment = false - break - case 'incontent': - inTitle = false - inContent = true - inHighlight = false - inComment = false - break - case 'inhighlight': - inTitle = false - inContent = false - inHighlight = true - inComment = false - break - case 'incomment': - inTitle = false - inContent = false - inHighlight = false - inComment = true - break - default: - discreteTerms.add(subTerm) - } - }) - } - return { - terms: [...discreteTerms], - phrases: [...phrases], - inTitle, - inContent, - inHighlight, - inComment, - matchTermsFuzzyStartsWith, - } -}