Merge branch 'feature/mobile-search' into hotfix/3.19.52

# Conflicts:
#	external/@worldbrain/memex-common

Author: poltak

external/@worldbrain/memex-common

@@ -1,7 +1,7 @@
 import React from 'react'
 import CopyPaster, { Props as CopyPasterProps } from './CopyPaster'
-import type { UnifiedSearchParams } from 'src/search/background/types'
 import type { Template } from './types'
+import type { UnifiedSearchParams } from '@worldbrain/memex-common/lib/search/types'
 
 export interface Props
     extends Omit<CopyPasterProps, 'renderTemplate' | 'renderPreview'> {

external/@worldbrain/memex-stemmer

@@ -1,7 +1,7 @@
 import React from 'react'
 import CopyPaster, { Props as CopyPasterProps } from './CopyPaster'
-import type { UnifiedSearchParams } from 'src/search/background/types'
 import type { Template } from './types'
+import type { UnifiedSearchParams } from '@worldbrain/memex-common/lib/search/types'
 
 export interface Props
     extends Omit<CopyPasterProps, 'renderTemplate' | 'renderPreview'> {

src/copy-paster/AnnotationSearchCopyPaster.tsx

@@ -1,5 +1,5 @@
 import type { Template } from '../types'
-import type { UnifiedSearchParams } from 'src/search/background/types'
+import type { UnifiedSearchParams } from '@worldbrain/memex-common/lib/search/types'
 
 export interface RemoteCopyPasterInterface {
     createTemplate(args: Omit<Template, 'id'>): Promise<number>

src/copy-paster/PageSearchCopyPaster.tsx

@@ -54,7 +54,7 @@ export const setPageSearchResult: DataSeederCreator<StandardSearchResponse> = (
             await storageManager.collection('annotations').createObject({
                 ...annot,
             })
-            if (annot.isShared) {
+            if (annot['isShared']) {
                 await storageManager
                     .collection('sharedAnnotationMetadata')
                     .createObject({
@@ -68,7 +68,7 @@ export const setPageSearchResult: DataSeederCreator<StandardSearchResponse> = (
                         id: idCounter++,
                         annotation: annot.url,
                         createdWhen: new Date(),
-                        privacyLevel: annot.isBulkShareProtected
+                        privacyLevel: annot['isBulkShareProtected']
                             ? AnnotationPrivacyLevels.SHARED_PROTECTED
                             : AnnotationPrivacyLevels.SHARED,
                     })

src/copy-paster/background/types.ts

@@ -1581,7 +1581,7 @@ export class DashboardLogic extends UILogic<State, Events> {
                             byId: {
                                 [event.noteId]: {
                                     comment: { $set: existing.comment },
-                                    color: { $set: event.color as RGBAColor },
+                                    color: { $set: event.color as string },
                                 },
                             },
                         },
@@ -3674,7 +3674,7 @@ export class DashboardLogic extends UILogic<State, Events> {
                                     lists: { $set: nextLists },
                                     color: {
                                         $set: (event.color ??
-                                            existing.color) as RGBAColor,
+                                            existing.color) as string,
                                     },
                                 },
                             },

src/dashboard-refactor/logic.test.util.ts

@@ -6,7 +6,7 @@ import {
     PageSearchCopyPaster,
 } from 'src/copy-paster'
 import { SearchType } from '../types'
-import { UnifiedSearchParams } from 'src/search/background/types'
+import type { UnifiedSearchParams } from '@worldbrain/memex-common/lib/search/types'
 import Icon from '@worldbrain/memex-common/lib/common-ui/components/icon'
 import { PopoutBox } from '@worldbrain/memex-common/lib/common-ui/components/popout-box'
 import { TooltipBox } from '@worldbrain/memex-common/lib/common-ui/components/tooltip-box'

src/dashboard-refactor/logic.ts

@@ -65,7 +65,7 @@ import { RemoteSyncSettingsInterface } from 'src/sync-settings/background/types'
 import TutorialBox from '@worldbrain/memex-common/lib/common-ui/components/tutorial-box'
 import { SpaceSearchSuggestion } from '@worldbrain/memex-common/lib/editor'
 import type { HighlightColor } from '@worldbrain/memex-common/lib/common-ui/components/highlightColorPicker/types'
-import { splitQueryIntoTerms } from 'src/search/background/utils'
+import { splitQueryIntoTerms } from '@worldbrain/memex-common/lib/search/terms-search'
 
 const timestampToString = (timestamp: number) =>
     timestamp === -1 ? undefined : formatDayGroupTime(timestamp)

src/dashboard-refactor/search-results/components/search-copy-paster.tsx

@@ -142,7 +142,7 @@ export interface NoteData {
     selector?: Anchor
     isShared?: boolean
     isBulkShareProtected?: boolean
-    color?: RGBAColor
+    color?: string
     isInFocus?: boolean
 }
 

src/dashboard-refactor/search-results/index.tsx

@@ -1,9 +1,9 @@
 import type {
     StandardSearchResponse,
     SearchResultPage,
-    UnifiedSearchPaginationParams,
     SearchResultAnnotation,
 } from 'src/search/background/types'
+import type { UnifiedSearchPaginationParams } from '@worldbrain/memex-common/lib/search/types'
 import type {
     PageData,
     PageResult,
@@ -177,12 +177,12 @@ const annotationToNoteData = (
     return {
         url: annotation.url,
         pageUrl: annotation.pageUrl,
-        highlight: annotation.body,
+        highlight: annotation['body'],
         comment: annotation.comment,
-        tags: annotation.tags ?? [],
+        tags: [],
         lists,
-        selector: annotation.selector,
-        color: annotation.color as RGBAColor,
+        selector: annotation['selector'],
+        color: annotation['color'],
         createdWhen: annotation.createdWhen,
         displayTime: new Date(
             annotation.lastEdited ?? annotation.createdWhen,
@@ -193,8 +193,8 @@ const annotationToNoteData = (
         ...getInitialNoteResultState(),
         editNoteForm: {
             inputValue: annotation.comment ?? '',
-            bodyInputValue: annotation.body ?? '',
-            tags: annotation.tags ?? [],
+            bodyInputValue: annotation['body'] ?? '',
+            tags: [],
             lists,
             isTagPickerShown: false,
             isListPickerShown: false,

src/dashboard-refactor/search-results/types.ts

@@ -3,13 +3,7 @@ import type {
     TermsSearchOpts,
     UnifiedSearchPaginationParams,
     UnifiedSearchParams,
-} from 'src/search/background/types'
-import type { PageResult } from './search-results/types'
-import {
-    initNormalizedState,
-    NormalizedState,
-} from '@worldbrain/memex-common/lib/common-ui/utils/normalized-state'
-import { EVENT_PROVIDER_DOMAINS } from '@worldbrain/memex-common/lib/constants'
+} from '@worldbrain/memex-common/lib/search/types'
 import type {
     PageAnnotationsCacheInterface,
     UnifiedList,

src/dashboard-refactor/search-results/util.ts

@@ -7,14 +7,23 @@ import { makeRemotelyCallable } from 'src/util/webextensionRPC'
 import type {
     RemoteSearchInterface,
     SearchResultPage,
-    UnifiedSearchParams,
     UnifiedBlankSearchParams,
-    UnifiedTermsSearchParams,
+    UnifiedSearchLookupData,
+} from './types'
+import type {
     ResultDataByPage,
+    UnifiedSearchParams,
     IntermediarySearchResult,
     UnifiedSearchPaginationParams,
-    UnifiedSearchLookupData,
-} from './types'
+} from '@worldbrain/memex-common/lib/search/types'
+import {
+    needToFilterSearchByUrl,
+    sortSearchResult,
+} from '@worldbrain/memex-common/lib/search/utils'
+import {
+    splitQueryIntoTerms,
+    unifiedTermsSearch,
+} from '@worldbrain/memex-common/lib/search/terms-search'
 import { SearchError, BadTermError } from './errors'
 import type { PageIndexingBackground } from 'src/page-indexing/background'
 import {
@@ -37,14 +46,7 @@ import type {
     Visit,
     FavIcon,
 } from '@worldbrain/memex-common/lib/types/core-data-types/client'
-import {
-    sortSearchResult,
-    queryAnnotationsByTerms,
-    queryPagesByTerms,
-    splitQueryIntoTerms,
-    sliceSearchResult,
-    needToFilterSearchByUrl,
-} from './utils'
+import { queryAnnotationsByTerms, queryPagesByTerms } from './utils'
 import { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types'
 import { SPECIAL_LIST_IDS } from '@worldbrain/memex-common/lib/storage/modules/lists/constants'
 import {
@@ -678,53 +680,6 @@ export default class SearchBackground {
         }
     }
 
-    private async unifiedTermsSearch(
-        params: UnifiedTermsSearchParams,
-    ): Promise<IntermediarySearchResult> {
-        const resultDataByPage: ResultDataByPage = new Map()
-
-        const [pages, annotations] = await Promise.all([
-            params.queryPages(params.terms, params.phrases),
-            params.queryAnnotations(params.terms, params.phrases),
-        ])
-
-        // Add in all the annotations to the results
-        const annotsByPage = groupBy(annotations, (a) => a.pageUrl)
-        for (const [pageId, annots] of Object.entries(annotsByPage)) {
-            const sortedAnnots = annots.sort(
-                (a, b) => b.lastEdited.valueOf() - a.lastEdited.valueOf(),
-            )
-            resultDataByPage.set(pageId, {
-                annotations: sortedAnnots,
-                // This gets overwritten in the next loop by the latest visit/bookmark time (if exist in this results "page")
-                latestPageTimestamp: sortedAnnots[0].lastEdited.valueOf(),
-                oldestTimestamp: 0,
-            })
-        }
-
-        // Add in all the pages to the results
-        for (const { id, latestTimestamp } of pages) {
-            const annotations = resultDataByPage.get(id)?.annotations ?? []
-            resultDataByPage.set(id, {
-                annotations,
-                latestPageTimestamp: latestTimestamp,
-                oldestTimestamp: 0,
-            })
-        }
-
-        await this.filterUnifiedSearchResultsByFilters(resultDataByPage, params)
-
-        // Paginate!
-        const sortedResultPages = sortSearchResult(resultDataByPage)
-        const paginatedResults = sliceSearchResult(sortedResultPages, params)
-
-        return {
-            oldestResultTimestamp: null,
-            resultDataByPage: paginatedResults,
-            resultsExhausted: paginatedResults.size < params.limit,
-        }
-    }
-
     private async unifiedIntermediarySearch(
         params: UnifiedSearchParams & UnifiedSearchPaginationParams,
     ): Promise<IntermediarySearchResult> {
@@ -781,8 +736,9 @@ export default class SearchBackground {
         params.phrases = phrases
         params.terms = terms
         params.matchTermsFuzzyStartsWith = matchTermsFuzzyStartsWith
-        return this.unifiedTermsSearch({
+        return unifiedTermsSearch({
             ...params,
+            storageManager: this.options.storageManager,
             queryPages: queryPagesByTerms(this.options.storageManager, params),
             queryAnnotations: queryAnnotationsByTerms(
                 this.options.storageManager,

src/dashboard-refactor/util.ts

@@ -1,9 +1,11 @@
-import type { User } from 'src/social-integration/types'
 import type SearchStorage from './storage'
-import type { Annotation } from 'src/annotations/types'
 import type { PageIndexingBackground } from 'src/page-indexing/background'
-import type { Annotation as _Annotation } from '@worldbrain/memex-common/lib/types/core-data-types/client'
+import type { Annotation } from '@worldbrain/memex-common/lib/types/core-data-types/client'
 import type { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types'
+import type {
+    UnifiedSearchParams,
+    UnifiedSearchPaginationParams,
+} from '@worldbrain/memex-common/lib/search/types'
 
 export type SearchResultAnnotation = Annotation & {
     lists: number[]
@@ -25,6 +27,9 @@ export interface SearchResultPage {
     text: string
 }
 
+/**
+ * @deprecated
+ */
 export interface AnnotSearchParams {
     /** Main text to search against annot (pre-processed). */
     query?: string
@@ -73,13 +78,6 @@ export interface AnnotsByPageUrl {
     [pageUrl: string]: Annotation[]
 }
 
-export interface SocialSearchParams extends AnnotSearchParams {
-    usersInc?: User[]
-    usersExc?: User[]
-    hashtagsInc?: string[]
-    hashtagsExc?: string[]
-}
-
 export interface StandardSearchResponse {
     resultsExhausted: boolean
     totalCount?: number
@@ -103,47 +101,6 @@ export interface RemoteSearchInterface {
     }>
 }
 
-export type UnifiedSearchParams = TermsSearchOpts & {
-    query: string
-    terms?: string[]
-    phrases?: string[]
-    fromWhen?: number
-    untilWhen?: number
-    filterByDomains: string[]
-    filterByListIds: number[]
-    filterByPDFs?: boolean
-    filterByVideos?: boolean
-    filterByTweets?: boolean
-    filterByEvents?: boolean
-    omitPagesWithoutAnnotations?: boolean
-}
-
-export interface TermsSearchOpts {
-    /** Set to enable fuzzy startsWith terms matching, instead of exact match. */
-    matchTermsFuzzyStartsWith?: boolean
-    matchPageText?: boolean
-    matchPageTitleUrl?: boolean
-    matchHighlights?: boolean
-    matchNotes?: boolean
-}
-
-export interface UnifiedSearchPaginationParams {
-    skip: number
-    limit: number
-}
-
-export type UnifiedTermsSearchParams = UnifiedSearchParams &
-    UnifiedSearchPaginationParams & {
-        queryPages: (
-            terms: string[],
-            phrases?: string[],
-        ) => Promise<Array<{ id: string; latestTimestamp: number }>>
-        queryAnnotations: (
-            terms: string[],
-            phrases?: string[],
-        ) => Promise<_Annotation[]>
-    }
-
 /**
  * Note that, unlike terms search, blank search does not use the traditional pagination params.
  * Instead it expects the caller to keep a state of the oldest search result so far
@@ -164,29 +121,6 @@ export type UnifiedSearchResult = {
     oldestResultTimestamp: number
 }
 
-export type IntermediarySearchResult = {
-    resultsExhausted: boolean
-    resultDataByPage: ResultDataByPage
-    /** Always null for terms search. */
-    oldestResultTimestamp: number | null
-}
-
-export type ResultDataByPage = Map<string, UnifiedSearchPageResultData>
-
-export type UnifiedSearchPageResultData = {
-    annotations: Annotation[]
-    /**
-     * Contains the latest timestamp associated with the page bookmark or visits. Not annotation.
-     * Used primarily for sorting and display in UI.
-     */
-    latestPageTimestamp: number
-    /**
-     * Contains the oldest timestamp associated with the page, including bookmarks, visits, or annotations.
-     * Used for paging back through blank search results (oldest timestamp of batch N is the upper bound for batch N+1)
-     */
-    oldestTimestamp: number
-}
-
 export type UnifiedSearchLookupData = {
     pages: Map<string, Omit<SearchResultPage, 'annotations' | 'hasBookmark'>>
     annotations: Map<string, SearchResultAnnotation>

src/search/background/index.ts

@@ -2,14 +2,15 @@ import omit from 'lodash/omit'
 import type Storex from '@worldbrain/storex'
 import * as DATA from './unified-search.test.data'
 import { setupBackgroundIntegrationTest } from 'src/tests/background-integration-tests'
-import { sortSearchResult, splitQueryIntoTerms } from './utils'
 import type {
     UnifiedSearchPaginationParams,
     IntermediarySearchResult,
     UnifiedSearchParams,
-} from './types'
+} from '@worldbrain/memex-common/lib/search/types'
 import type { BackgroundModules } from 'src/background-script/setup'
 import { AnnotationPrivacyLevels } from '@worldbrain/memex-common/lib/annotations/types'
+import { sortSearchResult } from '@worldbrain/memex-common/lib/search/utils'
+import { splitQueryIntoTerms } from '@worldbrain/memex-common/lib/search/terms-search'
 
 async function insertTestData(storageManager: Storex) {
     for (const doc of Object.values(DATA.LISTS)) {