Merge pull request #3339 from obsidian-tasks-group/cache-user-search-data

spike: Experimentally cache data from query.allTasks calculations

Author: claremacrae

resources/sample_vaults/Tasks-Demo/Functions/Custom Filters With Complex Caching.md

@@ -0,0 +1,86 @@
+# Custom Filters With Complex Caching
+
+> [!error] WARNING
+> These searches use a `query.searchCache` mechanism that is experimental, and may be changed or removed at any time: it is not recommended for use by users at this stage.
+
+This is too complex an example for a general demonstration, or for the documentation.
+
+But it was useful for confirming my (Clare's) understanding of the behaviour.
+
+## Find tasks with duplicate descriptions - counts in group names differ from number of displayed
+
+Show Tasks with more than one instance of the same description.
+
+The counting is done across all tasks in the vault, including ones that do not match the Global Query, so some of the tasks counts in the groups are obviously wrong.
+
+```tasks
+not done
+
+filter by function { \
+    const cacheKey = 'descriptionCountsAllTasks'; \
+    const getDescription = (t) => t.descriptionWithoutTags; \
+    if (!query.searchCache[cacheKey]) { \
+        console.log('Computing and caching description counts...'); \
+        const taskCounts = new Map(); \
+        query.allTasks.forEach(t => { \
+            const group = getDescription(t); \
+            taskCounts.set(group, (taskCounts.get(group) || 0) + 1); \
+        }); \
+        query.searchCache[cacheKey] = taskCounts; \
+    } \
+    const taskCounts = query.searchCache[cacheKey]; \
+    const group = getDescription(task); \
+    const counts = taskCounts.get(group); \
+    return counts > 1; \
+}
+
+group by function { \
+    const cacheKey = 'descriptionCountsAllTasks'; \
+    const getDescription = (t) => t.descriptionWithoutTags; \
+    const group = getDescription(task); \
+    const counts = query.searchCache[cacheKey].get(group); \
+    return `%%${1000000 - counts}%%` + group + " (" + (counts || 0) + " tasks)"; \
+}
+```
+
+## Find tasks with duplicate descriptions - counts in group names match the number displayed
+
+Show Tasks with more than one instance of the same description.
+
+The counting is done only on tasks that match the query, by splitting it across the final two filter instructions in the search.
+
+```tasks
+not done
+
+# Count the number of instances of descriptions that match this query.
+# This must be the second-from-last filter in the query, to ensure the
+# counts are accurate.
+filter by function { \
+    const cacheKey = 'descriptionCountsForMatchingTasks'; \
+    if (!query.searchCache[cacheKey]) { \
+        console.log('Initialising description counts map...'); \
+        const taskCounts = new Map(); \
+        query.searchCache[cacheKey] = taskCounts; \
+    } \
+    const group = task.descriptionWithoutTags; \
+    taskCounts = query.searchCache[cacheKey]; \
+    taskCounts.set(group, (taskCounts.get(group) || 0) + 1); \
+    return true; \
+}
+
+# Filter out the instances with fewer than 2 actual matches...
+# This must be the last filter in the query.
+filter by function { \
+    const cacheKey = 'descriptionCountsForMatchingTasks'; \
+    const group = task.descriptionWithoutTags; \
+    const count = query.searchCache[cacheKey].get(group); \
+    return count > 1; \
+}
+
+group by function { \
+    const cacheKey = 'descriptionCountsForMatchingTasks'; \
+    const group = task.descriptionWithoutTags; \
+    const count = query.searchCache[cacheKey].get(group); \
+    return `%%${1000000 - count}%%` + group + " (" + (count || 0) + " tasks)"; \
+}
+```

resources/sample_vaults/Tasks-Demo/Functions/Custom Filters With Simple Caching.md

@@ -0,0 +1,44 @@
+# Custom Filters With Simple Caching
+
+> [!error] WARNING
+> These searches use a `query.searchCache` mechanism that is experimental, and may be changed or removed at any time: it is not recommended for use by users at this stage.
+
+## Find tasks with duplicate non-empty IDs - slow on many tasks
+
+```tasks
+filter by function { \
+    const thisValue = task.id; \
+    if (thisValue === '') return false; \
+    let count = 0; \
+    query.allTasks.forEach(t => { \
+        if (t.id === thisValue) count += 1; \
+    }); \
+    return (count > 1); \
+}
+
+group by id
+```
+
+## Find tasks with duplicate non-empty IDs - fast on many tasks
+
+```tasks
+filter by function { \
+    const cacheKey = 'idCounts'; \
+    const getValue = (task) => task.id; \
+    if (!query.searchCache[cacheKey]) { \
+        const taskCounts = new Map(); \
+        query.allTasks.forEach(t => { \
+            const group = getValue(t); \
+            taskCounts.set(group, (taskCounts.get(group) || 0) + 1); \
+        }); \
+        query.searchCache[cacheKey] = taskCounts; \
+    } \
+    const taskCounts = query.searchCache[cacheKey]; \
+    const value = getValue(task); \
+    if (value === '') return false; \
+    const count = taskCounts.get(value); \
+    return count > 1; \
+}
+
+group by id
+```

src/Query/SearchInfo.ts

@@ -14,10 +14,12 @@ export class SearchInfo {
      */
     public readonly allTasks: Readonly<Task[]>;
     public readonly tasksFile: OptionalTasksFile;
+    private readonly _queryContext: QueryContext | undefined;
 
     public constructor(tasksFile: OptionalTasksFile, allTasks: Task[]) {
         this.tasksFile = tasksFile;
         this.allTasks = [...allTasks];
+        this._queryContext = this.tasksFile ? makeQueryContextWithTasks(this.tasksFile, this.allTasks) : undefined;
     }
 
     public static fromAllTasks(tasks: Task[]): SearchInfo {
@@ -35,6 +37,6 @@ export class SearchInfo {
      * @return A QueryContext, or undefined if the path to the query file is unknown.
      */
     public queryContext(): QueryContext | undefined {
-        return this.tasksFile ? makeQueryContextWithTasks(this.tasksFile, this.allTasks) : undefined;
+        return this._queryContext;
     }
 }

src/Scripting/QueryContext.ts

@@ -22,6 +22,7 @@ export interface QueryContext {
     query: {
         file: TasksFile;
         allTasks: Readonly<Task[]>;
+        searchCache: Record<string, any>; // Added caching capability
     };
 }
 
@@ -54,6 +55,7 @@ export function makeQueryContextWithTasks(tasksFile: TasksFile, allTasks: Readon
         query: {
             file: tasksFile,
             allTasks: allTasks,
+            searchCache: {}, // Added for caching
         },
     };
 }

tests/Query/SearchInfo.test.ts

@@ -3,6 +3,9 @@ import { TaskBuilder } from '../TestingTools/TaskBuilder';
 import { TasksFile } from '../../src/Scripting/TasksFile';
 
 describe('SearchInfo', () => {
+    const path = 'a/b/c.md';
+    const tasksFile = new TasksFile(path);
+
     it('should not be able to modify SearchInfo.allTasks directly', () => {
         const tasks = [new TaskBuilder().build()];
         const searchInfo = SearchInfo.fromAllTasks(tasks);
@@ -25,16 +28,13 @@ describe('SearchInfo', () => {
     });
 
     it('should provide access to query search path', () => {
-        const path = 'a/b/c.md';
-        const tasksFile = new TasksFile(path);
         const searchInfo = new SearchInfo(tasksFile, []);
 
         expect(searchInfo.tasksFile).toEqual(tasksFile);
         expect(searchInfo.tasksFile?.path).toEqual(path);
     });
 
     it('should create a QueryContext from a known path', () => {
-        const tasksFile = new TasksFile('a/b/c.md');
         const searchInfo = new SearchInfo(tasksFile, []);
 
         const queryContext = searchInfo.queryContext();
@@ -50,4 +50,12 @@ describe('SearchInfo', () => {
 
         expect(queryContext).toBeUndefined();
     });
+
+    it('should give the same QueryContext on successive calls, for caching data', () => {
+        const searchInfo = new SearchInfo(tasksFile, [new TaskBuilder().build()]);
+
+        searchInfo.queryContext()!.query.searchCache['saved'] = 1;
+
+        expect(searchInfo.queryContext()!.query.searchCache['saved']).toBe(1);
+    });
 });

tests/Scripting/QueryContext.test.ts

@@ -55,5 +55,27 @@ describe('QueryContext', () => {
             // Assert
             expect(group).toEqual(['1']);
         });
+
+        it('query.searchCache should be empty initially', () => {
+            // Arrange
+            const searchInfo = new SearchInfo(tasksFile, [task]);
+            const queryContext = searchInfo.queryContext();
+
+            expect(queryContext?.query?.searchCache).toEqual({});
+        });
+
+        it('query.searchCache should cache a value', () => {
+            // Arrange
+            const searchInfo = new SearchInfo(tasksFile, [task]);
+            const queryContext = searchInfo.queryContext();
+            expect(queryContext).not.toBeNull();
+            const cacheKey = 'function1';
+
+            // Act
+            queryContext!.query.searchCache[cacheKey] = 1;
+
+            // Assert
+            expect(queryContext!.query.searchCache[cacheKey]).toEqual(1);
+        });
     });
 });