Merge pull request #3192 from obsidian-tasks-group/refactor-emoji-parsing

Refactor emoji parsing

Author: claremacrae

src/TaskSerializer/DefaultTaskSerializer.ts

@@ -55,6 +55,19 @@ export const taskIdRegex = /[a-zA-Z0-9-_]+/;
 // The allowed characters in a comma-separated sequence of task ids:
 export const taskIdSequenceRegex = new RegExp(taskIdRegex.source + '( *, *' + taskIdRegex.source + ' *)*');
 
+function dateFieldRegex(symbols: string) {
+    return fieldRegex(symbols, '(\\d{4}-\\d{2}-\\d{2})');
+}
+
+function fieldRegex(symbols: string, valueRegexString: string) {
+    let source = symbols;
+    if (valueRegexString !== '') {
+        source += ' *' + valueRegexString;
+    }
+    source += '$';
+    return new RegExp(source, 'u');
+}
+
 /**
  * A symbol map for obsidian-task's default task style.
  * Uses emojis to concisely convey meaning
@@ -83,17 +96,17 @@ export const DEFAULT_SYMBOLS: DefaultTaskSerializerSymbols = {
         // The following regex's end with `$` because they will be matched and
         // removed from the end until none are left.
         // \uFE0F? allows an optional Variant Selector 16 on emojis.
-        priorityRegex: /([🔺⏫🔼🔽⏬])\uFE0F?$/u,
-        startDateRegex: /🛫 *(\d{4}-\d{2}-\d{2})$/u,
-        createdDateRegex: /➕ *(\d{4}-\d{2}-\d{2})$/u,
-        scheduledDateRegex: /[⏳⌛] *(\d{4}-\d{2}-\d{2})$/u,
-        dueDateRegex: /[📅📆🗓] *(\d{4}-\d{2}-\d{2})$/u,
-        doneDateRegex: /✅ *(\d{4}-\d{2}-\d{2})$/u,
-        cancelledDateRegex: /❌ *(\d{4}-\d{2}-\d{2})$/u,
-        recurrenceRegex: /🔁 ?([a-zA-Z0-9, !]+)$/iu,
-        onCompletionRegex: /🏁 *([a-zA-Z]+)$/iu,
-        dependsOnRegex: new RegExp('⛔\uFE0F? *(' + taskIdSequenceRegex.source + ')$', 'iu'),
-        idRegex: new RegExp('🆔 *(' + taskIdRegex.source + ')$', 'iu'),
+        priorityRegex: fieldRegex('([🔺⏫🔼🔽⏬])\uFE0F?', ''),
+        startDateRegex: dateFieldRegex('🛫'),
+        createdDateRegex: dateFieldRegex('➕'),
+        scheduledDateRegex: dateFieldRegex('[⏳⌛]'),
+        dueDateRegex: dateFieldRegex('[📅📆🗓]'),
+        doneDateRegex: dateFieldRegex('✅'),
+        cancelledDateRegex: dateFieldRegex('❌'),
+        recurrenceRegex: fieldRegex('🔁', '([a-zA-Z0-9, !]+)'),
+        onCompletionRegex: fieldRegex('🏁', '([a-zA-Z]+)'),
+        dependsOnRegex: fieldRegex('⛔\uFE0F?', '(' + taskIdSequenceRegex.source + ')'),
+        idRegex: fieldRegex('🆔', '(' + taskIdRegex.source + ')'),
     },
 } as const;
 

src/lib/StringHelpers.ts

@@ -1,3 +1,14 @@
 export function capitalizeFirstLetter(text: string): string {
     return text.charAt(0).toUpperCase() + text.slice(1);
 }
+
+/**
+ * Escape non-visible Variation Selectors characters in string.
+ * Converts them into their Unicode escape sequences for explicit visibility.
+ */
+export function escapeInvisibleCharacters(input: string): string {
+    return input.replace(
+        /[\uFE00-\uFE0F]/g, // Matches Variation Selectors
+        (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`,
+    );
+}

tests/TaskSerializer/DefaultTaskSerializer.test.ts

@@ -13,6 +13,7 @@ import {
 import { TaskBuilder } from '../TestingTools/TaskBuilder';
 import { OnCompletion } from '../../src/Task/OnCompletion';
 import { Priority } from '../../src/Task/Priority';
+import { escapeInvisibleCharacters } from '../../src/lib/StringHelpers';
 
 jest.mock('obsidian');
 window.moment = moment;
@@ -57,23 +58,25 @@ describe('validate emoji regular expressions', () => {
                 throw new Error(`Unexpected value for ${key}: Not a regular expression.`);
             }
         });
-        // Concatenate all entries into a single string
-        return regexDetails.join('\n');
+        // Concatenate all entries into a single string, with any Variation Selectors made visible
+        return escapeInvisibleCharacters('\n' + regexDetails.join('\n') + '\n');
     }
 
     it('regular expressions should have expected source', () => {
         expect(generateRegexApprovalTest()).toMatchInlineSnapshot(`
-            "priorityRegex: /([🔺⏫🔼🔽⏬])\\uFE0F?$/u
+            "
+            priorityRegex: /([🔺⏫🔼🔽⏬])\\ufe0f?$/u
             startDateRegex: /🛫 *(\\d{4}-\\d{2}-\\d{2})$/u
             createdDateRegex: /➕ *(\\d{4}-\\d{2}-\\d{2})$/u
             scheduledDateRegex: /[⏳⌛] *(\\d{4}-\\d{2}-\\d{2})$/u
             dueDateRegex: /[📅📆🗓] *(\\d{4}-\\d{2}-\\d{2})$/u
             doneDateRegex: /✅ *(\\d{4}-\\d{2}-\\d{2})$/u
             cancelledDateRegex: /❌ *(\\d{4}-\\d{2}-\\d{2})$/u
-            recurrenceRegex: /🔁 ?([a-zA-Z0-9, !]+)$/iu
-            onCompletionRegex: /🏁 *([a-zA-Z]+)$/iu
-            dependsOnRegex: /⛔️? *([a-zA-Z0-9-_]+( *, *[a-zA-Z0-9-_]+ *)*)$/iu
-            idRegex: /🆔 *([a-zA-Z0-9-_]+)$/iu"
+            recurrenceRegex: /🔁 *([a-zA-Z0-9, !]+)$/u
+            onCompletionRegex: /🏁 *([a-zA-Z]+)$/u
+            dependsOnRegex: /⛔\\ufe0f? *([a-zA-Z0-9-_]+( *, *[a-zA-Z0-9-_]+ *)*)$/u
+            idRegex: /🆔 *([a-zA-Z0-9-_]+)$/u
+            "
         `);
     });
 });

tests/lib/StringHelpers.test.ts

@@ -0,0 +1,68 @@
+import { escapeInvisibleCharacters } from '../../src/lib/StringHelpers';
+
+describe('StringHelpers', () => {
+    it('escapeInvisibleCharacters', () => {
+        const value = `
+// Variation Selectors
+⛔\uFE00 // VS1
+⛔\uFE01 // VS2
+⛔\uFE02 // VS3
+⛔\uFE03 // VS4
+⛔\uFE04 // VS5
+⛔\uFE05 // VS6
+⛔\uFE06 // VS7
+⛔\uFE07 // VS8
+⛔\uFE08 // VS9
+⛔\uFE09 // VS10
+⛔\uFE0A // VS11
+⛔\uFE0B // VS12
+⛔\uFE0C // VS13
+⛔\uFE0D // VS14
+⛔\uFE0E // VS15
+⛔\uFE0F // VS16
+`;
+        expect(value).toMatchInlineSnapshot(`
+            "
+            // Variation Selectors
+            ⛔︀ // VS1
+            ⛔︁ // VS2
+            ⛔︂ // VS3
+            ⛔︃ // VS4
+            ⛔︄ // VS5
+            ⛔︅ // VS6
+            ⛔︆ // VS7
+            ⛔︇ // VS8
+            ⛔︈ // VS9
+            ⛔︉ // VS10
+            ⛔︊ // VS11
+            ⛔︋ // VS12
+            ⛔︌ // VS13
+            ⛔︍ // VS14
+            ⛔︎ // VS15
+            ⛔️ // VS16
+            "
+        `);
+
+        expect(escapeInvisibleCharacters(value)).toMatchInlineSnapshot(`
+            "
+            // Variation Selectors
+            ⛔\\ufe00 // VS1
+            ⛔\\ufe01 // VS2
+            ⛔\\ufe02 // VS3
+            ⛔\\ufe03 // VS4
+            ⛔\\ufe04 // VS5
+            ⛔\\ufe05 // VS6
+            ⛔\\ufe06 // VS7
+            ⛔\\ufe07 // VS8
+            ⛔\\ufe08 // VS9
+            ⛔\\ufe09 // VS10
+            ⛔\\ufe0a // VS11
+            ⛔\\ufe0b // VS12
+            ⛔\\ufe0c // VS13
+            ⛔\\ufe0d // VS14
+            ⛔\\ufe0e // VS15
+            ⛔\\ufe0f // VS16
+            "
+        `);
+    });
+});