Write interpreter for CQL -> Grid structured query

to avoid a case where the grid string parser was turning dot-separated keys, e.g. leases.leases.access:deny-use, into space-separated string tokens

Author: jonathonherbert

kahuna/public/js/components/gr-cql-input/gr-cql-input.ts

@@ -10,6 +10,7 @@ import {
   QuerySuggestionsService
 } from "../../search/structured-query/query-suggestions";
 import { theme } from "./theme";
+import { cqlParserSettings } from "./syntax";
 
 export const grCqlInput = angular.module("gr.cqlInput", [
   querySuggestions.name
@@ -55,14 +56,7 @@ grCqlInput.directive<
 
     const CqlInput = createCqlInput(typeahead, {
       theme,
-      lang: {
-        operators: false,
-        groups: false,
-        shortcuts: {
-          "#": "label",
-          "~": "collection"
-        }
-      }
+      lang: cqlParserSettings
     });
 
     customElements.define("cql-input", CqlInput);
@@ -119,7 +113,7 @@ grCqlInput.directive<
           "queryChange",
           (event: QueryChangeEventDetail) => {
             if (event.detail?.queryAst) {
-              scope.onChange()(event.detail.queryStr);
+              scope.onChange()(event.detail?.queryAst);
             }
           },
         );

kahuna/public/js/components/gr-cql-input/syntax.spec.ts

@@ -0,0 +1,103 @@
+import { createParser } from "@guardian/cql";
+import { describe, it, expect } from "@jest/globals";
+import { cqlParserSettings, structureCqlQuery } from "./syntax";
+
+const parser = createParser(cqlParserSettings);
+
+const queries = [
+  {
+    name: "plain text",
+    cql: "text",
+    structuredQuery: [{ type: "text", value: "text" }]
+  },
+  {
+    name: "a single chip",
+    cql: "has:chip",
+    structuredQuery: [
+      {
+        filterType: "inclusion",
+        key: "has",
+        type: "filter",
+        value: "chip"
+      }
+    ]
+  },
+  {
+    name: "multiple chips",
+    cql: "has:chip another:chip",
+    structuredQuery: [
+      {
+        filterType: "inclusion",
+        key: "has",
+        type: "filter",
+        value: "chip"
+      },
+      {
+        filterType: "inclusion",
+        key: "another",
+        type: "filter",
+        value: "chip"
+      }
+    ]
+  },
+  {
+    name: "consecutive text",
+    cql: "this should be a single text field",
+    structuredQuery: [
+      { type: "text", value: "this should be a single text field" }
+    ]
+  },
+  {
+    name: "chips and text in combination",
+    cql: "text has:chip another:chip more text",
+    structuredQuery: [
+      { type: "text", value: "text" },
+      {
+        filterType: "inclusion",
+        key: "has",
+        type: "filter",
+        value: "chip"
+      },
+      {
+        filterType: "inclusion",
+        key: "another",
+        type: "filter",
+        value: "chip"
+      },
+      { type: "text", value: "more text" }
+    ]
+  },
+  {
+    name: "chips with unusual keys",
+    cql: "text leases.leases.access:deny-use text usages@platform:print more text",
+    structuredQuery: [
+      { type: "text", value: "text" },
+      {
+        filterType: "inclusion",
+        key: "leases.leases.access",
+        type: "filter",
+        value: "deny-use"
+      },
+      { type: "text", value: "text" },
+      {
+        filterType: "inclusion",
+        key: "usages@platform",
+        type: "filter",
+        value: "print"
+      },
+      { type: "text", value: "more text" }
+    ]
+  }
+];
+
+describe("", () => {
+  queries.forEach((query) => {
+    it(`should parse a query of '${query.name}' into a Grid structured query`, () => {
+      const cqlAst = parser(query.cql).queryAst;
+
+      const structuredQuery = structureCqlQuery(cqlAst);
+
+      expect(structuredQuery).toEqual(query.structuredQuery);
+    });
+  });
+});

kahuna/public/js/components/gr-cql-input/syntax.ts

@@ -0,0 +1,83 @@
+import { CqlBinary, CqlExpr, CqlField, CqlQuery, CqlStr } from "@guardian/cql";
+import { StructuredQuery } from "../../search/structured-query/syntax";
+
+export const cqlParserSettings = {
+  operators: false,
+  groups: false,
+  shortcuts: {
+    "#": "label",
+    "~": "collection"
+  }
+};
+
+export const structureCqlQuery = (query: CqlQuery): StructuredQuery => {
+  if (!query.content) {
+    return [];
+  }
+
+  return combineConsecutiveTextFields(structuredQueryFromBinary(query.content));
+};
+
+const structuredQueryFromExpr = (expr: CqlExpr): StructuredQuery => {
+  switch (expr.content.type) {
+    case "CqlStr":
+      return structuredQueryFromStr(expr.content);
+    case "CqlBinary":
+      return structuredQueryFromBinary(expr.content);
+    case "CqlGroup":
+      return []; // No groups in Grid queries ... yet
+    case "CqlField":
+      return structuredQueryFromField(expr.content);
+  }
+};
+
+const structuredQueryFromBinary = (binary: CqlBinary): StructuredQuery => {
+  const left = structuredQueryFromExpr(binary.left);
+  const right = binary.right
+    ? structuredQueryFromBinary(binary.right.binary)
+    : [];
+
+  return left.concat(right);
+};
+
+const structuredQueryFromStr = (str: CqlStr): StructuredQuery => {
+  return [
+    {
+      type: "text",
+      value: str.searchExpr
+    }
+  ];
+};
+
+const structuredQueryFromField = ({
+  key: { literal, tokenType },
+  value
+}: CqlField): StructuredQuery => {
+  const type = literal === "collection" ? "static-filter" : "filter";
+  const filterType =
+    tokenType === "CHIP_KEY_NEGATIVE" ? "exclusion" : "inclusion";
+
+  return [
+    {
+      type,
+      filterType,
+      key: literal,
+      value: value.literal ?? ""
+    }
+  ];
+};
+
+const combineConsecutiveTextFields = (
+  query: StructuredQuery,
+): StructuredQuery =>
+  query.reduce((previousFields, structuredField) => {
+    const previousField = previousFields.at(-1);
+    if (structuredField.type !== "text" || previousField?.type !== "text") {
+      return previousFields.concat(structuredField);
+    }
+
+    return previousFields.slice(0, previousFields.length - 1).concat({
+      ...structuredField,
+      value: `${previousField.value} ${structuredField.value}`
+    });
+  }, [] as StructuredQuery);

kahuna/public/js/search/structured-query/structured-query.js

@@ -8,7 +8,7 @@ import { grChips } from "../../components/gr-chips/gr-chips";
 import { rxUtil } from "../../util/rx";
 
 import { querySuggestions } from "./query-suggestions";
-import { renderQuery, structureQuery } from "./syntax";
+import { renderQuery, structureCqlQuery, structureQuery } from "./syntax";
 import { getFeatureSwitchActive } from "../../components/gr-feature-switch-panel/gr-feature-switch-panel";
 import { grCqlInput } from "../../components/gr-cql-input/gr-cql-input";
 
@@ -89,9 +89,8 @@ grStructuredQuery.directive("grStructuredQuery", [
 
         if (ctrl.useCql) {
           ctrl.handleChange = (value) => {
-            ctrl.structuredQuery = structureQuery(value);
+            ctrl.structuredQuery = structureCqlQuery(value);
             ctrl.structuredQueryChanged(ctrl.structuredQuery);
-            ctrl.value = value;
           };
           ngModelCtrl.$render = function () {
             ctrl.value = ngModelCtrl.$viewValue || "";

…lic/js/search/structured-query/syntax.js …lic/js/search/structured-query/syntax.tskahuna/public/js/search/structured-query/syntax.js renamed to kahuna/public/js/search/structured-query/syntax.ts kahuna/public/js/search/structured-query/syntax.js renamed to kahuna/public/js/search/structured-query/syntax.ts

@@ -4,19 +4,36 @@ import {getLabel, getCollection} from '../../search-query/query-syntax';
 // Line too long for eslint, but can't break it down..
 /*eslint-disable max-len */
 const parserRe = /(-?)(?:(?:([\p{L}@><]+):|"([^"]+)":|'([^']+)':|(#)|(~))(?:([^ "']+)|"([^"]+)"|'([^']+)')|([\p{L}0-9:]+)|"([^"]*)"|'([^']*)')/gu;
+
 /*eslint-enable max-len */
-const falsyValuesToEmptyString = (value) => {
+const falsyValuesToEmptyString = (value: string | null | undefined) => {
     if (!value){
         return '';
     } else {
         return value.toString();
     }
 };
+
+type StructuredQueryFilter = {
+  type: 'filter' | 'static-filter' | 'collection'
+  filterType?: 'inclusion' | 'exclusion',
+  key: string,
+  value: string
+}
+
+type StructuredQueryText = {
+  type: 'text'
+  filterType?: 'inclusion' | 'exclusion',
+  value: string
+}
+
+export type StructuredQuery = (StructuredQueryFilter | StructuredQueryText)[];
+
 // TODO: expose the server-side query parser via an API instead of
 // replicating it poorly here
-export function structureQuery(query) {
+export function structureQuery(query: string) {
 
-    const struct = [];
+    const struct: StructuredQuery = [];
     let m;
     if (query === undefined) {
         return struct;
@@ -54,7 +71,7 @@ export function structureQuery(query) {
     return orderChips(struct);
 }
 
-function orderChips(structuredQuery){
+function orderChips(structuredQuery: StructuredQuery){
     const filterChips = structuredQuery.filter(e => e.type !== 'text');
     const textChipsValues = mergeTextValues(structuredQuery);
     const textChip = {
@@ -64,7 +81,7 @@ function orderChips(structuredQuery){
 
     const cleanStruct = [textChip].concat(filterChips);
 
-    function mergeTextValues(chips){
+    function mergeTextValues(chips: StructuredQuery){
         return chips.filter(e => e.type === 'text')
             .map(e => e.value)
             .join(' ');
@@ -73,7 +90,7 @@ function orderChips(structuredQuery){
     return cleanStruct;
 }
 
-function renderFilter(field, value) {
+function renderFilter(field: string, value: string) {
     switch (field) {
     case 'label':      return getLabel(value);
     case 'collection': return getCollection(value);
@@ -83,7 +100,7 @@ function renderFilter(field, value) {
 }
 
 // Serialise a structured query into a plain string query
-export function renderQuery(structuredQuery) {
+export function renderQuery(structuredQuery: StructuredQuery) {
     return structuredQuery.filter(item => item.value).map(item => {
         switch (item.type) {
         // Match both filters

kahuna/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "noImplicitAny": true,
     "module": "es6",
-    "target": "es5",
+    "target": "es6",
     "lib": ["es6", "dom"],
     "jsx": "react",
     "alwaysStrict": true,