diff --git a/bun.lock b/bun.lock index 9e34fcdfb..b5ba19fc8 100644 --- a/bun.lock +++ b/bun.lock @@ -5,6 +5,7 @@ "": { "name": "gitlab-ci-local", "dependencies": { + "@jsep-plugin/regex": "^1.0.4", "ajv": "8.x.x", "axios": "1.x.x", "base64url": "3.x.x", @@ -18,6 +19,7 @@ "fs-extra": "11.x.x", "globby": "16.x.x", "js-yaml": "4.x.x", + "jsep": "^1.4.0", "jsonpointer": "5.x.x", "micromatch": "4.x.x", "object-traversal": "1.x.x", @@ -114,6 +116,8 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@jsep-plugin/regex": ["@jsep-plugin/regex@1.0.4", "", { "peerDependencies": { "jsep": "^0.4.0||^1.0.0" } }, "sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg=="], + "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -506,6 +510,8 @@ "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + "jsep": ["jsep@1.4.0", "", {}, "sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw=="], + "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], diff --git a/package.json b/package.json index 481bbb707..7d379ac4f 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "fetch-schema": "curl https://gitlab.com/gitlab-org/gitlab/-/raw/master/app/assets/javascripts/editor/schema/ci.json -sf > src/schema.json" }, "dependencies": { + "@jsep-plugin/regex": "^1.0.4", "ajv": "8.x.x", "axios": "1.x.x", "base64url": "3.x.x", @@ -37,6 +38,7 @@ "fs-extra": "11.x.x", "globby": "16.x.x", "js-yaml": "4.x.x", + "jsep": "^1.4.0", "jsonpointer": "5.x.x", "micromatch": "4.x.x", "object-traversal": "1.x.x", diff --git a/src/utils.ts b/src/utils.ts index 0297853ff..fef79f2f0 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,8 @@ import "./global.js"; import {RE2JS} from "re2js"; import chalk from "chalk-template"; +import jsep from "jsep"; +import jsepRegex from "@jsep-plugin/regex"; import {Job, JobRule, Need, Service} from "./job.js"; import {needsComplex} from "./data-expander.js"; import fs from "fs-extra"; @@ -16,6 +18,10 @@ import {AxiosRequestConfig} from "axios"; import path from "node:path"; import {Argv} from "./argv.js"; +jsep.plugins.register(jsepRegex); // /pattern/flags literals +jsep.addBinaryOp("=~", 10); // regex match operator +jsep.addBinaryOp("!~", 10); // regex non-match operator + type RuleResultOpt = { argv: Argv; cwd: string; @@ -213,10 +219,39 @@ export class Utils { return {when, allowFailure, variables: ruleVariable, needs: ruleNeeds}; } + // Reconstruct the source string for an atomic (non-logical) jsep node. + // Identifiers keep their name ($VAR), literals use their raw form so that + // strings retain quotes and regexes retain slashes and flags. + private static _nodeToAtom (node: jsep.Expression): string { + switch (node.type) { + case "Identifier": return (node as jsep.Identifier).name; + case "Literal": return (node as jsep.Literal).raw; + case "BinaryExpression": { + const n = node as jsep.BinaryExpression; + return `${Utils._nodeToAtom(n.left)} ${n.operator} ${Utils._nodeToAtom(n.right)}`; + } + default: throw new Error(`Unsupported expression node type: ${(node as any).type}`); + } + } + + static stripQuotes (str: string) { + if (str.length < 2) return str; + const first = str[0]; + const last = str[str.length - 1]; + if ((first === "\"" && last === "\"") || (first === "'" && last === "'")) { + return str.slice(1, -1); + } + return str; + } + static evaluateRuleIf (ruleIf: string | undefined, envs: {[key: string]: string}): boolean { if (ruleIf === undefined) return true; assert(!/\$\{\w+\}/.test(ruleIf), chalk`rules:rule if invalid expression syntax: {blueBright ${ruleIf}}\nuse {green $VAR} not {red \${VAR\}} in rules:if`); let evalStr = ruleIf; + evalStr = this.expandTextWith(evalStr, { + unescape: JSON.stringify("$"), + variable: (name) => JSON.stringify(envs[name] ?? null).replaceAll("\\\\", "\\"), + }); // replace all $VAR by their values const flagsToBinary = (flags: string): number => { let binary = 0; @@ -231,87 +266,89 @@ export class Utils { } return binary; }; - - // Expand all variables - evalStr = this.expandTextWith(evalStr, { - unescape: JSON.stringify("$"), - variable: (name) => JSON.stringify(envs[name] ?? null).replaceAll("\\\\", "\\"), - }); - const expandedEvalStr = evalStr; - - // Scenario when RHS is a - // https://regexr.com/85sjo - const pattern1 = /\s*(?(?:=~)|(?:!~))\s*\/(?.*?[^\\])\/(?[igmsuy]*)(\s|$|\))/g; - evalStr = evalStr.replaceAll(pattern1, (_, operator, rhs, flags, remainingTokens) => { - let _operator; - switch (operator) { - case "=~": - _operator = "!="; - break; - case "!~": - _operator = "=="; - break; - default: - throw operator; + // jsep parses ruleIf into an AST, handling &&, ||, () and operator precedence. + const walk = (node: jsep.Expression): boolean => { + if (node.type === "BinaryExpression") { + const n = node as jsep.BinaryExpression; + if (n.operator === "&&") return walk(n.left) && walk(n.right); + if (n.operator === "||") return walk(n.left) || walk(n.right); + if (n.operator === "=~" || n.operator === "!~") { + assert(n.left.type === "Literal", `Not a Literal: ${JSON.stringify(n.left)}`); + assert(n.right.type === "Literal", `Not a Literal: ${JSON.stringify(n.right)}`); + const leftStr = n.left as jsep.Literal; + const rightStr = n.right as jsep.Literal; + if (leftStr.value === null) + return n.operator === "!~"; // null =~ /p/ → false; null !~ /p/ → true + if (rightStr.value === null) + return false; + let regexStr: string = rightStr.raw; + regexStr = this.stripQuotes(regexStr); + + const regex = /\/(?.*)\/(?[igmsuy]*)/; + const _rhs = regexStr.replace(regex, (_: string, pattern: string, flags: string) => { + const flagsBinary = flagsToBinary(flags); + return `RE2JS.compile(${JSON.stringify(pattern)}, ${flagsBinary})`; + }); + + const assertMsg = [ + "RHS (${rhs}) must be a regex pattern. Do not rely on this behavior!", + "Refer to https://docs.gitlab.com/ee/ci/jobs/job_rules.html#unexpected-behavior-from-regular-expression-matching-with- for more info...", + ]; + assert(_rhs !== regexStr, assertMsg.join("\n")); + + const _operator = n.operator === "=~" ? "!=" : "=="; // =~ -> !=; !~ -> == + + const evalStr = `${leftStr.raw}.matchRE2JS(${_rhs}) ${_operator} null`; + + let res; + try { + (globalThis as any).RE2JS = RE2JS; + res = (0, eval)(evalStr); // indirect eval + delete (globalThis as any).RE2JS; + } catch (error) { + console.error(error); + const assertMsg = [ + "Error attempting to evaluate the following rules:", + " rules:", + ` - if: '${Utils._nodeToAtom(node)}'`, + "as", + "```javascript", + `${evalStr}`, + "```", + ]; + assert(false, assertMsg.join("\n")); + } + return Boolean(res); + } } - const _rhs = JSON.stringify(rhs); // JSON.stringify for escaping `"` - const containsNonEscapedSlash = /(?=~|!~)\s*(["'])(?(?:\\.|[^\\])*?)\2/g; - evalStr = evalStr.replaceAll(pattern2, (_, operator, __, rhs) => { - let _operator; - switch (operator) { - case "=~": - _operator = "!="; - break; - case "!~": - _operator = "=="; - break; - default: - throw operator; + const atom = Utils._nodeToAtom(node); + let res; + try { + (globalThis as any).RE2JS = RE2JS; + res = (0, eval)(atom); + delete (globalThis as any).RE2JS; + } catch { + assert(false, [ + "Error attempting to evaluate the following rules:", + " rules:", + ` - if: '${ruleIf}'`, + "as", + "```javascript", + `${atom}`, + "```", + ].join("\n")); } + return Boolean(res); + }; - const assertMsg = [ - "RHS (${rhs}) must be a regex pattern. Do not rely on this behavior!", - "Refer to https://docs.gitlab.com/ee/ci/jobs/job_rules.html#unexpected-behavior-from-regular-expression-matching-with- for more info...", - ]; - assert((/\/(.*)\/(\w*)/.test(rhs)), assertMsg.join("\n")); - - const regex = /\/(?.*)\/(?[igmsuy]*)/; - const _rhs = rhs.replace(regex, (_: string, pattern: string, flags: string) => { - const flagsBinary = flagsToBinary(flags); - return `RE2JS.compile("${pattern}", ${flagsBinary})`; - }); - return `.matchRE2JS(${_rhs}) ${_operator} null`; - }); - - evalStr = evalStr.replaceAll(/null.matchRE2JS\(.+?\)\s*!=\s*null/g, "false"); - evalStr = evalStr.replaceAll(/null.matchRE2JS\(.+?\)\s*==\s*null/g, "true"); - - evalStr = evalStr.trim(); - - let res; + let ast; try { - (globalThis as any).RE2JS = RE2JS; - res = (0, eval)(evalStr); // indirect eval - delete (globalThis as any).RE2JS; + ast = jsep(evalStr); } catch { const assertMsg = [ "Error attempting to evaluate the following rules:", " rules:", - ` - if: '${expandedEvalStr}'`, + ` - if: '${ruleIf}'`, "as", "```javascript", `${evalStr}`, @@ -319,9 +356,10 @@ export class Utils { ]; assert(false, assertMsg.join("\n")); } - return Boolean(res); + return walk(ast!); } + static evaluateRuleExist (cwd: string, ruleExists: string[] | {paths: string[]} | undefined): boolean { if (ruleExists === undefined) return true; diff --git a/tests/rules-regex.test.ts b/tests/rules-regex.test.ts index 34c551443..abac00f8a 100644 --- a/tests/rules-regex.test.ts +++ b/tests/rules-regex.test.ts @@ -18,108 +18,27 @@ beforeEach(async () => { /* eslint-disable @stylistic/quotes */ const tests = [ - { - rule: '"Hello World" =~ "/hello world/i"', - jsExpression: '"Hello World".matchRE2JS(RE2JS.compile("hello world", 1)) != null', - evalResult: true, - }, - { - rule: '"Hello World" =~ /hello world/i', - jsExpression: '"Hello World".matchRE2JS(RE2JS.compile("hello world", 1)) != null', - evalResult: true, - }, - { - rule: '"Hello World" =~ /Hello (?i)world/', - jsExpression: '"Hello World".matchRE2JS(RE2JS.compile("Hello (?i)world", 0)) != null', - evalResult: true, - }, - { - rule: '"1.11" =~ /^([[:digit:]]+(.[[:digit:]]+)*|latest)$/', - jsExpression: '"1.11".matchRE2JS(RE2JS.compile("^([[:digit:]]+(.[[:digit:]]+)*|latest)$", 0)) != null', - evalResult: true, - }, - { - rule: '"foo" !~ /foo/', - jsExpression: '"foo".matchRE2JS(RE2JS.compile("foo", 0)) == null', - evalResult: false, - }, - { - rule: '"foo" =~ /foo/', - jsExpression: '"foo".matchRE2JS(RE2JS.compile("foo", 0)) != null', - evalResult: true, - }, - { - rule: '"foo"=~ /foo/', - jsExpression: '"foo".matchRE2JS(RE2JS.compile("foo", 0)) != null', - evalResult: true, - }, - { - rule: '"foo"=~/foo/', - jsExpression: '"foo".matchRE2JS(RE2JS.compile("foo", 0)) != null', - evalResult: true, - }, - { - rule: '"foo"=~ /foo/', - jsExpression: '"foo".matchRE2JS(RE2JS.compile("foo", 0)) != null', - evalResult: true, - }, - { - rule: '"foo" =~ "/foo/"', - jsExpression: '"foo".matchRE2JS(RE2JS.compile("foo", 0)) != null', - evalResult: true, - }, - { - rule: '"test/url" =~ "/test/ur/"', - jsExpression: '"test/url".matchRE2JS(RE2JS.compile("test/ur", 0)) != null', - evalResult: true, - }, - { - rule: '"test/url" =~ "/test\\/ur/"', - jsExpression: '"test/url".matchRE2JS(RE2JS.compile("test\\/ur", 0)) != null', - evalResult: true, - }, - { - rule: '"test/url" =~ /test/ur/', - expectedErrSubStr: "Error attempting to evaluate the following rules:", - }, - { - rule: '"master" =~ /master$/', - jsExpression: '"master".matchRE2JS(RE2JS.compile("master$", 0)) != null', - evalResult: true, - }, - { - rule: '"23" =~ "1234"', - expectedErrSubStr: "must be a regex pattern. Do not rely on this behavior!", - }, - { - rule: '"23" =~ \'1234\'', - expectedErrSubStr: "must be a regex pattern. Do not rely on this behavior!", - }, - { - rule: '"23" =~ /1234/', - jsExpression: '"23".matchRE2JS(RE2JS.compile("1234", 0)) != null', - evalResult: false, - }, - { - rule: '$CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $CI_GITLAB_FIPS_MODE == "true"', - jsExpression: 'null && false && null == "true"', - evalResult: false, - }, - { - rule: '($CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^perf_.*$/)', - jsExpression: '(false)', // (null.matchRE2JS(RE2JS.compile("^perf_.*$", 0)) != null => (false) - evalResult: false, - }, - { - rule: '("qwerty" =~ /^perf_.*$/)', - jsExpression: '("qwerty".matchRE2JS(RE2JS.compile("^perf_.*$", 0)) != null)', - evalResult: false, - }, - { - rule: `"product-name/v0.0.0+build.0" =~ /^(?:product-name\\/)?v\\d+\\.\\d+\\.\\d+.*/`, - jsExpression: "\"product-name/v0.0.0+build.0\".matchRE2JS(RE2JS.compile(\"^(?:product-name\\\\/)?v\\\\d+\\\\.\\\\d+\\\\.\\\\d+.*\", 0)) != null", - evalResult: true, - }, + {rule: '"Hello World" =~ "/hello world/i"', evalResult: true}, + {rule: '"Hello World" =~ /hello world/i', evalResult: true}, + {rule: '"Hello World" =~ /Hello (?i)world/', expectedErrSubStr: "Error attempting to evaluate the following rules:"}, + {rule: '"1.11" =~ /^([[:digit:]]+(.[[:digit:]]+)*|latest)$/', evalResult: true}, + {rule: '"foo" !~ /foo/', evalResult: false}, + {rule: '"foo" =~ /foo/', evalResult: true}, + {rule: '"foo"=~ /foo/', evalResult: true}, + {rule: '"foo"=~/foo/', evalResult: true}, + {rule: '"foo"=~ /foo/', evalResult: true}, + {rule: '"foo" =~ "/foo/"', evalResult: true}, + {rule: '"test/url" =~ "/test/ur/"', evalResult: true}, + {rule: '"test/url" =~ "/test\\/ur/"', evalResult: true}, + {rule: '"test/url" =~ /test/ur/', expectedErrSubStr: "Error attempting to evaluate the following rules:"}, + {rule: '"master" =~ /master$/', evalResult: true}, + {rule: '"23" =~ "1234"', expectedErrSubStr: "must be a regex pattern. Do not rely on this behavior!"}, + {rule: '"23" =~ \'1234\'', expectedErrSubStr: "must be a regex pattern. Do not rely on this behavior!"}, + {rule: '"23" =~ /1234/', evalResult: false}, + {rule: '$CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $CI_GITLAB_FIPS_MODE == "true"', evalResult: false}, + {rule: '($CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^perf_.*$/)', evalResult: false}, + {rule: '("qwerty" =~ /^perf_.*$/)', evalResult: false}, + {rule: `"product-name/v0.0.0+build.0" =~ /^(?:product-name\\/)?v\\d+\\.\\d+\\.\\d+.*/`, evalResult: true}, ]; /* eslint-enable @stylistic/quotes */ @@ -128,12 +47,10 @@ describe("gitlab rules regex", () => { .forEach((t) => { test(`- if: '${t.rule}'\n\t => ${t.evalResult}`, async () => { const rules = [ {if: t.rule} ]; - const evalSpy = vi.spyOn(global, "eval"); const evaluateRuleIfSpy = vi.spyOn(Utils, "evaluateRuleIf"); Utils.getRulesResult({argv, cwd: "", rules, variables: {}}, gitData); expect(evaluateRuleIfSpy).toHaveReturnedWith(t.evalResult); - expect(evalSpy).toHaveBeenCalledWith(t.jsExpression); }); }); });