Merge branch 'master' into fix_print_with_precedence

Author: antonmedv

.gitattributes

@@ -0,0 +1 @@
+*\[generated\].go linguist-language=txt

.github/scripts/coverage.mjs

@@ -2,13 +2,15 @@
 
 const expected = 90
 const exclude = [
-  'expr/test',
-  'checker/mock',
-  'vm/func_types',
-  'vm/runtime/helpers',
-  'internal/difflib',
-  'internal/spew',
-  'internal/testify',
+  'expr/test', // We do not need to test the test package.
+  'checker/mock', // Mocks only used for testing.
+  'vm/func_types', // Generated files.
+  'vm/runtime/helpers', // Generated files.
+  'internal/difflib', // Test dependency. This is vendored dependency, and ideally we also have good tests for it.
+  'internal/spew', // Test dependency.
+  'internal/testify', // Test dependency.
+  'patcher/value', // Contains a lot of repeating code. Ideally we should have a test for it.
+  'pro', // Expr Pro is not a part of the main codebase.
 ]
 
 cd(path.resolve(__dirname, '..', '..'))

.github/workflows/build.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-versions: [ '1.18', '1.22' ]
+        go-versions: [ '1.18', '1.22', '1.24' ]
         go-arch: [ '386' ]
     steps:
       - uses: actions/checkout@v3

.github/workflows/diff.yml

@@ -13,7 +13,9 @@ jobs:
         with:
           go-version: 1.18
       - name: Install benchstat
-        run: go install golang.org/x/perf/cmd/benchstat@latest
+        # NOTE: benchstat@latest requires go 1.23 since 2025-02-14 - this is the last go 1.18 ref
+        # https://cs.opensource.google/go/x/perf/+/c95ad7d5b636f67d322a7e4832e83103d0fdd292
+        run: go install golang.org/x/perf/cmd/benchstat@884df5810d2850d775c2cb4885a7ea339128a17d
 
       - uses: actions/checkout@v3
       - name: Benchmark new code

.github/workflows/fuzz.yml

@@ -21,14 +21,14 @@ jobs:
           fuzz-seconds: 600
           output-sarif: true
       - name: Upload Crash
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure() && steps.build.outcome == 'success'
         with:
           name: artifacts
           path: ./out/artifacts
       - name: Upload Sarif
         if: always() && steps.build.outcome == 'success'
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           # Path to SARIF file relative to the root of the repository
           sarif_file: cifuzz-sarif/results.sarif

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-versions: [ '1.18', '1.19', '1.20', '1.21', '1.22' ]
+        go-versions: [ '1.18', '1.19', '1.20', '1.21', '1.22', '1.23', '1.24' ]
     steps:
       - uses: actions/checkout@v3
       - name: Setup Go ${{ matrix.go-version }}

README.md

@@ -162,9 +162,14 @@ func main() {
 * [Visually.io](https://visually.io) employs Expr as a business rule engine for its personalization targeting algorithm.
 * [Akvorado](https://github.com/akvorado/akvorado) utilizes Expr to classify exporters and interfaces in network flows.
 * [keda.sh](https://keda.sh) uses Expr to allow customization of its Kubernetes-based event-driven autoscaling.
-* [Span Digital](https://spandigital.com/) uses Expr in it's Knowledge Management products.
+* [Span Digital](https://spandigital.com/) uses Expr in its Knowledge Management products.
 * [Xiaohongshu](https://www.xiaohongshu.com/) combining yaml with Expr for dynamically policies delivery.
 * [Melrōse](https://melrōse.org) uses Expr to implement its music programming language.
+* [Tork](https://www.tork.run/) integrates Expr into its workflow execution.
+* [Critical Moments](https://criticalmoments.io) uses Expr for its mobile realtime conditional targeting system.
+* [WoodpeckerCI](https://woodpecker-ci.org) uses Expr for [filtering workflows/steps](https://woodpecker-ci.org/docs/usage/workflow-syntax#evaluate).
+* [FastSchema](https://github.com/fastschema/fastschema) - A BaaS leveraging Expr for its customizable and dynamic Access Control system.
+* [WunderGraph Cosmo](https://github.com/wundergraph/cosmo) - GraphQL Federeration Router uses Expr to customize Middleware behaviour
 
 [Add your company too](https://github.com/expr-lang/expr/edit/master/README.md)
 

builtin/builtin.go

@@ -830,6 +830,57 @@ var Builtins = []*Function{
 			}
 		},
 	},
+
+	{
+		Name: "uniq",
+		Func: func(args ...any) (any, error) {
+			if len(args) != 1 {
+				return nil, fmt.Errorf("invalid number of arguments (expected 1, got %d)", len(args))
+			}
+
+			v := reflect.ValueOf(deref.Deref(args[0]))
+			if v.Kind() != reflect.Array && v.Kind() != reflect.Slice {
+				return nil, fmt.Errorf("cannot uniq %s", v.Kind())
+			}
+
+			size := v.Len()
+			ret := []any{}
+
+			eq := func(i int) bool {
+				for _, r := range ret {
+					if runtime.Equal(v.Index(i).Interface(), r) {
+						return true
+					}
+				}
+
+				return false
+			}
+
+			for i := 0; i < size; i += 1 {
+				if eq(i) {
+					continue
+				}
+
+				ret = append(ret, v.Index(i).Interface())
+			}
+
+			return ret, nil
+		},
+
+		Validate: func(args []reflect.Type) (reflect.Type, error) {
+			if len(args) != 1 {
+				return anyType, fmt.Errorf("invalid number of arguments (expected 1, got %d)", len(args))
+			}
+
+			switch kind(args[0]) {
+			case reflect.Interface, reflect.Slice, reflect.Array:
+				return arrayType, nil
+			default:
+				return anyType, fmt.Errorf("cannot uniq %s", args[0])
+			}
+		},
+	},
+
 	{
 		Name: "concat",
 		Safe: func(args ...any) (any, uint, error) {
@@ -873,6 +924,37 @@ var Builtins = []*Function{
 			return arrayType, nil
 		},
 	},
+	{
+		Name: "flatten",
+		Safe: func(args ...any) (any, uint, error) {
+			var size uint
+			if len(args) != 1 {
+				return nil, 0, fmt.Errorf("invalid number of arguments (expected 1, got %d)", len(args))
+			}
+			v := reflect.ValueOf(deref.Deref(args[0]))
+			if v.Kind() != reflect.Array && v.Kind() != reflect.Slice {
+				return nil, size, fmt.Errorf("cannot flatten %s", v.Kind())
+			}
+			ret := flatten(v)
+			size = uint(len(ret))
+			return ret, size, nil
+		},
+		Validate: func(args []reflect.Type) (reflect.Type, error) {
+			if len(args) != 1 {
+				return anyType, fmt.Errorf("invalid number of arguments (expected 1, got %d)", len(args))
+			}
+
+			for _, arg := range args {
+				switch kind(deref.Type(arg)) {
+				case reflect.Interface, reflect.Slice, reflect.Array:
+				default:
+					return anyType, fmt.Errorf("cannot flatten %s", arg)
+				}
+			}
+
+			return arrayType, nil
+		},
+	},
 	{
 		Name: "sort",
 		Safe: func(args ...any) (any, uint, error) {

builtin/builtin_test.go

@@ -152,6 +152,11 @@ func TestBuiltin(t *testing.T) {
 		{`reduce([], 5, 0)`, 0},
 		{`concat(ArrayOfString, ArrayOfInt)`, []any{"foo", "bar", "baz", 1, 2, 3}},
 		{`concat(PtrArrayWithNil, [nil])`, []any{42, nil}},
+		{`flatten([["a", "b"], [1, 2]])`, []any{"a", "b", 1, 2}},
+		{`flatten([["a", "b"], [1, 2, [3, 4]]])`, []any{"a", "b", 1, 2, 3, 4}},
+		{`flatten([["a", "b"], [1, 2, [3, [[[["c", "d"], "e"]]], 4]]])`, []any{"a", "b", 1, 2, 3, "c", "d", "e", 4}},
+		{`uniq([1, 15, "a", 2, 3, 5, 2, "a", 2, "b"])`, []any{1, 15, "a", 2, 3, 5, "b"}},
+		{`uniq([[1, 2], "a", 2, 3, [1, 2], [1, 3]])`, []any{[]any{1, 2}, "a", 2, 3, []any{1, 3}}},
 	}
 
 	for _, test := range tests {
@@ -236,6 +241,8 @@ func TestBuiltin_errors(t *testing.T) {
 		{`now(nil)`, "invalid number of arguments (expected 0, got 1)"},
 		{`date(nil)`, "interface {} is nil, not string (1:1)"},
 		{`timezone(nil)`, "cannot use nil as argument (type string) to call timezone (1:10)"},
+		{`flatten([1, 2], [3, 4])`, "invalid number of arguments (expected 1, got 2)"},
+		{`flatten(1)`, "cannot flatten int"},
 	}
 	for _, test := range errorTests {
 		t.Run(test.input, func(t *testing.T) {

builtin/lib.go

@@ -359,3 +359,17 @@ func median(args ...any) ([]float64, error) {
 	}
 	return values, nil
 }
+
+func flatten(arg reflect.Value) []any {
+	ret := []any{}
+	for i := 0; i < arg.Len(); i++ {
+		v := deref.Value(arg.Index(i))
+		if v.Kind() == reflect.Array || v.Kind() == reflect.Slice {
+			x := flatten(v)
+			ret = append(ret, x...)
+		} else {
+			ret = append(ret, v.Interface())
+		}
+	}
+	return ret
+}

checker/checker.go

@@ -13,6 +13,40 @@ import (
 	"github.com/expr-lang/expr/parser"
 )
 
+// Run visitors in a given config over the given tree
+// runRepeatable controls whether to filter for only vistors that require multiple passes or not
+func runVisitors(tree *parser.Tree, config *conf.Config, runRepeatable bool) {
+	for {
+		more := false
+		for _, v := range config.Visitors {
+			// We need to perform types check, because some visitors may rely on
+			// types information available in the tree.
+			_, _ = Check(tree, config)
+
+			r, repeatable := v.(interface {
+				Reset()
+				ShouldRepeat() bool
+			})
+
+			if repeatable {
+				if runRepeatable {
+					r.Reset()
+					ast.Walk(&tree.Node, v)
+					more = more || r.ShouldRepeat()
+				}
+			} else {
+				if !runRepeatable {
+					ast.Walk(&tree.Node, v)
+				}
+			}
+		}
+
+		if !more {
+			break
+		}
+	}
+}
+
 // ParseCheck parses input expression and checks its types. Also, it applies
 // all provided patchers. In case of error, it returns error with a tree.
 func ParseCheck(input string, config *conf.Config) (*parser.Tree, error) {
@@ -22,25 +56,11 @@ func ParseCheck(input string, config *conf.Config) (*parser.Tree, error) {
 	}
 
 	if len(config.Visitors) > 0 {
-		for i := 0; i < 1000; i++ {
-			more := false
-			for _, v := range config.Visitors {
-				// We need to perform types check, because some visitors may rely on
-				// types information available in the tree.
-				_, _ = Check(tree, config)
-
-				ast.Walk(&tree.Node, v)
-
-				if v, ok := v.(interface {
-					ShouldRepeat() bool
-				}); ok {
-					more = more || v.ShouldRepeat()
-				}
-			}
-			if !more {
-				break
-			}
-		}
+		// Run all patchers that dont support being run repeatedly first
+		runVisitors(tree, config, false)
+
+		// Run patchers that require multiple passes next (currently only Operator patching)
+		runVisitors(tree, config, true)
 	}
 	_, err = Check(tree, config)
 	if err != nil {

checker/checker_test.go

@@ -1072,14 +1072,16 @@ func TestCheck_builtin_without_call(t *testing.T) {
 
 func TestCheck_types(t *testing.T) {
 	env := types.Map{
-		"foo": types.StrictMap{
+		"foo": types.Map{
 			"bar": types.Map{
-				"baz": types.String,
+				"baz":       types.String,
+				types.Extra: types.String,
 			},
 		},
-		"arr": types.Array(types.StrictMap{
+		"arr": types.Array(types.Map{
 			"value": types.String,
 		}),
+		types.Extra: types.Any,
 	}
 
 	noerr := "no error"
@@ -1088,9 +1090,11 @@ func TestCheck_types(t *testing.T) {
 		err  string
 	}{
 		{`unknown`, noerr},
+		{`[unknown + 42, another_unknown + "foo"]`, noerr},
 		{`foo.bar.baz > 0`, `invalid operation: > (mismatched types string and int)`},
 		{`foo.unknown.baz`, `unknown field unknown (1:5)`},
 		{`foo.bar.unknown`, noerr},
+		{`foo.bar.unknown + 42`, `invalid operation: + (mismatched types string and int)`},
 		{`[foo] | map(.unknown)`, `unknown field unknown`},
 		{`[foo] | map(.bar) | filter(.baz)`, `predicate should return boolean (got string)`},
 		{`arr | filter(.value > 0)`, `invalid operation: > (mismatched types string and int)`},
@@ -1102,7 +1106,8 @@ func TestCheck_types(t *testing.T) {
 			tree, err := parser.Parse(test.code)
 			require.NoError(t, err)
 
-			_, err = checker.Check(tree, conf.New(env))
+			config := conf.New(env)
+			_, err = checker.Check(tree, config)
 			if test.err == noerr {
 				require.NoError(t, err)
 			} else {

checker/nature/nature.go

@@ -12,16 +12,17 @@ var (
 )
 
 type Nature struct {
-	Type         reflect.Type      // Type of the value. If nil, then value is unknown.
-	Func         *builtin.Function // Used to pass function type from callee to CallNode.
-	ArrayOf      *Nature           // Elem nature of array type (usually Type is []any, but ArrayOf can be any nature).
-	PredicateOut *Nature           // Out nature of predicate.
-	Fields       map[string]Nature // Fields of map type.
-	Strict       bool              // If map is types.StrictMap.
-	Nil          bool              // If value is nil.
-	Method       bool              // If value retrieved from method. Usually used to determine amount of in arguments.
-	MethodIndex  int               // Index of method in type.
-	FieldIndex   []int             // Index of field in type.
+	Type            reflect.Type      // Type of the value. If nil, then value is unknown.
+	Func            *builtin.Function // Used to pass function type from callee to CallNode.
+	ArrayOf         *Nature           // Elem nature of array type (usually Type is []any, but ArrayOf can be any nature).
+	PredicateOut    *Nature           // Out nature of predicate.
+	Fields          map[string]Nature // Fields of map type.
+	DefaultMapValue *Nature           // Default value of map type.
+	Strict          bool              // If map is types.StrictMap.
+	Nil             bool              // If value is nil.
+	Method          bool              // If value retrieved from method. Usually used to determine amount of in arguments.
+	MethodIndex     int               // Index of method in type.
+	FieldIndex      []int             // Index of field in type.
 }
 
 func (n Nature) String() string {
@@ -54,7 +55,12 @@ func (n Nature) Key() Nature {
 
 func (n Nature) Elem() Nature {
 	switch n.Kind() {
-	case reflect.Map, reflect.Ptr:
+	case reflect.Ptr:
+		return Nature{Type: n.Type.Elem()}
+	case reflect.Map:
+		if n.DefaultMapValue != nil {
+			return *n.DefaultMapValue
+		}
 		return Nature{Type: n.Type.Elem()}
 	case reflect.Array, reflect.Slice:
 		if n.ArrayOf != nil {