⭐ add: having(), to chain non-empty sets

tldr: Introduce `having` on lists/maps, which acts like a `where` filter that also checks that the resulting list is not empty.

```coffee
my.collection.having( some condition ).all( ... )
```

**Problem**

As explained by @LittleSalkin1806 in #6633 (comment)

> If the bucket has no tags at all, the where clause will return an empty list, the values will take values from no existing key (empty list) and the all function will evaluate to true because there are no elements that violate the condition. This can lead to false positives in the compliance checks.

I ran into this same problem way too often myself! I've been working on a function to address this problem.

The `where` clause behaves like a traiditional `filter` in JS/TS and is close to SQL, so we wanted to keep their behavior/meaning. However, there is clearly a need for the combination, because this pattern happens way too often:

```coffee
collection.where( condition ) != empty
collection.where( condition ).otherCalls...
```

Using @LittleSalkin1806 's example:

```coffee
aws.s3.bucket.tags.where(key.downcase == 'dtit:sec:infosecclass') != empty
aws.s3.bucket.tags.where(key.downcase == 'dtit:sec:infosecclass').values.map(downcase).in([...])
```

**Solution**

This PR introduces the `having` keyword. It acts like `where` that also makes sure that the list is not empty:

```coffee
collection.having( condition ).otherCalls...
```

or with the above example:

```coffee
aws.s3.bucket.tags.having(key.downcase == 'dtit:sec:infosecclass').values.map(downcase).in([ ... ])
```

If the filtered list is empty, the query fails. If it is non-empty, it
uses the filtered values and looks at the next condition.

Signed-off-by: Dominik Richter <dominik.richter@gmail.com>

Author: arlimus

mqlc/builtin.go

@@ -166,8 +166,9 @@ func init() {
 				compile: compileDictNone, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}},
 				desc: "Check if no entry in this array or map satisfies a given condition",
 			},
-			"map":  {compile: compileArrayMap, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
-			"flat": {compile: compileDictFlat, signature: FunctionSignature{}},
+			"having": {compile: compileDictHaving, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
+			"map":    {compile: compileArrayMap, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
+			"flat":   {compile: compileDictFlat, signature: FunctionSignature{}},
 			// map-ish
 			"keys":   {typ: stringArrayType, signature: FunctionSignature{}},
 			"values": {typ: dictArrayType, signature: FunctionSignature{}},
@@ -208,6 +209,7 @@ func init() {
 			"any":          {compile: compileArrayAny, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"one":          {compile: compileArrayOne, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"none":         {compile: compileArrayNone, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
+			"having":       {compile: compileArrayHaving, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"map":          {compile: compileArrayMap, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"flat":         {compile: compileArrayFlat, signature: FunctionSignature{}},
 			"reverse":      {typHandler: &sameType, signature: FunctionSignature{}},
@@ -225,6 +227,7 @@ func init() {
 			"all":      {compile: compileMapAll, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"one":      {compile: compileMapOne, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"none":     {compile: compileMapNone, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
+			"having":   {compile: compileMapHaving, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 		},
 		types.ResourceLike: {
 			"first":                    {compile: compileResourceChildAccess, signature: FunctionSignature{}},
@@ -237,6 +240,7 @@ func init() {
 			"any":                      {compile: compileResourceAny, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"one":                      {compile: compileResourceOne, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"none":                     {compile: compileResourceNone, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
+			"having":                   {compile: compileResourceHaving, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"map":                      {compile: compileResourceMap, signature: FunctionSignature{Required: 1, Args: []types.Type{types.FunctionLike}}},
 			"==" + string(types.Empty): {compile: compileResourceCmpEmpty},
 			"!=" + string(types.Empty): {compile: compileResourceCmpEmpty},

mqlc/builtin_array.go

@@ -447,6 +447,38 @@ func compileArrayNone(c *compiler, typ types.Type, ref uint64, id string, call *
 	return types.Bool, nil
 }
 
+func compileArrayHaving(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) {
+	_, err := compileArrayWhere(c, typ, ref, "where", call)
+	if err != nil {
+		return types.Nil, err
+	}
+	whereRef := c.tailRef()
+
+	if err := compileListAssertionMsg(c, typ, ref, ref, whereRef); err != nil {
+		return types.Nil, err
+	}
+
+	c.addChunk(&llx.Chunk{
+		Call: llx.Chunk_FUNCTION,
+		Id:   "$any",
+		Function: &llx.Function{
+			Type:    string(types.Bool),
+			Binding: whereRef,
+		},
+	})
+	anyRef := c.tailRef()
+
+	checksum := c.Result.CodeV2.Checksums[anyRef]
+	c.Result.Labels.Labels[checksum] = "[].having()"
+
+	c.block.Entrypoints = append(c.block.Entrypoints, anyRef)
+	c.block.Datapoints = append(c.block.Datapoints, whereRef)
+
+	c.overrideTailDataRef = whereRef
+
+	return typ, nil
+}
+
 func compileArrayMap(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) {
 	if call == nil {
 		return types.Nil, errors.New("missing filter argument for calling '" + id + "'")

mqlc/builtin_map.go

@@ -437,6 +437,38 @@ func compileDictNone(c *compiler, typ types.Type, ref uint64, id string, call *p
 	return types.Bool, nil
 }
 
+func compileDictHaving(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) {
+	_, err := compileDictQuery(c, typ, ref, "where", call)
+	if err != nil {
+		return types.Nil, err
+	}
+	whereRef := c.tailRef()
+
+	if err := compileListAssertionMsg(c, typ, ref, ref, whereRef); err != nil {
+		return types.Nil, err
+	}
+
+	c.addChunk(&llx.Chunk{
+		Call: llx.Chunk_FUNCTION,
+		Id:   "$any",
+		Function: &llx.Function{
+			Type:    string(types.Bool),
+			Binding: whereRef,
+		},
+	})
+	anyRef := c.tailRef()
+
+	checksum := c.Result.CodeV2.Checksums[anyRef]
+	c.Result.Labels.Labels[checksum] = "[].having()"
+
+	c.block.Entrypoints = append(c.block.Entrypoints, anyRef)
+	c.block.Datapoints = append(c.block.Datapoints, whereRef)
+
+	c.overrideTailDataRef = whereRef
+
+	return typ, nil
+}
+
 func compileDictFlat(c *compiler, _ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) {
 	if call != nil && len(call.Function) > 0 {
 		return types.Nil, errors.New("no arguments supported for '" + id + "'")
@@ -658,3 +690,35 @@ func compileMapNone(c *compiler, typ types.Type, ref uint64, id string, call *pa
 
 	return types.Bool, nil
 }
+
+func compileMapHaving(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) {
+	_, err := compileMapWhere(c, typ, ref, "where", call)
+	if err != nil {
+		return types.Nil, err
+	}
+	whereRef := c.tailRef()
+
+	if err := compileListAssertionMsg(c, typ, ref, ref, whereRef); err != nil {
+		return types.Nil, err
+	}
+
+	c.addChunk(&llx.Chunk{
+		Call: llx.Chunk_FUNCTION,
+		Id:   "$any",
+		Function: &llx.Function{
+			Type:    string(types.Bool),
+			Binding: whereRef,
+		},
+	})
+	anyRef := c.tailRef()
+
+	checksum := c.Result.CodeV2.Checksums[anyRef]
+	c.Result.Labels.Labels[checksum] = "{}.having()"
+
+	c.block.Entrypoints = append(c.block.Entrypoints, anyRef)
+	c.block.Datapoints = append(c.block.Datapoints, whereRef)
+
+	c.overrideTailDataRef = whereRef
+
+	return typ, nil
+}

mqlc/builtin_resource.go

@@ -206,7 +206,7 @@ func compileResourceSample(c *compiler, typ types.Type, ref uint64, id string, c
 		return types.Nil, errors.New("too many arguments when calling '" + id + "', only 1 is supported")
 	}
 
-	resourceRef := c.tailRef()
+	resourceRef := ref
 
 	listType, err := compileResourceDefault(c, typ, ref, "list", nil)
 	if err != nil {
@@ -265,6 +265,8 @@ func compileResourceMap(c *compiler, typ types.Type, ref uint64, id string, call
 		bindingName = arg.Name
 	}
 
+	resourceRef := ref
+
 	refs, err := c.blockExpressions([]*parser.Expression{arg.Value}, types.Array(types.Type(resource.ListType)), ref, bindingName)
 	if err != nil {
 		return types.Nil, err
@@ -279,8 +281,6 @@ func compileResourceMap(c *compiler, typ types.Type, ref uint64, id string, call
 		return types.Nil, multierr.Wrap(err, "called '"+id+"' with a bad function block, types don't match")
 	}
 
-	resourceRef := c.tailRef()
-
 	listType, err := compileResourceDefault(c, typ, ref, "list", nil)
 	if err != nil {
 		return listType, err
@@ -487,6 +487,45 @@ func compileResourceNone(c *compiler, typ types.Type, ref uint64, id string, cal
 	return types.Bool, nil
 }
 
+func compileResourceHaving(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) {
+	// resource.where
+	_, err := compileResourceWhere(c, typ, ref, "where", call)
+	if err != nil {
+		return types.Nil, err
+	}
+	whereRef := c.tailRef()
+
+	listType, err := compileResourceDefault(c, typ, whereRef, "list", nil)
+	if err != nil {
+		return listType, err
+	}
+	listRef := c.tailRef()
+
+	if err := compileListAssertionMsg(c, listType, whereRef-1, whereRef-1, listRef); err != nil {
+		return types.Nil, err
+	}
+
+	c.addChunk(&llx.Chunk{
+		Call: llx.Chunk_FUNCTION,
+		Id:   "$any",
+		Function: &llx.Function{
+			Type:    string(types.Bool),
+			Binding: listRef,
+		},
+	})
+	anyRef := c.tailRef()
+
+	checksum := c.Result.CodeV2.Checksums[anyRef]
+	c.Result.Labels.Labels[checksum] = typ.ResourceName() + ".having()"
+
+	c.block.Entrypoints = append(c.block.Entrypoints, anyRef)
+	c.block.Datapoints = append(c.block.Datapoints, whereRef)
+
+	c.overrideTailDataRef = whereRef
+
+	return typ, nil
+}
+
 func compileResourceChildAccess(c *compiler, typ types.Type, ref uint64, id string, call *parser.Call) (types.Type, error) {
 	if call != nil && len(call.Function) > 0 {
 		return types.Nil, errors.New("function " + id + " does not take arguments")
@@ -527,7 +566,7 @@ func compileResourceLength(c *compiler, typ types.Type, ref uint64, id string, c
 		return types.Nil, multierr.Wrap(err, "failed to compile "+id)
 	}
 
-	resourceRef := c.tailRef()
+	resourceRef := ref
 
 	t, err := compileResourceDefault(c, typ, ref, "list", nil)
 	if err != nil {

mqlc/mqlc.go

@@ -158,6 +158,10 @@ type compiler struct {
 
 	// helps chaining of builtin calls like `if (..) else if (..) else ..`
 	prevID string
+
+	// overrideTailDataRef is set by `having` so that subsequent chained calls
+	// bind to the where-filtered list instead of the $any boolean chunk.
+	overrideTailDataRef uint64
 }
 
 func (c *compiler) isInMyBlock(ref uint64) bool {
@@ -180,6 +184,18 @@ func (c *compiler) tailRef() uint64 {
 	return c.block.TailRef(c.blockRef)
 }
 
+// tailDataRef returns overrideTailDataRef if set (consuming it),
+// otherwise falls back to tailRef. Used after builtin calls like
+// `having` where the data continuation point differs from the tail.
+func (c *compiler) tailDataRef() uint64 {
+	if c.overrideTailDataRef != 0 {
+		ref := c.overrideTailDataRef
+		c.overrideTailDataRef = 0
+		return ref
+	}
+	return c.tailRef()
+}
+
 // Creates a new block and its accompanying compiler.
 // It carries a set of variables that apply within the scope of this block.
 func (c *compiler) newBlockCompiler(binding *variable) compiler {
@@ -1503,7 +1519,7 @@ func (c *compiler) compileOperand(operand *parser.Operand) (*llx.Primitive, erro
 			if call != nil && len(calls) > 0 {
 				calls = calls[1:]
 			}
-			ref = c.tailRef()
+			ref = c.tailDataRef()
 			res = llx.RefPrimitiveV2(ref)
 
 			continue

mqlc/mqlc_test.go

@@ -2147,7 +2147,7 @@ func TestSuggestions(t *testing.T) {
 		{
 			// list resource with empty field call
 			"users.",
-			[]string{"all", "any", "contains", "first", "last", "length", "list", "map", "none", "one", "sample", "where"},
+			[]string{"all", "any", "contains", "first", "having", "last", "length", "list", "map", "none", "one", "sample", "where"},
 			errors.New("incomplete query, missing identifier after '.' at <source>:1:7"),
 			nil,
 		},