ci: add github actions

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly 

.github/workflows/build-test.yml

@@ -0,0 +1,47 @@
+name: Build and Test
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-latest
+          - ubuntu-latest
+          # - windows-latest
+        ocaml-version:
+          - 4.14
+          - 5.4
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Use OCaml ${{ matrix.ocaml-version }}
+        uses: ocaml/setup-ocaml@v3
+        with:
+          ocaml-compiler: ${{ matrix.ocaml-version }}
+          dune-cache: true
+          allow-prerelease-opam: true
+
+      - run: opam install . --deps-only --with-test
+
+      - name: build project
+        run: opam exec -- dune build
+
+      - name: run test
+        run: opam exec -- dune runtest
+
+      - run: opam install . --deps-only --with-test --criteria='+removed,+count[version-lag,solution]' --solver=builtin-0install
+
+      - name: build project with lower bounds
+        run: opam exec -- dune build
+
+      - name: run test with lower bounds
+        run: opam exec -- dune runtest

.github/workflows/lint.yml

@@ -0,0 +1,55 @@
+name: Lint
+
+on:
+  - push
+  - pull_request
+
+jobs:
+
+  lint-fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Use OCaml 5.4
+        uses: ocaml/setup-ocaml@v3
+        with:
+          ocaml-compiler: 5.4
+          dune-cache: true
+          allow-prerelease-opam: true
+
+      - name: Lint fmt
+        uses: ocaml/setup-ocaml/lint-fmt@v3
+
+  lint-opam:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Use OCaml 5.4
+        uses: ocaml/setup-ocaml@v3
+        with:
+          ocaml-compiler: 5.4
+          dune-cache: true
+          allow-prerelease-opam: true
+
+      - name: Lint opam
+        uses: ocaml/setup-ocaml/lint-opam@v3
+
+  lint-doc:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Use OCaml 5.4
+        uses: ocaml/setup-ocaml@v3
+        with:
+          ocaml-compiler: 5.4
+          dune-cache: true
+          allow-prerelease-opam: true
+
+      - name: Lint doc
+        uses: ocaml/setup-ocaml/lint-doc@v3

README.md

@@ -69,7 +69,27 @@ Syntax for variables in template json files is as follows:
 
 - `$var` for regular required variable
 - `$var?` for optional variable (minimal surrounding scope is conditionally expunged)
-- full form `$(var:hint)` where `hint` can be either `list` or `list?` currently
+- `$(var:list)` for list variable
+- `$(var:<type>)` for typed variable with explicit type annotation (see below)
+
+### Typed variables in aggregations
+
+When you need the same aggregation query against different fields, use a typed variable instead of duplicating the query template:
+
+```json
+"aggregations": {
+  "grouped_keywords": {
+    "terms": {
+      "field": $(group_by:keyword),
+      "size": 10
+    }
+  }
+}
+```
+
+The type annotation determines the output bucket key type. The variable itself becomes a `string` input parameter — the caller passes the ES field name at runtime (e.g., `"keyword_en"`).
+
+Type can be any ES type - `keyword`, `text`, `long`, `double`, `float`, `boolean`, `date`, `int64`, `ip`, `murmur3`.
 
 ## Configuration via `_esgg`
 
@@ -186,7 +206,7 @@ The following aggregation types are supported:
 - [ ] [sampler](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-sampler-aggregation.html) - Sampler aggregation
 - [ ] [diversified_sampler](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-diversified-sampler-aggregation.html) - Diversified sampler
 - [ ] [composite](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-composite-aggregation.html) - Composite aggregation for pagination
-- [ ] [multi_terms](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-multi-terms-aggregation.html) - Multi-field terms aggregation
+- [x] [multi_terms](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-multi-terms-aggregation.html) - Multi-field terms aggregation
 - [ ] [adjacency_matrix](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-adjacency-matrix-aggregation.html) - Adjacency matrix
 - [ ] [categorize_text](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-categorize-text-aggregation.html) - Text categorization
 - [ ] [frequent_item_sets](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-frequent-item-sets-aggregation.html) - Frequent item sets

aggregations.ml

@@ -24,6 +24,7 @@ type agg_type =
 | Nested of string
 | Reverse_nested of string option
 | Bucket_sort of { from : Tjson.t; size : Tjson.t; }
+| Multi_terms of { terms : value list; size : Tjson.t }
 | Cumulative_sum of string
 
 type single = { name : string; agg : agg_type or_var; }
@@ -41,7 +42,11 @@ let analyze_single name agg_type json =
   let value json =
     match U.member "field" json with
     | `String s -> Field s
-(*     | `Var v -> Variable v *)
+    | `Var v ->
+      begin match v.Tjson.type_ with
+      | Some ft -> Field_var (v, simple_of_es_type ft)
+      | None -> fail "variable in aggregation field requires type annotation, use $(var:<type>)"
+      end
     | `Null ->
       begin match U.assoc "script" json with
       | exception exn -> fail ~exn "failed to get aggregation field (neither `field` nor `script` present)"
@@ -105,6 +110,9 @@ let analyze_single name agg_type json =
         }
       }
     | "terms" -> Terms { term = value json; size = U.member "size" json }
+    | "multi_terms" ->
+      let terms = U.(get "terms" (to_list value)) json in
+      Multi_terms { terms; size = U.member "size" json }
     | "significant_terms" -> Significant_terms { term = value json; size = U.member "size" json }
     | "significant_text" -> Significant_text { term = value json; size = U.member "size" json }
     | "histogram" -> Histogram (value json)
@@ -177,7 +185,7 @@ let derive_fields mapping fields =
   match Hit.of_mapping ~filter:{excludes=None;includes=Some fields} mapping with
   | Dict l ->
     l |> linearize_dict |> List.map begin function
-    | (k, (List _ | List_or_single _ | Dict _ | Object _)) -> fail "expected simple type for %S" k
+    | (k, (List _ | List_or_single _ | Dict _ | Object _ | Tuple _)) -> fail "expected simple type for %S" k
     | (k, Maybe t) -> k, List t (* what will ES do? but seems safe either way *)
     | (k, ((Ref _ | Simple _) as t)) -> k, List t
     end
@@ -188,6 +196,11 @@ let derive_highlight mapping hl =
 
 let dummy_expunge = Dict ["dummy_expunge", Simple Json]
 
+let field_var_constraints = function
+  | Field_var (v, _typ) -> [On_var (v, Eq_type String)] (* String is the type of the variable value (field name), not the aggregation result *)
+  | Field _name -> []
+  | Script (_lang, _src) -> []
+
 let infer_single mapping ~nested { name; agg; } sibling sub =
   let int = Simple Int in
   let double = Simple Double in
@@ -222,29 +235,31 @@ let infer_single mapping ~nested { name; agg; } sibling sub =
         | Simple Date | Ref (_,Date) -> "value_as_string" (* Date is mapped to string, but value in ES is numeric *)
         | _ -> "value"
       in
-      [], sub [ key, typ ]
+      field_var_constraints value, sub [ key, typ ]
     | Cumulative_sum path ->
       (* TODO https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-pipeline.html#buckets-path-syntax *)
       begin match List.assoc_opt path sibling with
       | None -> fail "cumulative_sum: buckets_path %S not found : available %s" path (String.concat " " @@ List.map fst sibling)
       | Some v -> [], v
       end
-    | Cardinality _value | Value_count _value -> [], sub ["value", int ]
-    | Weighted_avg { value=_; weight=_ } -> [], sub ["value", Maybe double]
-    | Terms { term; size } -> on_int_var size, buckets (typeof_value mapping term)
+    | Cardinality value | Value_count value -> field_var_constraints value, sub ["value", int ]
+    | Weighted_avg { value; weight } -> field_var_constraints value.value @ field_var_constraints weight.value, sub ["value", Maybe double]
+    | Terms { term; size } -> field_var_constraints term @ on_int_var size, buckets (typeof_value mapping term)
+    | Multi_terms { terms; size } ->
+      List.concat_map field_var_constraints terms @ on_int_var size, buckets (Tuple (List.map (typeof_value mapping) terms))
     | Significant_terms { term; size } ->
-      on_int_var size,
+      field_var_constraints term @ on_int_var size,
       Dict [
         "doc_count", int;
         "bg_count", int;
         "buckets", List (sub @@ ("key", typeof_value mapping term) :: ("doc_count", int) :: ("bg_count", int) :: ("score", double) ::[]) ]
     | Significant_text { term; size } ->
-      on_int_var size,
+      field_var_constraints term @ on_int_var size,
       Dict [
         "doc_count", int;
         "buckets", List (sub @@ ("key", typeof_value mapping term) :: ("doc_count", int) :: ("bg_count", int) :: ("score", double) ::[]) ]
-    | Histogram value -> [Field_num value], buckets double
-    | Date_histogram { on; format } -> [Field_date on], buckets int ~extra:(if format then ["key_as_string", string] else [])
+    | Histogram value -> Field_num value :: field_var_constraints value, buckets double
+    | Date_histogram { on; format } -> Field_date on :: field_var_constraints on, buckets int ~extra:(if format then ["key_as_string", string] else [])
     | Nested _ | Reverse_nested _ -> [], doc_count ()
     | Filter q -> dynamic_default [] Query.infer q, doc_count ()
     | Filters { filters = `Assoc filters; other_bucket } ->
@@ -258,10 +273,10 @@ let infer_single mapping ~nested { name; agg; } sibling sub =
     | Top_hits { source; highlight; } ->
       let highlight = Option.map (derive_highlight mapping) highlight in
       [], Dict [ "hits", sub ((Hit.hits_ mapping ~highlight ?nested source)) ]
-    | Range value -> [Field_num value], buckets string
-    | Range_keyed (value,keys) -> [Field_num value], keyed_buckets keys
+    | Range value -> Field_num value :: field_var_constraints value, buckets string
+    | Range_keyed (value,keys) -> Field_num value :: field_var_constraints value, keyed_buckets keys
     | Date_range { on; format=_; keys; ranges=_ } ->
-      [Field_date on], (match keys with None -> buckets string | Some keys -> keyed_buckets keys)
+      Field_date on :: field_var_constraints on, (match keys with None -> buckets string | Some keys -> keyed_buckets keys)
     | Bucket_sort { from; size } ->
       on_int_var from @ on_int_var size,
       dummy_expunge (* lazy hack to not wrap all results in option *)

atdgen.ml

@@ -17,6 +17,12 @@ let tvar t = Atd.Ast.Tvar (loc,t)
 let ptyp ?(a=[]) name params t = Atd.Ast.Type (loc, (name,params,annots a), t)
 let typ ?a name t = ptyp ?a name [] t
 
+let ends_with s suffix =
+  let s_len = Stdlib.String.length s in
+  let suffix_len = Stdlib.String.length suffix in
+  s_len >= suffix_len
+  && Stdlib.String.sub s (s_len - suffix_len) suffix_len = suffix
+
 let of_simple_type =
   function
   | Int -> tname "int"
@@ -240,7 +246,8 @@ end = struct
       | List (loc, ty, annot) ->
         let name =
           match name with
-          | Some name when String.ends_with name "s" && String.length name > 1 -> Some (String.slice ~last:(-1) name)
+          | Some name when ends_with name "s" && String.length name > 1 ->
+            Some (String.slice ~last:(-1) name)
           | Some name -> Some (name ^ "_elem")
           | None -> None
         in
@@ -317,6 +324,7 @@ let add_shape t name shape =
     | List t -> list (map t)
     | List_or_single (Ref (ref,_)) -> wrap_ref (ES_name.append ref "list_or_single") (tname "basic_json");
     | List_or_single t -> fail "cannot handle mixed_multi of %s" (show_result_type t)
+    | Tuple ts -> tuple (List.map map ts)
     | Object t -> list ~a:["json",["repr","object"]] (tuple [map (Simple String); map t])
     | Dict ["key",k; "doc_count", Simple Int] -> pname "doc_count" [map k]
     | Dict ["buckets", List t] -> pname "buckets" [map t]
@@ -352,7 +360,7 @@ let add_vars t (l:input_vars) =
   in
   Types.add t @@ typ "input" (map l)
 
-let make_module ~init f : Atd.Ast.full_module =
+let make_module ~init f =
   let t = Types.empty () in
   List.iter (Types.add t) (snd init);
   f t;

common.ml

@@ -17,10 +17,6 @@ let var_or conv x =
   | _ ->
     try Static (conv x) with Failure s -> fail "expected var or %s" s
 
-type value = Field of string | Script of [`Painless|`Id] * string or_var [@@deriving show]
-
-type value_with_missing = { value : value; missing : Tjson.t }
-
 module U = struct
 
 (** @return specified [name] from [json] dict or [`Null] when [name] is absent.
@@ -113,6 +109,10 @@ let show_simple_type = function
 
 let pp_simple_type ppf x = Format.pp_print_text ppf (show_simple_type x)
 
+type value = Field of string | Field_var of Tjson.var * simple_type | Script of [`Painless|`Id] * string or_var [@@deriving show]
+
+type value_with_missing = { value : value; missing : Tjson.t }
+
 type cardinality = One | Many
 
 type var_type = { cardinality : cardinality; ref : ES_name.t option; typ : simple_type; }
@@ -141,6 +141,7 @@ type result_type =
   | List of result_type
   | List_or_single of result_type
   | Object of result_type
+  | Tuple of result_type list
   | Dict of (string * result_type) list
   | Ref of (ES_name.t * simple_type)
   | Maybe of result_type
@@ -158,6 +159,7 @@ let simple_of_es_type t =
   | "int64" | "murmur3" -> Int64
   | _ -> fail "simple_of_es_type: cannot handle %S" t
 
+
 let get_meta json =
   match U.member "_meta" json with
   | `Null -> `Assoc []
@@ -191,6 +193,7 @@ let typeof_value mapping value =
   match value with
   | Script (`Painless, Static "_score") -> Simple Double
   | Script _ -> Simple Json
+  | Field_var (_v, typ) -> Simple typ
   | Field f ->
     let name = ES_name.make mapping f in
     let t = typeof mapping name in

derive.ml

@@ -48,7 +48,22 @@ let output mapping query =
     let inner_hits_type = generate_inner_hits mapping source ?matched_queries inner_hits_specs in
     let hits = Hit.hits mapping ?inner_hits:inner_hits_type ~highlight ?fields ?matched_queries source in
     let aggs = List.map snd @@ snd @@ Aggregations.analyze mapping query in (* XXX discarding constraints *)
-    let shards = Maybe (Dict ["total", Simple Int; "successful", Simple Int; "skipped", Simple Int; "failed", Simple Int]) in
+    let shards = Maybe (Dict [
+      "total", Simple Int;
+      "successful", Simple Int;
+      "skipped", Simple Int;
+      "failed", Simple Int;
+      "failures", Maybe (List (Dict [
+        "reason", Dict [
+          "type", Simple String;
+          "reason", Maybe (Simple String);
+        ];
+        "index", Maybe (Simple String);
+        "node", Maybe (Simple String);
+        "shard", Maybe (Simple Int);
+        "status", Maybe (Simple String);
+      ]));
+    ]) in
     Dict (("hits", hits) :: ("_shards", shards) :: (if aggs = [] then [] else ["aggregations", Dict aggs]))
 
 let print_reflect name mapping =

esgg.opam

@@ -16,7 +16,7 @@ depends: [
   "ocaml" { >= "4.04.0" }
   "yojson" {>= "2.0.0"}
   "extlib" {>= "1.7.1"}
-  "atd" {>= "2.0.0"}
+  "atd" {>= "2.0.0" & < "3.0.0"}
   "easy-format"
   "jsonm"
   "ppx_deriving" {>= "4.2"}