feat: Allow validating testoperator in benchmark workflow (spiceai#5342)

* feat: Allow validation during testoperator benchmark

* fix: Update schema equivalence in validator

Author: peasee

.github/workflows/testoperator_run_bench.yml

@@ -61,6 +61,11 @@ on:
         options:
           - 'always'
           - 'no'
+      validate_results:
+        description: 'Validate the results?'
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   run-bench:
@@ -128,7 +133,8 @@ jobs:
             -d /opt/spiced/data \
             --query-set ${{ github.event.inputs.query_set }} \
             --ready-wait ${{ github.event.inputs.ready_wait }} \
-            --disable-progress-bars
+            --disable-progress-bars \
+            --validate=${{ github.event.inputs.validate_results }}
         env:
           S3_ENDPOINT: ${{ secrets.MINIO_ENDPOINT}}
           S3_KEY: ${{ secrets.MINIO_ACCESS_KEY}}
@@ -168,7 +174,15 @@ jobs:
         if: ${{ github.event.inputs.query_overrides != '' }}
         run: |
           rm -rf .spice/data
-          INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" testoperator run bench -s /usr/local/bin/spiced -p ${{ github.event.inputs.spicepod_path }} -d /opt/spiced/data --query-set ${{ github.event.inputs.query_set }} --query-overrides ${{ github.event.inputs.query_overrides }} --ready-wait ${{ github.event.inputs.ready_wait }} --disable-progress-bars
+          INSTA_WORKSPACE_ROOT="${PWD}" CARGO_MANIFEST_DIR="${PWD}" testoperator run bench \
+            -s /usr/local/bin/spiced \
+            -p ${{ github.event.inputs.spicepod_path }} \
+            -d /opt/spiced/data \
+            --query-set ${{ github.event.inputs.query_set }} \
+            --query-overrides ${{ github.event.inputs.query_overrides }} \
+            --ready-wait ${{ github.event.inputs.ready_wait }} \
+            --disable-progress-bars \
+            --validate=${{ github.event.inputs.validate_results }}
         env:
           S3_ENDPOINT: ${{ secrets.MINIO_ENDPOINT}}
           S3_KEY: ${{ secrets.MINIO_ACCESS_KEY}}

crates/test-framework/src/queries/validation/mod.rs

@@ -137,7 +137,7 @@ fn datatype_equivalent(expected_type: DataType, actual_type: DataType) -> bool {
             | (DataType::Int32, DataType::Int64)
             | (
                 DataType::Int64,
-                DataType::Int32 | DataType::Float64 | DataType::Utf8
+                DataType::Int32 | DataType::Float64 | DataType::Utf8 | DataType::LargeUtf8
             )
             | (DataType::Utf8, DataType::LargeUtf8)
     )

tools/testoperator/src/args/dataset.rs

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-use clap::{Parser, ValueEnum};
+use clap::{ArgAction, Parser, ValueEnum};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 use test_framework::queries::{QueryOverrides, QuerySet};
@@ -37,7 +37,7 @@ pub struct DatasetTestArgs {
     #[arg(long)]
     pub(crate) query_overrides: Option<QueryOverridesArg>,
 
-    #[arg(long)]
+    #[arg(long, action = ArgAction::Set, default_value_t = false, default_missing_value = "true", num_args = 0..=1, require_equals = false)]
     pub(crate) validate: bool,
 }