Skip to content

Commit ac1821e

Browse files
ManishSharma1609ManishSharma1609
authored
Merge branch 'trunk' into refactor/embedding-params-handling
2 parents 0f91de8 + b254021 commit ac1821e

43 files changed

Lines changed: 365 additions & 142 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/testoperator_dispatch_htap.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: testoperator dispatch htap
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
spiced_commit:
7+
description: 'An optional commit hash to use for spiced'
8+
required: false
9+
type: string
10+
scale_factor:
11+
description: 'Scale factor to run'
12+
required: true
13+
type: choice
14+
options:
15+
- 'sf1'
16+
- 'sf10'
17+
- 'sf100'
18+
- 'sf1000'
19+
20+
jobs:
21+
dispatch-htap:
22+
name: Dispatch HTAP - ${{ github.event.inputs.scale_factor }}
23+
runs-on: spiceai-dev-runners
24+
concurrency:
25+
group: testoperator-dispatch-htap-${{ github.event.ref }}
26+
cancel-in-progress: false
27+
steps:
28+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
29+
with:
30+
persist-credentials: false
31+
32+
- name: Install MinIO
33+
uses: ./.github/actions/setup-minio
34+
with:
35+
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
36+
minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
37+
minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
38+
39+
- name: Setup spiced
40+
uses: ./.github/actions/setup-spiced
41+
id: setup-spiced
42+
with:
43+
spiced_commit: ${{ github.event.inputs.spiced_commit }}
44+
ref: ${{ github.event.ref }}
45+
46+
- name: Display spiced commit
47+
run: echo "SPICED_COMMIT=${{ steps.setup-spiced.outputs.SPICED_COMMIT }}"
48+
49+
- name: Build Testoperator
50+
uses: ./.github/actions/build-testoperator
51+
with:
52+
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
53+
minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
54+
minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
55+
56+
- name: Build spicepod validator
57+
id: build-spicepod-validator
58+
uses: ./.github/actions/build-spicepod-validator
59+
with:
60+
minio_endpoint: ${{ secrets.TEST_MINIO_ENDPOINT }}
61+
minio_access_key: ${{ secrets.TEST_MINIO_ACCESS_KEY }}
62+
minio_secret_key: ${{ secrets.TEST_MINIO_SECRET_KEY }}
63+
64+
- name: Set spicepod validator path
65+
run: echo "SPICEPOD_VALIDATOR=${{ steps.build-spicepod-validator.outputs.validator-path }}" >> $GITHUB_ENV
66+
67+
- name: Validate spicepods - CH-BenCHmark
68+
run: |
69+
shopt -s globstar nullglob
70+
for file in ./test/spicepods/chbench/**/*.yaml; do
71+
echo "Validating $file"
72+
"$SPICEPOD_VALIDATOR" "$file"
73+
done
74+
75+
- name: Dispatch Testoperator - HTAP - CH-BenCHmark - ${{ github.event.inputs.scale_factor }}
76+
run: |
77+
testoperator dispatch ./tools/testoperator/dispatch/chbench/${{ github.event.inputs.scale_factor }} \
78+
--workflow htap
79+
env:
80+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
81+
SPICED_COMMIT: ${{ steps.setup-spiced.outputs.SPICED_COMMIT }}
82+
WORKFLOW_COMMIT: ${{ github.event.ref }}

‎crates/cayenne/tests/small_files_compaction_test.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,7 @@ async fn compaction_collapses_tiny_protected_snapshots(
408408
let config = VortexConfig {
409409
target_vortex_file_size_mb: 128,
410410
compaction_trigger_files: 4,
411+
compaction_trigger_protected_snapshots: 4,
411412
compaction_background_interval_ms: 0,
412413
..Default::default()
413414
};

‎crates/runtime/src/dataaccelerator/duckdb.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ pub(crate) fn create_factory() -> DuckDBTableProviderFactory {
9696
.with_function_support(deny_spice_functions_for_duckdb().as_ref().clone())
9797
}
9898

99-
pub(crate) const DEFAULT_MIN_IDLE_CONNECTIONS: u32 = 10;
10099
pub(crate) const DEFAULT_CONNECTION_POOL_SIZE: u32 = 10;
101100
pub(crate) const DEFAULT_EBS_CONNECTION_POOL_SIZE: u32 = 4;
102101
pub(crate) const SPICE_ACCELERATOR_METADATA_KEY: &str = "spice.accelerator";

‎crates/runtime/src/dataaccelerator/partitioned_duckdb.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ use super::{
5858
use crate::{
5959
component::dataset::acceleration::{Engine, Mode},
6060
dataaccelerator::{FilePathError, storage::resolve_acceleration_storage_async},
61-
datafusion::{dialect::new_duckdb_dialect, udf::deny_spice_functions_for_duckdb},
6261
parameters::ParameterSpec,
6362
register_data_accelerator, spice_data_base_path,
6463
};

‎crates/runtime/src/datafusion/builder.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,12 +1132,12 @@ mod tests {
11321132
.build();
11331133

11341134
let state = df.ctx.state();
1135-
let rule_names: Vec<&str> = state.optimizers().iter().map(|r| r.name()).collect();
11361135

11371136
assert!(
1138-
!rule_names
1137+
!state
1138+
.optimizers()
11391139
.iter()
1140-
.any(|name| *name == "cayenne_propagate_filter_across_equi_join_keys"),
1140+
.any(|r| r.name() == "cayenne_propagate_filter_across_equi_join_keys"),
11411141
"Cayenne logical filter propagation should be disabled by default"
11421142
);
11431143
}

‎crates/test-framework/src/queries/chbench/q10.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ SELECT
66
c_phone,
77
n_name
88
FROM
9-
customer, orders, order_line, nation
9+
customer, oorder, order_line, nation
1010
WHERE
1111
c_id = o_c_id
1212
AND c_w_id = o_w_id

‎crates/test-framework/src/queries/chbench/q12.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ SELECT
33
sum(CASE WHEN o_carrier_id = 1 OR o_carrier_id = 2 THEN 1 ELSE 0 END) AS high_line_count,
44
sum(CASE WHEN o_carrier_id <> 1 AND o_carrier_id <> 2 THEN 1 ELSE 0 END) AS low_line_count
55
FROM
6-
orders, order_line
6+
oorder, order_line
77
WHERE
88
ol_w_id = o_w_id
99
AND ol_d_id = o_d_id

‎crates/test-framework/src/queries/chbench/q13.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ SELECT
22
c_count, count(*) AS custdist
33
FROM
44
(SELECT c_id, count(o_id) AS c_count
5-
FROM customer LEFT OUTER JOIN orders ON (
5+
FROM customer LEFT OUTER JOIN oorder ON (
66
c_w_id = o_w_id
77
AND c_d_id = o_d_id
88
AND c_id = o_c_id

‎crates/test-framework/src/queries/chbench/q18.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ SELECT
66
o_ol_cnt,
77
sum(ol_amount) AS amount_sum
88
FROM
9-
customer, orders, order_line
9+
customer, oorder, order_line
1010
WHERE
1111
c_id = o_c_id
1212
AND c_w_id = o_w_id

‎crates/test-framework/src/queries/chbench/q21.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ SELECT
33
FROM
44
supplier,
55
order_line l1,
6-
orders,
6+
oorder,
77
stock,
88
nation
99
WHERE

0 commit comments

Comments
 (0)