Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,55 @@ jobs:
if: matrix.check == 'sort'
run: cargo sort --workspace --grouped --check

check-tls-aws-lc:
name: TLS Backend (AWS-LC only)
runs-on: blacksmith-4vcpu-ubuntu-2404
env:
# Library crates with default features off and only the AWS-LC backend selected.
TLS_FEATURES: etl/tls-rustls-aws-lc-rs,etl-postgres/tls-rustls-aws-lc-rs,etl-postgres/store,etl-maintenance/tls-rustls-aws-lc-rs,etl-destinations/tls-rustls-aws-lc-rs
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

- name: Set up Rust
uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1

- name: Cache Cargo
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
with:
key: tls-aws-lc

- name: Verify SQLx Ring Feature Is Absent
shell: bash
run: |
set -euo pipefail

# ducklake makes sqlx a dependency of etl-destinations and etl-maintenance;
# resolving the feature graph does not compile DuckDB.
sqlx_features="$(cargo tree \
-p etl -p etl-postgres -p etl-maintenance -p etl-destinations \
--no-default-features \
--features "${TLS_FEATURES},etl-destinations/ducklake" \
-e features -i sqlx)"

if ! grep -q 'sqlx feature "tls-rustls-aws-lc-rs"' <<< "${sqlx_features}"; then
echo "sqlx feature tls-rustls-aws-lc-rs is not enabled" >&2
exit 1
fi

if grep -q 'sqlx feature "tls-rustls-ring"' <<< "${sqlx_features}"; then
echo "sqlx feature tls-rustls-ring is enabled by:" >&2
grep -A5 'sqlx feature "tls-rustls-ring"' <<< "${sqlx_features}" >&2
exit 1
fi

- name: Check AWS-LC Build
run: |
cargo check \
-p etl -p etl-postgres -p etl-maintenance -p etl-destinations \
--no-default-features \
--features "${TLS_FEATURES}"

check-actions:
name: Actions Lint
runs-on: blacksmith-4vcpu-ubuntu-2404
Expand Down
10 changes: 5 additions & 5 deletions crates/etl-api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ base64 = { workspace = true, features = ["std"] }
chrono = { workspace = true }
configcat = { workspace = true }
constant_time_eq = { workspace = true }
etl = { workspace = true }
etl = { workspace = true, features = ["tls-rustls-ring"] }
etl-config = { workspace = true, features = ["utoipa", "supabase"] }
etl-destinations = { workspace = true }
etl-maintenance = { workspace = true }
etl-postgres = { workspace = true, features = ["store"] }
etl-destinations = { workspace = true, features = ["tls-rustls-ring"] }
etl-maintenance = { workspace = true, features = ["tls-rustls-ring"] }
etl-postgres = { workspace = true, features = ["store", "tls-rustls-ring"] }
etl-telemetry = { workspace = true }
k8s-openapi = { workspace = true, features = ["latest"] }
kube = { workspace = true, features = ["runtime", "derive", "client", "rustls-tls"] }
Expand All @@ -52,7 +52,7 @@ secrecy = { workspace = true }
sentry = { workspace = true, features = ["tower-http", "tower-axum-matched-path"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true, features = ["std"] }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls", "macros", "postgres", "json", "migrate"] }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls-ring", "macros", "postgres", "json", "migrate"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
tokio-rustls = { workspace = true, features = ["aws-lc-rs", "logging", "tls12"] }
Expand Down
8 changes: 4 additions & 4 deletions crates/etl-benchmarks/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,16 @@ snowflake = ["dep:etl-destinations", "etl-destinations/snowflake"]
[dependencies]
anyhow = { workspace = true }
clap = { workspace = true, default-features = true, features = ["std", "derive"] }
etl = { workspace = true, features = ["test-utils"] }
etl = { workspace = true, features = ["test-utils", "tls-rustls-ring"] }
etl-config = { workspace = true }
etl-destinations = { workspace = true, optional = true }
etl-postgres = { workspace = true, features = ["sqlx"] }
etl-destinations = { workspace = true, optional = true, features = ["tls-rustls-ring"] }
etl-postgres = { workspace = true, features = ["sqlx", "tls-rustls-ring"] }
etl-telemetry = { workspace = true }
k8s-openapi = { workspace = true, features = ["latest"] }
rustls = { workspace = true, features = ["aws-lc-rs", "logging"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls", "postgres", "migrate"] }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls-ring", "postgres", "migrate"] }
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "signal"] }
tracing = { workspace = true, default-features = true }
url = { workspace = true }
Expand Down
13 changes: 12 additions & 1 deletion crates/etl-destinations/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,17 @@ homepage.workspace = true
doctest = false

[features]
default = ["tls-rustls-ring"]
tls-rustls-ring = [
"etl/tls-rustls-ring",
"etl-maintenance?/tls-rustls-ring",
"sqlx?/tls-rustls-ring",
]
tls-rustls-aws-lc-rs = [
"etl/tls-rustls-aws-lc-rs",
"etl-maintenance?/tls-rustls-aws-lc-rs",
"sqlx?/tls-rustls-aws-lc-rs",
]
ducklake = [
"dep:duckdb",
"dep:etl-maintenance",
Expand Down Expand Up @@ -128,7 +139,7 @@ secrecy = { workspace = true, optional = true }
serde = { workspace = true, optional = true, features = ["derive"] }
serde_json = { workspace = true, optional = true, features = ["arbitrary_precision", "std"] }
sha2 = { workspace = true, optional = true }
sqlx = { workspace = true, optional = true, features = ["runtime-tokio", "tls-rustls", "postgres"] }
sqlx = { workspace = true, optional = true, features = ["runtime-tokio", "postgres"] }
thiserror = { workspace = true, optional = true }
tokio = { workspace = true, optional = true, features = ["rt", "sync", "time"] }
tokio-postgres = { workspace = true, optional = true }
Expand Down
2 changes: 1 addition & 1 deletion crates/etl-destinations/src/snowflake/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ To run a specific destination test directly:

```bash
source .env
cargo test -p etl-destinations --no-default-features --features snowflake,test-utils -- --ignored authenticate_against_snowflake
cargo test -p etl-destinations --no-default-features --features snowflake,test-utils,tls-rustls-ring -- --ignored authenticate_against_snowflake
```

### Connection String
Expand Down
4 changes: 2 additions & 2 deletions crates/etl-examples/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ snowflake = ["etl-destinations/snowflake"]
[dependencies]

clap = { workspace = true, default-features = true, features = ["std", "derive", "env"] }
etl = { workspace = true }
etl = { workspace = true, features = ["tls-rustls-ring"] }
etl-config = { workspace = true }
etl-destinations = { workspace = true }
etl-destinations = { workspace = true, features = ["tls-rustls-ring"] }
k8s-openapi = { workspace = true, features = ["latest"] }
rustls = { workspace = true, features = ["aws-lc-rs", "logging"] }
tokio = { workspace = true, features = ["macros", "signal", "time"] }
Expand Down
6 changes: 4 additions & 2 deletions crates/etl-maintenance/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ homepage.workspace = true
doctest = false

[features]
default = []
default = ["tls-rustls-ring"]
tls-rustls-ring = ["sqlx/tls-rustls-ring", "etl/tls-rustls-ring"]
tls-rustls-aws-lc-rs = ["sqlx/tls-rustls-aws-lc-rs", "etl/tls-rustls-aws-lc-rs"]
ducklake = [
"dep:duckdb",
"dep:kube",
Expand Down Expand Up @@ -40,7 +42,7 @@ r2d2 = { workspace = true, optional = true }
regex = { workspace = true, optional = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true, optional = true, features = ["arbitrary_precision", "std"] }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls", "postgres", "migrate", "chrono"] }
sqlx = { workspace = true, features = ["runtime-tokio", "postgres", "migrate", "chrono"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt", "sync", "time"] }
tokio-postgres = { workspace = true, optional = true }
Expand Down
5 changes: 4 additions & 1 deletion crates/etl-postgres/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ homepage.workspace = true
doctest = false

[features]
default = ["tls-rustls-ring"]
tls-rustls-ring = ["sqlx/tls-rustls-ring"]
tls-rustls-aws-lc-rs = ["sqlx/tls-rustls-aws-lc-rs"]
test-utils = []
tokio = []
sqlx = []
Expand All @@ -30,7 +33,7 @@ futures = { workspace = true }
pg_escape = { workspace = true }
rustls = { workspace = true, features = ["aws-lc-rs"] }
serde_json = { workspace = true }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls", "macros", "postgres", "json", "migrate"] }
sqlx = { workspace = true, features = ["runtime-tokio", "macros", "postgres", "json", "migrate"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
tokio-postgres = { workspace = true, features = ["runtime", "with-chrono-0_4", "with-uuid-1", "with-serde_json-1"] }
Expand Down
10 changes: 10 additions & 0 deletions crates/etl-postgres/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,16 @@
//! This crate owns Postgres-specific helpers that need to be shared across
//! crates: source database access, replication slot naming, ETL metadata-store
//! queries, schema primitives, value wrappers, and type conversion helpers.
//!
//! Select the rustls cryptography backend with the `tls-rustls-ring` (default)
//! or `tls-rustls-aws-lc-rs` feature; the build fails when neither is enabled.

// SQLx is a required dependency used by the source, slot, lag, and store
// modules, so fail loudly instead of silently building it without TLS.
#[cfg(not(any(feature = "tls-rustls-ring", feature = "tls-rustls-aws-lc-rs")))]
compile_error!(
"Either the `tls-rustls-ring` or the `tls-rustls-aws-lc-rs` feature must be enabled."
);

pub mod application_name;
pub mod default_expression;
Expand Down
8 changes: 4 additions & 4 deletions crates/etl-replicator/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ egress = ["etl/egress"]
[dependencies]

configcat = { workspace = true }
etl = { workspace = true }
etl = { workspace = true, features = ["tls-rustls-ring"] }
etl-config = { workspace = true, features = ["supabase"] }
etl-destinations = { workspace = true }
etl-maintenance = { workspace = true }
etl-destinations = { workspace = true, features = ["tls-rustls-ring"] }
etl-maintenance = { workspace = true, features = ["tls-rustls-ring"] }
etl-telemetry = { workspace = true }
k8s-openapi = { workspace = true, features = ["latest"] }
metrics = { workspace = true }
Expand All @@ -39,7 +39,7 @@ secrecy = { workspace = true }
sentry = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls", "postgres", "migrate"] }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls-ring", "postgres", "migrate"] }

tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] }
tracing = { workspace = true, default-features = true }
Expand Down
6 changes: 4 additions & 2 deletions crates/etl/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@ repository.workspace = true
homepage.workspace = true

[features]
default = ["tls-rustls-ring"]
tls-rustls-ring = ["sqlx/tls-rustls-ring", "etl-postgres/tls-rustls-ring"]
tls-rustls-aws-lc-rs = ["sqlx/tls-rustls-aws-lc-rs", "etl-postgres/tls-rustls-aws-lc-rs"]
test-utils = ["etl-postgres/test-utils", "dep:proptest"]
failpoints = ["fail/failpoints"]
egress = []
fuzzing = []
default = []

[dependencies]

Expand All @@ -36,7 +38,7 @@ rustls = { workspace = true, features = ["aws-lc-rs", "logging"] }
serde = { workspace = true, features = ["derive", "alloc"] }
serde_json = { workspace = true, features = ["arbitrary_precision", "std"] }
simdutf8 = { workspace = true }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls", "postgres", "migrate"] }
sqlx = { workspace = true, features = ["runtime-tokio", "postgres", "migrate"] }
sysinfo = { workspace = true, features = ["system"] }
tokio = { workspace = true, features = ["rt-multi-thread", "test-util"] }
tokio-postgres = { workspace = true, features = ["runtime", "with-chrono-0_4", "with-uuid-1", "with-serde_json-1"] }
Expand Down
14 changes: 14 additions & 0 deletions crates/etl/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,9 +173,23 @@
//!
//! # Feature Flags
//!
//! - `tls-rustls-ring` (default): Use `ring` as the rustls cryptography backend
//! for SQLx connections
//! - `tls-rustls-aws-lc-rs`: Use `aws-lc-rs` as the rustls cryptography backend
//! for SQLx connections
//! - `egress`: Enable structured billing usage logs
//! - `test-utils`: Enable testing utilities and mock implementations
//! - `failpoints`: Enable fault injection for testing error scenarios
//!
//! At least one TLS backend must be enabled; the build fails when neither is.
//! To link only `aws-lc-rs`, disable default features and enable
//! `tls-rustls-aws-lc-rs`. When both backends are enabled, SQLx uses `ring`.

// Fail loudly instead of silently building SQLx without TLS.
#[cfg(not(any(feature = "tls-rustls-ring", feature = "tls-rustls-aws-lc-rs")))]
compile_error!(
"Either the `tls-rustls-ring` or the `tls-rustls-aws-lc-rs` feature must be enabled."
);

pub mod config;
pub mod data;
Expand Down
2 changes: 1 addition & 1 deletion crates/xtask/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ schemars = "0.8"
secrecy = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls", "postgres", "json"] }
sqlx = { workspace = true, features = ["runtime-tokio", "tls-rustls-ring", "postgres", "json"] }
tokio = { workspace = true, features = ["full"] }
toml = "1.1"
xshell = "0.2"
Expand Down
2 changes: 1 addition & 1 deletion crates/xtask/src/commands/test_snowflake.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ impl TestSnowflakeArgs {
eprintln!("{GREEN}❄️ running Snowflake destination integration tests.{RESET}");
let tests = CargoFeatureSelection::new(
true,
vec!["snowflake".to_owned(), "test-utils".to_owned()],
vec!["snowflake".to_owned(), "test-utils".to_owned(), "tls-rustls-ring".to_owned()],
vec!["etl-destinations".to_owned()],
)
.apply_to(
Expand Down
2 changes: 1 addition & 1 deletion fuzz/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ edition = "2024"
cargo-fuzz = true

[dependencies]
etl = { path = "../crates/etl", default-features = false, features = ["fuzzing"] }
etl = { path = "../crates/etl", default-features = false, features = ["fuzzing", "tls-rustls-ring"] }
libfuzzer-sys = "0.4"

[[bin]]
Expand Down