Skip to content

Commit 04baacc

Browse files
committed
Add support for hashing transformations and caching configuration
Enhancements: - Introduced HashTransform for hashing field values using algorithms like MD5, SHA256, and Murmur. - Added commit strategy configuration to control commit semantics (manual vs. auto) and enable dead letter queues. - Implemented caching backend configuration options, including local, Redis, and Kafka-backed caches. Documentation: - Updated README and SECRETS.md to reflect new features and usage examples. - Enhanced examples to demonstrate multi-cluster secure pipelines and secret management. Benefits: ✅ Improved data processing capabilities with hashing. ✅ Flexible commit strategies for message delivery. ✅ Enhanced caching options for better performance and scalability.
1 parent d0f3a60 commit 04baacc

23 files changed

Lines changed: 5635 additions & 54 deletions

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ Cargo.lock
77
config.json
88
*.log
99
test-values.yaml
10+
operator/target

Cargo.toml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,21 @@ async-trait = "0.1"
4545
bytes = "1.7"
4646
regex = "1.10" # Regular expressions
4747

48+
# Hashing functions
49+
md-5 = "0.10"
50+
sha2 = "0.10"
51+
murmur3 = "0.5"
52+
digest = "0.10"
53+
hex = "0.4"
54+
55+
# Caching
56+
moka = { version = "0.12", features = ["future"] }
57+
dashmap = "6.0" # Concurrent HashMap
58+
redis = { version = "0.24", features = ["tokio-comp", "connection-manager"], optional = true }
59+
60+
# Retry logic
61+
tokio-retry = "0.3"
62+
4863
[dev-dependencies]
4964
testcontainers = "0.23"
5065
criterion = { version = "0.5", features = ["html_reports"] }
@@ -60,3 +75,10 @@ harness = false
6075
[[bench]]
6176
name = "transform_benchmarks"
6277
harness = false
78+
79+
[features]
80+
default = ["hash-functions", "local-cache"]
81+
hash-functions = []
82+
local-cache = []
83+
redis-cache = ["redis"]
84+
all-caches = ["local-cache", "redis-cache"]

0 commit comments

Comments
 (0)