Skip to content

Commit bfe6b89

Browse files
committed
docs(clickhouse): remove deprecated unnecessary setting, document other setting
1 parent db87624 commit bfe6b89

File tree

1 file changed

+7
-4
lines changed

1 file changed

+7
-4
lines changed

backend/pkg/commons/db/clickhouse.go

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,11 @@ func MustInitClickhouseNative(writer *types.DatabaseConfig) ch.Conn {
5959
log.Debugf("CH NATIVE WRITER: "+s, p...)
6060
},
6161
Settings: ch.Settings{
62-
"deduplicate_blocks_in_dependent_materialized_views": "1",
63-
"update_insert_deduplication_token_in_dependent_materialized_views": "1",
62+
// https://clickhouse.com/docs/operations/settings/settings#deduplicate_blocks_in_dependent_materialized_views
63+
// when an insert to a table with dependent materialized views fails during the materialized view processing, said table will still retain the rows inserted.
64+
// this setting ensure that when the insert query gets retried by our code, the attempt doesn't get filtered out by the target table doing de-duplication,
65+
// and instead ensures that all dependent materialized views receive the data anyways
66+
"deduplicate_blocks_in_dependent_materialized_views": "1",
6467
// trade of higher background overhead for lower query specific memory pressure
6568
// reduces memory usage by 20-30% in our prod insert queries
6669
"optimize_on_insert": "0",
@@ -115,8 +118,8 @@ func UltraFastDumpToClickhouse[T UltraFastClickhouseStruct](data T, target_table
115118
abortCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
116119
defer cancel()
117120
ctx := ch.Context(abortCtx, ch.WithSettings(ch.Settings{
118-
"insert_deduplication_token": insert_uuid, // 重复数据插入时,会根据这个字段进行去重
119-
"insert_deduplicate": true,
121+
"insert_deduplication_token": insert_uuid, // this is used by tables & materialized views to correctly handle retries of inserts (skipping them if they already have the resulting rows, for example)
122+
"insert_deduplicate": true, // enforce deduplication to be done by tables & materialized views
120123
}), ch.WithLogs(func(l *ch.Log) {
121124
log.Debugf("CH NATIVE WRITER: %s", l.Text)
122125
}),

0 commit comments

Comments
 (0)