Skip to content

Commit e3213f7

Browse files
muglugmuglug
committed
Add flags to track replica queries and synthetic requests
1 parent 87cea53 commit e3213f7

File tree

12 files changed

+205
-37
lines changed

12 files changed

+205
-37
lines changed

‎src/Index.hack‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
namespace Slack\SQLFake;
22

33
final class Index {
4-
public function __construct(public string $name, public string $type, public keyset<string> $fields) {}
4+
public function __construct(
5+
public string $name,
6+
public string $type,
7+
public keyset<string> $fields,
8+
public bool $vitess_sharding_key = false,
9+
) {}
510
}

‎src/Query/DeleteQuery.php‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ public function __construct(public string $sql) {}
1212
public function execute(AsyncMysqlConnection $conn): int {
1313
$this->fromClause as nonnull;
1414
list($database, $table_name) = Query::parseTableName($conn, $this->fromClause['name']);
15-
$data = $conn->getServer()->getTableData($database, $table_name) ?? tuple(dict[], dict[]);
15+
$table_data = $conn->getServer()->getTableData($database, $table_name) ?? tuple(dict[], dict[], keyset[]);
1616
$schema = QueryContext::getSchema($database, $table_name);
1717

1818
Metrics::trackQuery(QueryType::DELETE, $conn->getServer()->name, $table_name, $this->sql);
@@ -26,10 +26,26 @@ public function execute(AsyncMysqlConnection $conn): int {
2626
}
2727
}
2828

29-
return $this->applyWhere($conn, $data[0], $data[1], $columns, $schema?->indexes)
29+
return $this->applyWhere(
30+
$conn,
31+
$table_data[0],
32+
$table_data[1],
33+
$table_data[2] ?? keyset[],
34+
$columns,
35+
$schema?->indexes,
36+
)
3037
|> $this->applyOrderBy($conn, $$)
3138
|> $this->applyLimit($$)
32-
|> $this->applyDelete($conn, $database, $table_name, $$, $data[0], $data[1], $schema);
39+
|> $this->applyDelete(
40+
$conn,
41+
$database,
42+
$table_name,
43+
$$,
44+
$table_data[0],
45+
$table_data[1],
46+
$table_data[2] ?? keyset[],
47+
$schema,
48+
);
3349
}
3450

3551
/**
@@ -42,11 +58,14 @@ protected function applyDelete(
4258
dataset $filtered_rows,
4359
dataset $original_table,
4460
index_refs $index_refs,
61+
keyset<arraykey> $dirty_pks,
4562
?TableSchema $table_schema,
4663
): int {
4764
$rows_to_delete = Keyset\keys($filtered_rows);
48-
$remaining_rows =
49-
Dict\filter_with_key($original_table, ($row_num, $_) ==> !C\contains_key($rows_to_delete, $row_num));
65+
$remaining_rows = Dict\filter_with_key(
66+
$original_table,
67+
($row_num, $_) ==> !C\contains_key($rows_to_delete, $row_num),
68+
);
5069
$rows_affected = C\count($original_table) - C\count($remaining_rows);
5170

5271
if ($table_schema is nonnull) {
@@ -57,6 +76,7 @@ protected function applyDelete(
5776
$table_schema->vitess_sharding->keyspace,
5877
'INDEX',
5978
keyset[$table_schema->vitess_sharding->sharding_key],
79+
true,
6080
);
6181
}
6282

@@ -70,11 +90,13 @@ protected function applyDelete(
7090
$index_refs[$index_name] = $specific_index_refs;
7191
}
7292
}
93+
94+
unset($dirty_pks[$row_id]);
7395
}
7496
}
7597

7698
// write it back to the database
77-
$conn->getServer()->saveTable($database, $table_name, $remaining_rows, $index_refs);
99+
$conn->getServer()->saveTable($database, $table_name, $remaining_rows, $index_refs, $dirty_pks);
78100
return $rows_affected;
79101
}
80102
}

‎src/Query/FromClause.php‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,20 @@ public function aliasRecentExpression(string $name): void {
3737
public function process(
3838
AsyncMysqlConnection $conn,
3939
string $sql,
40-
): (dataset, index_refs, vec<Index>, dict<string, Column>) {
40+
): (dataset, index_refs, keyset<arraykey>, vec<Index>, dict<string, Column>) {
4141

4242
$data = dict[];
4343
$is_first_table = true;
4444
$index_refs = dict[];
45+
$dirty_pks = keyset[];
46+
4547
$indexes = vec[];
4648
$columns = dict[];
4749

4850
foreach ($this->tables as $table) {
4951
$schema = null;
5052
$new_index_refs = dict[];
53+
$new_dirty_pks = keyset[];
5154
$new_indexes = vec[];
5255

5356
if (Shapes::keyExists($table, 'subquery')) {
@@ -63,12 +66,17 @@ public function process(
6366
$name = $table['alias'] ?? $table_name;
6467
$schema = QueryContext::getSchema($database, $table_name);
6568
if ($schema === null && QueryContext::$strictSchemaMode) {
66-
throw
67-
new SQLFakeRuntimeException("Table $table_name not found in schema and strict mode is enabled");
69+
throw new SQLFakeRuntimeException(
70+
"Table $table_name not found in schema and strict mode is enabled",
71+
);
6872
}
6973

70-
list($res, $new_index_refs) =
71-
$conn->getServer()->getTableData($database, $table_name) ?: tuple(dict[], dict[]);
74+
$table_data = $conn->getServer()->getTableData($database, $table_name) ?:
75+
tuple(dict[], dict[], keyset[]);
76+
77+
$res = $table_data[0];
78+
$new_index_refs = $table_data[1];
79+
$new_dirty_pks = $table_data[2] ?? keyset[];
7280

7381
if (C\count($this->tables) > 1) {
7482
$new_index_refs = Dict\map_keys($new_index_refs, $k ==> $name.'.'.$k);
@@ -94,6 +102,7 @@ public function process(
94102
$prefix.$schema->vitess_sharding->keyspace,
95103
'INDEX',
96104
keyset[$prefix.$schema->vitess_sharding->sharding_key],
105+
true,
97106
);
98107
}
99108

@@ -111,6 +120,7 @@ public function process(
111120
}
112121

113122
$index_refs = Dict\merge($index_refs, $new_index_refs);
123+
$dirty_pks = Keyset\union($dirty_pks, $new_dirty_pks);
114124
}
115125

116126
$new_dataset = dict[];
@@ -156,10 +166,10 @@ public function process(
156166

157167
if ($data || !$is_first_table) {
158168
// do the join here. based on join type, pass in $data and $res to filter. and aliases
159-
list($data, $index_refs) = JoinProcessor::process(
169+
list($data, $index_refs, $dirty_pks) = JoinProcessor::process(
160170
$conn,
161-
tuple($data, $index_refs),
162-
tuple($new_dataset, $new_index_refs),
171+
tuple($data, $index_refs, keyset[]),
172+
tuple($new_dataset, $new_index_refs, $new_dirty_pks),
163173
$name,
164174
$table['join_type'],
165175
$table['join_operator'] ?? null,
@@ -180,6 +190,6 @@ public function process(
180190
}
181191
}
182192

183-
return tuple($data, $index_refs, $indexes, $columns);
193+
return tuple($data, $index_refs, $dirty_pks, $indexes, $columns);
184194
}
185195
}

‎src/Query/InsertQuery.php‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@ public function __construct(public string $table, public string $sql, public boo
1818
*/
1919
public function execute(AsyncMysqlConnection $conn): int {
2020
list($database, $table_name) = Query::parseTableName($conn, $this->table);
21-
list($table, $index_refs) = $conn->getServer()->getTableData($database, $table_name) ?? tuple(dict[], dict[]);
21+
$data = $conn->getServer()->getTableData($database, $table_name) ?? tuple(dict[], dict[], keyset[]);
22+
23+
$table = $data[0];
24+
$index_refs = $data[1];
25+
$dirty_pks = $data[2] ?? keyset[];
2226

2327
Metrics::trackQuery(QueryType::INSERT, $conn->getServer()->name, $table_name, $this->sql);
2428

@@ -65,6 +69,7 @@ public function execute(AsyncMysqlConnection $conn): int {
6569
$table_schema->vitess_sharding->keyspace,
6670
'INDEX',
6771
keyset[$table_schema->vitess_sharding->sharding_key],
72+
true,
6873
);
6974
}
7075

@@ -113,6 +118,7 @@ public function execute(AsyncMysqlConnection $conn): int {
113118
dict[$row_id => $existing_row],
114119
$table,
115120
$index_refs,
121+
$dirty_pks,
116122
$this->updateExpressions,
117123
$table_schema,
118124
$row,
@@ -135,12 +141,16 @@ public function execute(AsyncMysqlConnection $conn): int {
135141
$index_refs[$index_name] = $specific_index_refs;
136142
}
137143

144+
if (QueryContext::$inRequest) {
145+
$dirty_pks[] = $primary_key;
146+
}
147+
138148
$table[$primary_key] = $row;
139149
$rows_affected++;
140150
}
141151

142152
// write it back to the database
143-
$conn->getServer()->saveTable($database, $table_name, $table, $index_refs);
153+
$conn->getServer()->saveTable($database, $table_name, $table, $index_refs, $dirty_pks);
144154
return $rows_affected;
145155
}
146156
}

‎src/Query/JoinProcessor.php‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ public static function process(
6262
$left_mappings = dict[];
6363
$right_mappings = dict[];
6464

65+
$dirty_pks = keyset[];
66+
6567
switch ($join_type) {
6668
case JoinType::JOIN:
6769
case JoinType::STRAIGHT:
@@ -77,6 +79,9 @@ public static function process(
7779
$left_mappings[$left_row_id][] = $insert_id;
7880
$right_mappings[$right_row_id] ??= keyset[];
7981
$right_mappings[$right_row_id][] = $insert_id;
82+
if (isset($left_dataset[2][$left_row_id]) || isset($right_dataset[2][$right_row_id])) {
83+
$dirty_pks[] = $insert_id;
84+
}
8085
}
8186
}
8287
}
@@ -104,6 +109,9 @@ public static function process(
104109
$right_mappings[$right_row_id] ??= keyset[];
105110
$right_mappings[$right_row_id][] = $insert_id;
106111
$any_match = true;
112+
if (isset($left_dataset[2][$left_row_id]) || isset($right_dataset[2][$right_row_id])) {
113+
$dirty_pks[] = $insert_id;
114+
}
107115
}
108116
}
109117

@@ -120,6 +128,10 @@ public static function process(
120128
$insert_id = C\count($out) - 1;
121129
$left_mappings[$left_row_id] ??= keyset[];
122130
$left_mappings[$left_row_id][] = $insert_id;
131+
132+
if (isset($left_dataset[2][$left_row_id])) {
133+
$dirty_pks[] = $insert_id;
134+
}
123135
}
124136
}
125137
break;
@@ -146,6 +158,9 @@ public static function process(
146158
$left_mappings[$left_row_id][] = $insert_id;
147159
$right_mappings[$right_row_id] ??= keyset[];
148160
$right_mappings[$right_row_id][] = $insert_id;
161+
if (isset($left_dataset[2][$left_row_id]) || isset($right_dataset[2][$right_row_id])) {
162+
$dirty_pks[] = $insert_id;
163+
}
149164
}
150165
}
151166

@@ -164,6 +179,9 @@ public static function process(
164179
$left_mappings[$left_row_id][] = $insert_id;
165180
$right_mappings[$right_row_id] ??= keyset[];
166181
$right_mappings[$right_row_id][] = $insert_id;
182+
if (isset($left_dataset[2][$left_row_id]) || isset($right_dataset[2][$right_row_id])) {
183+
$dirty_pks[] = $insert_id;
184+
}
167185
}
168186
}
169187
break;
@@ -183,6 +201,9 @@ public static function process(
183201
$left_mappings[$left_row_id][] = $insert_id;
184202
$right_mappings[$right_row_id] ??= keyset[];
185203
$right_mappings[$right_row_id][] = $insert_id;
204+
if (isset($left_dataset[2][$left_row_id]) || isset($right_dataset[2][$right_row_id])) {
205+
$dirty_pks[] = $insert_id;
206+
}
186207
}
187208
}
188209
}
@@ -198,7 +219,7 @@ public static function process(
198219
$right_indexes,
199220
);
200221

201-
return tuple(dict($out), $index_refs);
222+
return tuple(dict($out), $index_refs, $dirty_pks);
202223
}
203224

204225
/**
@@ -308,6 +329,7 @@ private static function processHashJoin(
308329

309330
$left_mappings = dict[];
310331
$right_mappings = dict[];
332+
$dirty_pks = keyset[];
311333

312334
switch ($join_type) {
313335
case JoinType::JOIN:
@@ -322,6 +344,9 @@ private static function processHashJoin(
322344
$left_mappings[$left_row_id][] = $insert_id;
323345
$right_mappings[$k] ??= keyset[];
324346
$right_mappings[$k][] = $insert_id;
347+
if (isset($left_dataset[2][$left_row_id]) || isset($right_dataset[2][$k])) {
348+
$dirty_pks[] = $insert_id;
349+
}
325350
}
326351
}
327352
break;
@@ -348,6 +373,9 @@ private static function processHashJoin(
348373
$right_mappings[$k] ??= keyset[];
349374
$right_mappings[$k][] = $insert_id;
350375
$any_match = true;
376+
if (isset($left_dataset[2][$left_row_id]) || isset($right_dataset[2][$k])) {
377+
$dirty_pks[] = $insert_id;
378+
}
351379
}
352380
}
353381

@@ -364,6 +392,9 @@ private static function processHashJoin(
364392
$insert_id = C\count($out) - 1;
365393
$left_mappings[$left_row_id] ??= keyset[];
366394
$left_mappings[$left_row_id][] = $insert_id;
395+
if (isset($left_dataset[2][$left_row_id])) {
396+
$dirty_pks[] = $insert_id;
397+
}
367398
}
368399
}
369400
break;
@@ -380,7 +411,7 @@ private static function processHashJoin(
380411
$right_indexes,
381412
);
382413

383-
return tuple(dict($out), $index_refs);
414+
return tuple(dict($out), $index_refs, $dirty_pks);
384415
}
385416

386417
private static function getIndexRefsFromMappings(

0 commit comments

Comments
 (0)