-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcompatibility.test.ts
More file actions
227 lines (192 loc) · 7.8 KB
/
compatibility.test.ts
File metadata and controls
227 lines (192 loc) · 7.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
import { describe, it, expect } from 'vitest';
import { engineCatalogRules, engineReadRules, pairOverrides, ENGINES, CATALOGS } from './compatibility';
import type { CatalogId } from './compatibility';
describe('data completeness', () => {
it('covers all engines', () => {
for (const engine of ENGINES) {
expect(engineCatalogRules).toHaveProperty(engine);
}
});
it('covers all catalogs for every engine', () => {
for (const engine of ENGINES) {
for (const catalog of CATALOGS) {
expect(engineCatalogRules[engine]).toHaveProperty(catalog);
}
}
});
});
describe('data quality invariants', () => {
it('partial entries always have at least one limitation', () => {
for (const engine of ENGINES) {
for (const catalog of CATALOGS) {
const entry = engineCatalogRules[engine][catalog];
if (entry.support === 'partial') {
expect(
entry.limitations.length,
`${engine}→${catalog} is partial but has no limitations`
).toBeGreaterThan(0);
}
}
}
});
it('no empty limitation strings', () => {
for (const engine of ENGINES) {
for (const catalog of CATALOGS) {
for (const lim of engineCatalogRules[engine][catalog].limitations) {
expect(lim.trim().length, `empty limitation in ${engine}→${catalog}`).toBeGreaterThan(0);
}
}
}
});
it('pairOverrides keys are valid PairKeys', () => {
const validEngines = new Set<string>(ENGINES);
for (const key of Object.keys(pairOverrides)) {
const [write, read] = key.split('__');
expect(validEngines.has(write), `invalid write engine: ${write}`).toBe(true);
expect(validEngines.has(read), `invalid read engine: ${read}`).toBe(true);
}
});
it('pairOverrides partial entries have limitations', () => {
for (const [key, rule] of Object.entries(pairOverrides)) {
if (!rule) continue;
for (const catalog of CATALOGS) {
const entry = rule[catalog];
if (entry.support === 'partial') {
expect(
entry.limitations.length,
`${key}→${catalog} is partial but has no limitations`
).toBeGreaterThan(0);
}
}
}
});
});
describe('engineReadRules', () => {
it('only references valid engine ids', () => {
const validEngines = new Set<string>(ENGINES);
for (const engine of Object.keys(engineReadRules)) {
expect(validEngines.has(engine), `invalid engine in engineReadRules: ${engine}`).toBe(true);
}
});
it('partial read entries have at least one limitation', () => {
for (const [engine, overrides] of Object.entries(engineReadRules)) {
if (!overrides) continue;
for (const [catalog, entry] of Object.entries(overrides)) {
if (!entry) continue;
if (entry.support === 'partial') {
expect(
entry.limitations.length,
`engineReadRules ${engine}→${catalog} is partial but has no limitations`
).toBeGreaterThan(0);
}
}
}
});
it('databricks can read from REST catalogs', () => {
expect(engineReadRules.databricks?.rest?.support).toBe('partial');
});
it('databricks can read from Glue', () => {
expect(engineReadRules.databricks?.glue?.support).toBe('partial');
});
it('databricks can read from S3 Tables', () => {
expect(engineReadRules.databricks?.s3tables?.support).toBe('partial');
});
it('redshift has full write support for Glue', () => {
expect(engineCatalogRules.redshift.glue.support).toBe('full');
});
it('redshift has full write support for S3 Tables', () => {
expect(engineCatalogRules.redshift.s3tables.support).toBe('full');
});
});
describe('known facts', () => {
it('snowflake has full glue support via CLD', () => {
expect(engineCatalogRules.snowflake.glue.support).toBe('full');
});
it('snowflake has full unity support via CLD', () => {
expect(engineCatalogRules.snowflake.unity.support).toBe('full');
});
it('databricks has full unity catalog support', () => {
expect(engineCatalogRules.databricks.unity.support).toBe('full');
});
it('databricks cannot write iceberg to glue', () => {
expect(engineCatalogRules.databricks.glue.support).toBe('none');
});
it('athena has full glue support', () => {
expect(engineCatalogRules.athena.glue.support).toBe('full');
});
it('athena has full S3 Tables support', () => {
expect(engineCatalogRules.athena.s3tables.support).toBe('full');
});
it('trino has full support for glue, rest, hive, s3tables', () => {
const fullCatalogs: CatalogId[] = ['glue', 'rest', 'hive', 's3tables'];
for (const catalog of fullCatalogs) {
expect(engineCatalogRules.trino[catalog].support, `trino→${catalog}`).toBe('full');
}
});
it('trino has partial unity support (write not GA in OSS Trino)', () => {
expect(engineCatalogRules.trino.unity.support).toBe('partial');
});
it('snowflake has full s3tables support via CLD', () => {
expect(engineCatalogRules.snowflake.s3tables.support).toBe('full');
});
it('duckdb has partial s3tables support via REST catalog', () => {
expect(engineCatalogRules.duckdb.s3tables.support).toBe('partial');
});
it('postgres has no support for any catalog', () => {
for (const catalog of CATALOGS) {
expect(engineCatalogRules.postgres[catalog].support).toBe('none');
}
});
it('ducklake is none for engines without a DuckLake client', () => {
const noClient = ENGINES.filter(e => !(['duckdb', 'trino', 'databricks'] as string[]).includes(e));
for (const engine of noClient) {
expect(engineCatalogRules[engine].ducklake.support, `${engine} should have no ducklake support`).toBe('none');
}
});
it('ducklake is partial for engines with a DuckLake client', () => {
for (const engine of ['duckdb', 'trino', 'databricks'] as const) {
expect(engineCatalogRules[engine].ducklake.support, `${engine} should have partial ducklake support`).toBe('partial');
}
});
it('s3tables is none by default for bigquery and postgres', () => {
expect(engineCatalogRules.bigquery.s3tables.support).toBe('none');
expect(engineCatalogRules.postgres.s3tables.support).toBe('none');
});
it('duckdb+duckdb override has partial ducklake support', () => {
const override = pairOverrides['duckdb__duckdb'];
expect(override).toBeDefined();
expect(override?.ducklake.support).toBe('partial');
});
});
describe('vendor_bridge catalog', () => {
it('vendor_bridge is the last catalog in CATALOGS', () => {
expect(CATALOGS[CATALOGS.length - 1]).toBe('vendor_bridge');
});
it('snowflake has full write support for vendor_bridge', () => {
expect(engineCatalogRules.snowflake.vendor_bridge.support).toBe('full');
});
it('all non-snowflake engines have none write support for vendor_bridge', () => {
const nonSnowflake = ENGINES.filter(e => e !== 'snowflake');
for (const engine of nonSnowflake) {
expect(
engineCatalogRules[engine].vendor_bridge?.support,
`${engine} write vendor_bridge should be none`
).toBe('none');
}
});
it('databricks can read vendor_bridge (Catalog Federation)', () => {
expect(engineReadRules.databricks?.vendor_bridge?.support).toBe('partial');
});
it('databricks vendor_bridge read entry has limitations', () => {
const limitations = engineReadRules.databricks?.vendor_bridge?.limitations ?? [];
expect(limitations.length).toBeGreaterThan(0);
});
it('databricks vendor_bridge read entry has sourceUrls', () => {
const entry = engineReadRules.databricks?.vendor_bridge;
expect(entry?.sourceUrls).toBeDefined();
expect(entry?.sourceUrls?.length).toBeGreaterThan(0);
});
it('snowflake cannot read vendor_bridge (not a Catalog Federation consumer)', () => {
expect(engineReadRules.snowflake?.vendor_bridge?.support).toBe('none');
});
});