Skip to content

Commit 5c3cb05

Browse files
jxomjxom
committed
feat: read wide unsigned integers
1 parent b7c3ec5 commit 5c3cb05

5 files changed

Lines changed: 269 additions & 0 deletions

File tree

‎doc/pg_clickhouse.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -942,7 +942,9 @@ the PostgreSQL column when importing columns; additional types may be used in
942942
| Int8 | smallint | |
943943
| JSON | jsonb, json | |
944944
| String | text, bytea | |
945+
| UInt128 | numeric | Read-only |
945946
| UInt16 | integer | |
947+
| UInt256 | numeric | Read-only |
946948
| UInt32 | bigint | |
947949
| UInt64 | bigint | Errors on values > BIGINT max |
948950
| UInt8 | smallint | |

‎src/binary/decode.c‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ ch_kind_to_pg_oid(const chc_type* type) {
8585
}
8686

8787
switch (kind) {
88+
case CHC_UINT128:
89+
case CHC_UINT256:
90+
return NUMERICOID;
8891
case CHC_VOID:
8992
case CHC_NOTHING:
9093
return InvalidOid;
@@ -115,6 +118,53 @@ read_value(
115118
bool* is_null
116119
);
117120

121+
/* Convert little-endian base-256 bytes without host-sized wide integers. */
122+
static Datum
123+
read_wide_uint(const chc_column* col, uint64_t row) {
124+
size_t width;
125+
const uint8_t* data = (const uint8_t*)chc_column_fixed_data(col, &width);
126+
uint8_t magnitude[32];
127+
char reversed[80];
128+
char text[80];
129+
size_t digits = 0;
130+
bool nonzero;
131+
132+
if (width != 16 && width != 32) {
133+
ereport(
134+
ERROR,
135+
errcode(ERRCODE_FDW_INVALID_DATA_TYPE),
136+
errmsg("pg_clickhouse: unexpected wide unsigned integer width %zu", width)
137+
);
138+
}
139+
140+
memcpy(magnitude, data + (size_t)row * width, width);
141+
do {
142+
uint16_t remainder = 0;
143+
144+
nonzero = false;
145+
for (size_t i = width; i-- > 0;) {
146+
uint16_t value = (uint16_t)((remainder << 8) | magnitude[i]);
147+
148+
magnitude[i] = (uint8_t)(value / 10);
149+
remainder = value % 10;
150+
if (magnitude[i] != 0) {
151+
nonzero = true;
152+
}
153+
}
154+
Assert(digits < sizeof(reversed));
155+
reversed[digits++] = (char)('0' + remainder);
156+
} while (nonzero);
157+
158+
for (size_t i = 0; i < digits; i++) {
159+
text[i] = reversed[digits - i - 1];
160+
}
161+
text[digits] = '\0';
162+
163+
return DirectFunctionCall3(
164+
numeric_in, CStringGetDatum(text), ObjectIdGetDatum(0), Int32GetDatum(-1)
165+
);
166+
}
167+
118168
/*
119169
* Format a ClickHouse Decimal (two's-complement signed integer in LE bytes of
120170
* width 4/8/16/32 for Decimal32/64/128/256, with `scale` fractional digits
@@ -600,6 +650,10 @@ read_value(
600650
*valtype = INT8OID;
601651
return Int64GetDatum((int64)v);
602652
}
653+
case CHC_UINT128:
654+
case CHC_UINT256:
655+
*valtype = NUMERICOID;
656+
return read_wide_uint(col, row);
603657
case CHC_FLOAT32:
604658
*valtype = FLOAT4OID;
605659
return Float4GetDatum(

‎src/pglink.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1396,6 +1396,8 @@ binary_finalize_insert(void* istate) {
13961396
('UInt32', 'bigint', ''),
13971397
('Int64', 'bigint', ''),
13981398
('UInt64', 'bigint', 'Errors on values > BIGINT max'),
1399+
('UInt128', 'numeric', 'Read-only'),
1400+
('UInt256', 'numeric', 'Read-only'),
13991401
('Float32', 'real', ''),
14001402
('Float64', 'double precision', ''),
14011403
('Decimal', 'numeric', ''),
@@ -1424,6 +1426,8 @@ static char* str_types_map[][2] = {
14241426
{ "UInt32", "INT8" },
14251427
{ "Int64", "INT8" },
14261428
{ "UInt64", "INT8" },
1429+
{ "UInt128", "NUMERIC" },
1430+
{ "UInt256", "NUMERIC" },
14271431
{ "Float32", "REAL" },
14281432
{ "Float64", "DOUBLE PRECISION" },
14291433
{ "Decimal", "NUMERIC" },

‎test/expected/uint256.out‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
CREATE SERVER uint256_http_server FOREIGN DATA WRAPPER clickhouse_fdw
2+
OPTIONS (dbname 'uint256_test', driver 'http');
3+
CREATE SERVER uint256_binary_server FOREIGN DATA WRAPPER clickhouse_fdw
4+
OPTIONS (dbname 'uint256_test', driver 'binary');
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER uint256_http_server;
6+
CREATE USER MAPPING FOR CURRENT_USER SERVER uint256_binary_server;
7+
CREATE SCHEMA uint256_http;
8+
CREATE SCHEMA uint256_binary;
9+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS uint256_test');
10+
clickhouse_raw_query
11+
----------------------
12+
13+
(1 row)
14+
15+
SELECT clickhouse_raw_query('CREATE DATABASE uint256_test');
16+
clickhouse_raw_query
17+
----------------------
18+
19+
(1 row)
20+
21+
SELECT clickhouse_raw_query($$
22+
CREATE TABLE uint256_test.wide_values (
23+
id UInt8,
24+
u128 UInt128,
25+
u256 UInt256,
26+
n256 Nullable(UInt256),
27+
a256 Array(UInt256)
28+
) ENGINE = MergeTree ORDER BY id
29+
$$);
30+
clickhouse_raw_query
31+
----------------------
32+
33+
(1 row)
34+
35+
SELECT clickhouse_raw_query($$
36+
INSERT INTO uint256_test.wide_values VALUES
37+
(
38+
0,
39+
toUInt128(0),
40+
toUInt256(0),
41+
NULL,
42+
[]
43+
),
44+
(
45+
1,
46+
toUInt128('170141183460469231731687303715884105728'),
47+
toUInt256('57896044618658097711785492504343953926634992332820282019728792003956564819968'),
48+
toUInt256('57896044618658097711785492504343953926634992332820282019728792003956564819968'),
49+
[
50+
toUInt256(0),
51+
toUInt256('57896044618658097711785492504343953926634992332820282019728792003956564819968')
52+
]
53+
),
54+
(
55+
2,
56+
toUInt128('340282366920938463463374607431768211455'),
57+
toUInt256('115792089237316195423570985008687907853269984665640564039457584007913129639935'),
58+
toUInt256('115792089237316195423570985008687907853269984665640564039457584007913129639935'),
59+
[toUInt256('115792089237316195423570985008687907853269984665640564039457584007913129639935')]
60+
)
61+
$$);
62+
clickhouse_raw_query
63+
----------------------
64+
65+
(1 row)
66+
67+
IMPORT FOREIGN SCHEMA uint256_test LIMIT TO (wide_values)
68+
FROM SERVER uint256_http_server INTO uint256_http;
69+
IMPORT FOREIGN SCHEMA uint256_test LIMIT TO (wide_values)
70+
FROM SERVER uint256_binary_server INTO uint256_binary;
71+
SELECT n.nspname AS schema_name,
72+
a.attname AS column_name,
73+
format_type(a.atttypid, a.atttypmod) AS postgres_type
74+
FROM pg_attribute AS a
75+
JOIN pg_class AS c ON c.oid = a.attrelid
76+
JOIN pg_namespace AS n ON n.oid = c.relnamespace
77+
WHERE n.nspname IN ('uint256_http', 'uint256_binary')
78+
AND c.relname = 'wide_values'
79+
AND a.attnum > 0
80+
AND NOT a.attisdropped
81+
AND a.attname <> 'id'
82+
ORDER BY schema_name, column_name;
83+
schema_name | column_name | postgres_type
84+
----------------+-------------+---------------
85+
uint256_binary | a256 | numeric[]
86+
uint256_binary | n256 | numeric
87+
uint256_binary | u128 | numeric
88+
uint256_binary | u256 | numeric
89+
uint256_http | a256 | numeric[]
90+
uint256_http | n256 | numeric
91+
uint256_http | u128 | numeric
92+
uint256_http | u256 | numeric
93+
(8 rows)
94+
95+
SELECT id, u128, u256, n256, a256
96+
FROM uint256_http.wide_values
97+
ORDER BY id;
98+
id | u128 | u256 | n256 | a256
99+
----+-----------------------------------------+--------------------------------------------------------------------------------+--------------------------------------------------------------------------------+-----------------------------------------------------------------------------------
100+
0 | 0 | 0 | | {}
101+
1 | 170141183460469231731687303715884105728 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 | {0,57896044618658097711785492504343953926634992332820282019728792003956564819968}
102+
2 | 340282366920938463463374607431768211455 | 115792089237316195423570985008687907853269984665640564039457584007913129639935 | 115792089237316195423570985008687907853269984665640564039457584007913129639935 | {115792089237316195423570985008687907853269984665640564039457584007913129639935}
103+
(3 rows)
104+
105+
SELECT id, u128, u256, n256, a256
106+
FROM uint256_binary.wide_values
107+
ORDER BY id;
108+
id | u128 | u256 | n256 | a256
109+
----+-----------------------------------------+--------------------------------------------------------------------------------+--------------------------------------------------------------------------------+-----------------------------------------------------------------------------------
110+
0 | 0 | 0 | | {}
111+
1 | 170141183460469231731687303715884105728 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 | {0,57896044618658097711785492504343953926634992332820282019728792003956564819968}
112+
2 | 340282366920938463463374607431768211455 | 115792089237316195423570985008687907853269984665640564039457584007913129639935 | 115792089237316195423570985008687907853269984665640564039457584007913129639935 | {115792089237316195423570985008687907853269984665640564039457584007913129639935}
113+
(3 rows)
114+
115+
DROP USER MAPPING FOR CURRENT_USER SERVER uint256_http_server;
116+
DROP USER MAPPING FOR CURRENT_USER SERVER uint256_binary_server;
117+
SELECT clickhouse_raw_query('DROP DATABASE uint256_test');
118+
clickhouse_raw_query
119+
----------------------
120+
121+
(1 row)
122+
123+
DROP SERVER uint256_http_server CASCADE;
124+
NOTICE: drop cascades to foreign table uint256_http.wide_values
125+
DROP SERVER uint256_binary_server CASCADE;
126+
NOTICE: drop cascades to foreign table uint256_binary.wide_values
127+
DROP SCHEMA uint256_http;
128+
DROP SCHEMA uint256_binary;

‎test/sql/uint256.sql‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
CREATE SERVER uint256_http_server FOREIGN DATA WRAPPER clickhouse_fdw
2+
OPTIONS (dbname 'uint256_test', driver 'http');
3+
CREATE SERVER uint256_binary_server FOREIGN DATA WRAPPER clickhouse_fdw
4+
OPTIONS (dbname 'uint256_test', driver 'binary');
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER uint256_http_server;
6+
CREATE USER MAPPING FOR CURRENT_USER SERVER uint256_binary_server;
7+
CREATE SCHEMA uint256_http;
8+
CREATE SCHEMA uint256_binary;
9+
10+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS uint256_test');
11+
SELECT clickhouse_raw_query('CREATE DATABASE uint256_test');
12+
SELECT clickhouse_raw_query($$
13+
CREATE TABLE uint256_test.wide_values (
14+
id UInt8,
15+
u128 UInt128,
16+
u256 UInt256,
17+
n256 Nullable(UInt256),
18+
a256 Array(UInt256)
19+
) ENGINE = MergeTree ORDER BY id
20+
$$);
21+
SELECT clickhouse_raw_query($$
22+
INSERT INTO uint256_test.wide_values VALUES
23+
(
24+
0,
25+
toUInt128(0),
26+
toUInt256(0),
27+
NULL,
28+
[]
29+
),
30+
(
31+
1,
32+
toUInt128('170141183460469231731687303715884105728'),
33+
toUInt256('57896044618658097711785492504343953926634992332820282019728792003956564819968'),
34+
toUInt256('57896044618658097711785492504343953926634992332820282019728792003956564819968'),
35+
[
36+
toUInt256(0),
37+
toUInt256('57896044618658097711785492504343953926634992332820282019728792003956564819968')
38+
]
39+
),
40+
(
41+
2,
42+
toUInt128('340282366920938463463374607431768211455'),
43+
toUInt256('115792089237316195423570985008687907853269984665640564039457584007913129639935'),
44+
toUInt256('115792089237316195423570985008687907853269984665640564039457584007913129639935'),
45+
[toUInt256('115792089237316195423570985008687907853269984665640564039457584007913129639935')]
46+
)
47+
$$);
48+
49+
IMPORT FOREIGN SCHEMA uint256_test LIMIT TO (wide_values)
50+
FROM SERVER uint256_http_server INTO uint256_http;
51+
IMPORT FOREIGN SCHEMA uint256_test LIMIT TO (wide_values)
52+
FROM SERVER uint256_binary_server INTO uint256_binary;
53+
54+
SELECT n.nspname AS schema_name,
55+
a.attname AS column_name,
56+
format_type(a.atttypid, a.atttypmod) AS postgres_type
57+
FROM pg_attribute AS a
58+
JOIN pg_class AS c ON c.oid = a.attrelid
59+
JOIN pg_namespace AS n ON n.oid = c.relnamespace
60+
WHERE n.nspname IN ('uint256_http', 'uint256_binary')
61+
AND c.relname = 'wide_values'
62+
AND a.attnum > 0
63+
AND NOT a.attisdropped
64+
AND a.attname <> 'id'
65+
ORDER BY schema_name, column_name;
66+
67+
SELECT id, u128, u256, n256, a256
68+
FROM uint256_http.wide_values
69+
ORDER BY id;
70+
71+
SELECT id, u128, u256, n256, a256
72+
FROM uint256_binary.wide_values
73+
ORDER BY id;
74+
75+
DROP USER MAPPING FOR CURRENT_USER SERVER uint256_http_server;
76+
DROP USER MAPPING FOR CURRENT_USER SERVER uint256_binary_server;
77+
SELECT clickhouse_raw_query('DROP DATABASE uint256_test');
78+
DROP SERVER uint256_http_server CASCADE;
79+
DROP SERVER uint256_binary_server CASCADE;
80+
DROP SCHEMA uint256_http;
81+
DROP SCHEMA uint256_binary;

0 commit comments

Comments
 (0)