Skip to content

Commit 75f9b32

Browse files
YamingPeiYamingPei
authored
feat(taosdump): support decimal/blob data types and stmt2 data import (#34636)
Closes: - [6622691504](https://project.feishu.cn/taosdata_td/feature/detail/6622691504) - [6835134117](https://project.feishu.cn/taosdata_td/feature/detail/6835134117) - [6834892123](https://project.feishu.cn/taosdata_td/feature/detail/6834892123) Changes: * feat(taosdump): supports dump out decimal type data * feat(taosdump): support dump in decimal type data * feat(taosdump): support stmt v2 for data import * feat(taosdump): support dump in/out blob type data * fix(taosdump): fix the issue of reading the length of strings in avro format * test(taosdump): add test cases for decimal/blog data type * test(taosdump): optimize test cases * feat(taosdump): when binding data, do not provide table name
1 parent 3b7e884 commit 75f9b32

File tree

3 files changed

+323
-137
lines changed

3 files changed

+323
-137
lines changed

‎test/cases/81-Tools/04-Taosdump/test_taosdump_basic.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,28 +33,28 @@ def do_taosdump_test_basic(self):
3333

3434
tdSql.execute("use db")
3535
tdSql.execute(
36-
"create table st(ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED) tags(n1 INT, w2 BOOL, t3 TINYINT, t4 SMALLINT, t5 BIGINT, t6 FLOAT, t7 DOUBLE, t8 TIMESTAMP, t9 BINARY(10), t10 NCHAR(10), t11 TINYINT UNSIGNED, t12 SMALLINT UNSIGNED, t13 INT UNSIGNED, t14 BIGINT UNSIGNED)"
36+
"create table st(ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED, c15 DECIMAL(30, 16), c16 BLOB) tags(n1 INT, w2 BOOL, t3 TINYINT, t4 SMALLINT, t5 BIGINT, t6 FLOAT, t7 DOUBLE, t8 TIMESTAMP, t9 BINARY(10), t10 NCHAR(10), t11 TINYINT UNSIGNED, t12 SMALLINT UNSIGNED, t13 INT UNSIGNED, t14 BIGINT UNSIGNED)"
3737
)
3838
tdSql.execute(
3939
"create table t1 using st tags(1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)"
4040
)
4141
tdSql.execute(
42-
"insert into t1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)"
42+
"insert into t1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1, 12345678901234.1234567890123456, '\\x393866343633')"
4343
)
4444
tdSql.execute(
4545
"create table t2 using st tags(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
4646
)
4747
tdSql.execute(
48-
"insert into t2 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
48+
"insert into t2 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
4949
)
5050
tdSql.execute(
51-
"create table db.nt1 (ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED)"
51+
"create table db.nt1 (ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED, c15 DECIMAL(30, 16), c16 BLOB)"
5252
)
5353
tdSql.execute(
54-
"insert into nt1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)"
54+
"insert into nt1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1, 12345678901234.1234567890123456, '\\x393866343633')"
5555
)
5656
tdSql.execute(
57-
"insert into nt1 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
57+
"insert into nt1 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
5858
)
5959

6060
# virtual table

‎test/cases/81-Tools/04-Taosdump/test_taosdump_datatypes.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,135 @@ def do_taosdump_type_binary(self, mode):
175175

176176
print("do type binary ........................ [passed]")
177177

178+
def do_taosdump_type_decimal(self, mode):
179+
if mode == "-Z 'WebSocket'":
180+
tdLog.debug("WebSocket mode does not support decimal type, skip this test")
181+
return
182+
183+
tdSql.execute("drop database if exists db")
184+
tdSql.execute("create database db keep 3649 ")
185+
186+
tdSql.execute("create table db.st(ts timestamp, c1 DECIMAL(30, 16), c2 DECIMAL(16, 10)) tags(dtag DOUBLE)")
187+
tdSql.execute("create table db.t1 using db.st tags(98765.123456)")
188+
tdSql.execute("insert into db.t1 values(1640000000000, '98765432109876.1234567890123456', '56789.1234567890')")
189+
tdSql.execute("create table db.t2 using db.st tags(NULL)")
190+
tdSql.execute("insert into db.t2 values(1640000000000, NULL, NULL)")
191+
192+
if not os.path.exists(self.tmpdir):
193+
os.makedirs(self.tmpdir)
194+
else:
195+
196+
os.system("rm -rf %s" % self.tmpdir)
197+
os.makedirs(self.tmpdir)
198+
199+
os.system(f"%s {mode} -D db -o %s" % (self.binPath, self.tmpdir))
200+
tdSql.execute("drop database db")
201+
202+
os.system(f"%s {mode} -i %s" % (self.binPath, self.tmpdir))
203+
204+
tdSql.query("show databases")
205+
dbresult = tdSql.queryResult
206+
assert any(row[0] == "db" for row in dbresult)
207+
208+
tdSql.query("show db.stables")
209+
tdSql.checkRows(1)
210+
tdSql.checkData(0, 0, "st")
211+
212+
tdSql.query("show db.tables")
213+
tdSql.checkRows(2)
214+
dbresult = tdSql.queryResult
215+
print(dbresult)
216+
for i in range(len(dbresult)):
217+
assert dbresult[i][0] in ("t1", "t2")
218+
219+
tdSql.query("select distinct(dtag) from db.st where tbname = 't1'")
220+
tdSql.checkRows(1)
221+
tdSql.checkData(0, 0, 98765.123456)
222+
223+
tdSql.query("select distinct(dtag) from db.st where tbname = 't2'")
224+
tdSql.checkRows(1)
225+
tdSql.checkData(0, 0, None)
226+
227+
tdSql.query("select * from db.st where dtag = 98765.123456")
228+
tdSql.checkRows(1)
229+
tdSql.checkData(0, 1, "98765432109876.1234567890123456")
230+
tdSql.checkData(0, 2, "56789.1234567890")
231+
232+
tdSql.query("select * from db.st where dtag is null")
233+
tdSql.checkRows(1)
234+
tdSql.checkData(0, 1, None)
235+
tdSql.checkData(0, 2, None)
236+
237+
print("do type decimal ........................ [passed]")
238+
239+
240+
def do_taosdump_type_blob(self, mode):
241+
if mode == "-Z 'WebSocket'":
242+
tdLog.debug("WebSocket mode does not support blob type, skip this test")
243+
return
244+
245+
tdSql.execute("drop database if exists db")
246+
tdSql.execute("create database db keep 3649 ")
247+
248+
tdSql.execute("create table db.st(ts timestamp, c1 BLOB) tags(ntag INT)")
249+
tdSql.execute("create table db.t1 using db.st tags(1)")
250+
tdSql.execute("insert into db.t1 values(1640000000000, 'abc')")
251+
tdSql.execute("insert into db.t1 values(1640000000001, '\\x61620063')")
252+
tdSql.execute("create table db.t2 using db.st tags(NULL)")
253+
tdSql.execute("insert into db.t2 values(1640000000000, NULL)")
254+
255+
if not os.path.exists(self.tmpdir):
256+
os.makedirs(self.tmpdir)
257+
else:
258+
os.system("rm -rf %s" % self.tmpdir)
259+
os.makedirs(self.tmpdir)
260+
261+
os.system(f"%s {mode} -D db -o %s" % (self.binPath, self.tmpdir))
262+
tdSql.execute("drop database db")
263+
264+
os.system(f"%s {mode} -i %s" % (self.binPath, self.tmpdir))
265+
266+
tdSql.query("show databases")
267+
dbresult = tdSql.queryResult
268+
assert any(row[0] == "db" for row in dbresult)
269+
270+
tdSql.query("show db.stables")
271+
tdSql.checkRows(1)
272+
tdSql.checkData(0, 0, "st")
273+
274+
tdSql.query("show db.tables")
275+
tdSql.checkRows(2)
276+
dbresult = tdSql.queryResult
277+
for i in range(len(dbresult)):
278+
assert dbresult[i][0] in ("t1", "t2")
279+
280+
# verify tag
281+
tdSql.query("select distinct(ntag) from db.st where tbname = 't1'")
282+
tdSql.checkRows(1)
283+
tdSql.checkData(0, 0, 1)
284+
285+
tdSql.query("select distinct(ntag) from db.st where tbname = 't2'")
286+
tdSql.checkRows(1)
287+
tdSql.checkData(0, 0, None)
288+
289+
# row with simple strings
290+
tdSql.query("select * from db.st where ntag = 1 order by ts limit 1")
291+
tdSql.checkRows(1)
292+
tdSql.checkData(0, 1, b"abc")
293+
294+
# row with embedded 0-byte
295+
tdSql.query("select c1 from db.st where ntag = 1 order by ts limit 1 offset 1")
296+
tdSql.checkRows(1)
297+
tdSql.checkData(0, 0, b"\x61\x62\x00\x63")
298+
299+
# null row
300+
tdSql.query("select * from db.st where ntag is null")
301+
tdSql.checkRows(1)
302+
tdSql.checkData(0, 1, None)
303+
304+
print("do type blob .......................... [passed]")
305+
306+
178307
#
179308
# ------------------- test_taosdump_test_type_bool.py ----------------
180309
#
@@ -1358,6 +1487,8 @@ def do_all_datatypes(self, mode):
13581487
self.do_taosdump_type_unsigned_int(mode)
13591488
self.do_taosdump_type_unsigned_small_int(mode)
13601489
self.do_taosdump_type_unsigned_tiny_int(mode)
1490+
self.do_taosdump_type_decimal(mode)
1491+
self.do_taosdump_type_blob(mode)
13611492

13621493
def test_taosdump_datatypes(self):
13631494
"""taosdump data types

0 commit comments

Comments
 (0)