[drizzle-kit] push to d1-http failed (#4268)

* drizzle-kit and push to d1-http failed
There are two reasons:
- d1-http don't support transactions with db.run("begin/commit/rollback")
- introspections run against some CF-D1 internal tables and fails later.
  that could mitigated with a right tablesFilter: ['!_cf_KV'].
  There was a incomplete mitigation in place which now includes _cf_KV
  tables.

* chore: there are more _cf_ prefixed tables

* fix: Add escaping to sqlite pull queries

* should we pass-with-no-tests ??

* D P R I N T

* Add ci vitest config

* Bump kit, added release notes

---------

Co-authored-by: Roman <[email protected]>
Co-authored-by: AndriiSherman <[email protected]>

Author: 3 people

.github/workflows/release-feature-branch.yaml

@@ -191,7 +191,7 @@ jobs:
 
             neon-serverless)
               docker compose -f docker-neon.yml up -d
-              pnpm vitest run tests/pg/neon-serverless.test.ts
+              pnpm vitest run --config=./vitest-ci.config.ts tests/pg/neon-serverless.test.ts
               docker compose -f docker-neon.yml down
             ;;
 

.github/workflows/release-latest.yaml

@@ -285,6 +285,9 @@ jobs:
 
   release:
     permissions: write-all
+    needs:
+      - test
+      - attw
     strategy:
       fail-fast: false
       matrix:

changelogs/drizzle-kit/0.30.6.md

@@ -0,0 +1,4 @@
+### Bug fixes
+
+- [[BUG]: d1 push locally is not working](https://github.com/drizzle-team/drizzle-orm/issues/4099) - thanks @mabels and @RomanNabukhotnyi
+- [[BUG] Cloudflare D1: drizzle-kit push is not working (error 7500 SQLITE_AUTH)](https://github.com/drizzle-team/drizzle-orm/issues/3728) - thanks @mabels and @RomanNabukhotnyi

drizzle-kit/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "drizzle-kit",
-	"version": "0.30.5",
+	"version": "0.30.6",
 	"homepage": "https://orm.drizzle.team",
 	"keywords": [
 		"drizzle",

drizzle-kit/src/cli/commands/push.ts

@@ -511,18 +511,20 @@ export const sqlitePush = async (
 		if (statementsToExecute.length === 0) {
 			render(`\n[${chalk.blue('i')}] No changes detected`);
 		} else {
-			if (!('driver' in credentials)) {
-				await db.run('begin');
-				try {
-					for (const dStmnt of statementsToExecute) {
-						await db.run(dStmnt);
-					}
-					await db.run('commit');
-				} catch (e) {
-					console.error(e);
-					await db.run('rollback');
-					process.exit(1);
+			// D1-HTTP does not support transactions
+			// there might a be a better way to fix this
+			// in the db connection itself
+			const isNotD1 = !('driver' in credentials && credentials.driver === 'd1-http');
+			isNotD1 ?? await db.run('begin');
+			try {
+				for (const dStmnt of statementsToExecute) {
+					await db.run(dStmnt);
 				}
+				isNotD1 ?? await db.run('commit');
+			} catch (e) {
+				console.error(e);
+				isNotD1 ?? await db.run('rollback');
+				process.exit(1);
 			}
 			render(`[${chalk.green('✓')}] Changes applied`);
 		}

drizzle-kit/src/serializer/sqliteSerializer.ts

@@ -495,6 +495,18 @@ function extractGeneratedColumns(input: string): Record<string, ColumnInfo> {
 	return columns;
 }
 
+function filterIgnoredTablesByField(fieldName: string) {
+	// _cf_ is a prefix for internal Cloudflare D1 tables (e.g. _cf_KV, _cf_METADATA)
+	// _litestream_ is a prefix for internal Litestream tables (e.g. _litestream_seq, _litestream_lock)
+	// libsql_ is a prefix for internal libSQL tables (e.g. libsql_wasm_func_table)
+	// sqlite_ is a prefix for internal SQLite tables (e.g. sqlite_sequence, sqlite_stat1)
+	return `${fieldName} != '__drizzle_migrations'
+			AND ${fieldName} NOT LIKE '\\_cf\\_%' ESCAPE '\\'
+			AND ${fieldName} NOT LIKE '\\_litestream\\_%' ESCAPE '\\'
+			AND ${fieldName} NOT LIKE 'libsql\\_%' ESCAPE '\\'
+			AND ${fieldName} NOT LIKE 'sqlite\\_%' ESCAPE '\\'`;
+}
+
 export const fromDatabase = async (
 	db: SQLiteDB,
 	tablesFilter: (table: string) => boolean = (table) => true,
@@ -518,33 +530,30 @@ export const fromDatabase = async (
 		hidden: number;
 		sql: string;
 		type: 'view' | 'table';
-	}>(
-		`SELECT 
-    m.name as "tableName", p.name as "columnName", p.type as "columnType", p."notnull" as "notNull", p.dflt_value as "defaultValue", p.pk as pk, p.hidden as hidden, m.sql, m.type as type
-    FROM sqlite_master AS m JOIN pragma_table_xinfo(m.name) AS p
-    WHERE (m.type = 'table' OR m.type = 'view')
-    and m.tbl_name != 'sqlite_sequence' 
-    and m.tbl_name != 'sqlite_stat1' 
-    and m.tbl_name != '_litestream_seq' 
-    and m.tbl_name != '_litestream_lock' 
-    and m.tbl_name != 'libsql_wasm_func_table' 
-    and m.tbl_name != '__drizzle_migrations' 
-    and m.tbl_name != '_cf_KV';
-    `,
-	);
+	}>(`SELECT 
+		  m.name as "tableName",
+		  p.name as "columnName",
+		  p.type as "columnType",
+		  p."notnull" as "notNull",
+		  p.dflt_value as "defaultValue",
+		  p.pk as pk,
+		  p.hidden as hidden,
+		  m.sql,
+		  m.type as type
+		FROM sqlite_master AS m
+		JOIN pragma_table_xinfo(m.name) AS p
+		WHERE (m.type = 'table' OR m.type = 'view') 
+		  AND ${filterIgnoredTablesByField('m.tbl_name')};`);
 
 	const tablesWithSeq: string[] = [];
 
 	const seq = await db.query<{
 		name: string;
-	}>(
-		`SELECT * FROM sqlite_master WHERE name != 'sqlite_sequence' 
-    and name != 'sqlite_stat1' 
-    and name != '_litestream_seq' 
-    and name != '_litestream_lock' 
-    and tbl_name != '_cf_KV' 
-    and sql GLOB '*[ *' || CHAR(9) || CHAR(10) || CHAR(13) || ']AUTOINCREMENT[^'']*';`,
-	);
+	}>(`SELECT
+		  *
+		FROM sqlite_master
+		WHERE sql GLOB '*[ *' || CHAR(9) || CHAR(10) || CHAR(13) || ']AUTOINCREMENT[^'']*'
+    	  AND ${filterIgnoredTablesByField('tbl_name')};`);
 
 	for (const s of seq) {
 		tablesWithSeq.push(s.name);
@@ -685,11 +694,19 @@ export const fromDatabase = async (
 			onDelete: string;
 			seq: number;
 			id: number;
-		}>(
-			`SELECT m.name as "tableFrom", f.id as "id", f."table" as "tableTo", f."from", f."to", f."on_update" as "onUpdate", f."on_delete" as "onDelete", f.seq as "seq"
-      FROM sqlite_master m, pragma_foreign_key_list(m.name) as f 
-      where m.tbl_name != '_cf_KV';`,
-		);
+		}>(`SELECT
+			  m.name as "tableFrom",
+			  f.id as "id",
+			  f."table" as "tableTo",
+			  f."from",
+			  f."to",
+			  f."on_update" as "onUpdate",
+			  f."on_delete" as "onDelete",
+			  f.seq as "seq"
+      		FROM
+			  sqlite_master m,
+			  pragma_foreign_key_list(m.name) as f
+      		WHERE ${filterIgnoredTablesByField('m.tbl_name')};`);
 
 		const fkByTableName: Record<string, ForeignKey> = {};
 
@@ -752,21 +769,20 @@ export const fromDatabase = async (
 		columnName: string;
 		isUnique: number;
 		seq: string;
-	}>(
-		`SELECT 
-    m.tbl_name as tableName,
-    il.name as indexName,
-    ii.name as columnName,
-    il.[unique] as isUnique,
-    il.seq as seq
-FROM sqlite_master AS m,
-    pragma_index_list(m.name) AS il,
-    pragma_index_info(il.name) AS ii
-WHERE 
-    m.type = 'table' 
-    and il.name NOT LIKE 'sqlite_autoindex_%'
-    and m.tbl_name != '_cf_KV';`,
-	);
+	}>(`SELECT 
+    	  m.tbl_name as tableName,
+    	  il.name as indexName,
+    	  ii.name as columnName,
+    	  il.[unique] as isUnique,
+    	  il.seq as seq
+		FROM 
+		  sqlite_master AS m,
+    	  pragma_index_list(m.name) AS il,
+    	  pragma_index_info(il.name) AS ii
+		WHERE 
+		  m.type = 'table' 
+    	  AND il.name NOT LIKE 'sqlite\\_autoindex\\_%' ESCAPE '\\'
+    	  AND ${filterIgnoredTablesByField('m.tbl_name')};`);
 
 	for (const idxRow of idxs) {
 		const tableName = idxRow.tableName;
@@ -862,9 +878,15 @@ WHERE
 	const unnamedCheckPattern = /CHECK\s*\((.*?)\)/gi;
 	let checkCounter = 0;
 	const checkConstraints: Record<string, CheckConstraint> = {};
-	const checks = await db.query<{ tableName: string; sql: string }>(`SELECT name as "tableName", sql as "sql"
+	const checks = await db.query<{
+		tableName: string;
+		sql: string;
+	}>(`SELECT
+		  name as "tableName",
+		  sql as "sql"
 		FROM sqlite_master 
-		WHERE type = 'table' AND name != 'sqlite_sequence';`);
+		WHERE type = 'table'
+		  AND ${filterIgnoredTablesByField('tbl_name')};`);
 	for (const check of checks) {
 		if (!tablesFilter(check.tableName)) continue;
 

integration-tests/package.json

@@ -6,7 +6,7 @@
 	"scripts": {
 		"test:types": "tsc",
 		"test": "pnpm test:vitest",
-		"test:vitest": "vitest run",
+		"test:vitest": "vitest run --pass-with-no-tests",
 		"test:esm": "node tests/imports.test.mjs && node tests/imports.test.cjs",
 		"test:data-api": "sst shell vitest run tests/pg/awsdatapi.test.ts"
 	},

integration-tests/vitest-ci.config.ts

@@ -0,0 +1,22 @@
+import 'dotenv/config';
+import tsconfigPaths from 'vite-tsconfig-paths';
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+	test: {
+		typecheck: {
+			tsconfig: 'tsconfig.json',
+		},
+		testTimeout: 100000,
+		hookTimeout: 200000,
+		isolate: true,
+		poolOptions: {
+			threads: {
+				singleThread: true,
+			},
+		},
+		maxWorkers: 1,
+		fileParallelism: false,
+	},
+	plugins: [tsconfigPaths()],
+});