fix: import real data from Postgres system tables. (#299)

Author: TianyuZhang1214

catalog/initial_data.go

@@ -0,0 +1,22 @@
+package catalog
+
+var InitialDataTables = struct {
+	PGNamespace [][]any
+	PGRange     [][]any
+}{
+	PGNamespace: [][]any{
+		{"99", "pg_toast", "10", ""},
+		{"11", "pg_catalog", "10", "{postgres=UC/postgres,=U/postgres}"},
+		{"2200", "public", "6171", "{pg_database_owner,=UC/pg_database_owner,=U/pg_database_owner}"},
+		{"13219", "information_schema", "10", "{postgres=UC/postgres,=U/postgres}"},
+		{"16395", "test_schema", "10", ""},
+	},
+	PGRange: [][]any{
+		{"3904", "23", "4451", "0", "1978", "int4range_canonical", "int4range_subdiff"},
+		{"3906", "1700", "4532", "0", "3125", "-", "numrange_subdiff"},
+		{"3908", "1114", "4533", "0", "3128", "-", "tsrange_subdiff"},
+		{"3910", "1184", "4534", "0", "3127", "-", "tstzrange_subdiff"},
+		{"3912", "1082", "4535", "0", "3122", "daterange_canonical", "daterange_subdiff"},
+		{"3926", "20", "4536", "0", "3124", "int8range_canonical", "int8range_subdiff"},
+	},
+}

catalog/internal_tables.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/apecloud/myduckserver/adapter"
 	"github.com/apecloud/myduckserver/configuration"
+	"github.com/apecloud/myduckserver/initialdata"
 )
 
 type DatabaseProvider struct {
@@ -143,6 +144,43 @@ func (prov *DatabaseProvider) initCatalog() error {
 				return fmt.Errorf("failed to insert initial data into internal table %q: %w", t.Name, err)
 			}
 		}
+
+		initialFileContent := initialdata.InitialTableDataMap[t.Name]
+		if initialFileContent != "" {
+			var count int
+			// Count rows in the internal table
+			if err := prov.storage.QueryRow(t.CountAllStmt()).Scan(&count); err != nil {
+				return fmt.Errorf("failed to count rows in internal table %q: %w", t.Name, err)
+			}
+
+			if count == 0 {
+				// Create temporary file to store initial data
+				tmpFile, err := os.CreateTemp("", "initial-data-"+t.Name+".csv")
+				if err != nil {
+					return fmt.Errorf("failed to create temporary file for initial data: %w", err)
+				}
+				// Ensure the temporary file is removed after usage
+				defer os.Remove(tmpFile.Name())
+				defer tmpFile.Close()
+
+				// Write the initial data to the temporary file
+				if _, err := tmpFile.WriteString(initialFileContent); err != nil {
+					return fmt.Errorf("failed to write initial data to temporary file: %w", err)
+				}
+
+				if err = tmpFile.Sync(); err != nil {
+					return fmt.Errorf("failed to sync initial data file: %w", err)
+				}
+
+				// Execute the COPY command to insert data into the table
+				if _, err := prov.storage.ExecContext(
+					context.Background(),
+					fmt.Sprintf("COPY %s FROM '%s' (DELIMITER ',', HEADER)", t.QualifiedName(), tmpFile.Name()),
+				); err != nil {
+					return fmt.Errorf("failed to insert initial data from file into internal table %q: %w", t.Name, err)
+				}
+			}
+		}
 	}
 
 	if _, err := prov.pool.ExecContext(context.Background(), "PRAGMA enable_checkpoint_on_shutdown"); err != nil {

catalog/provider.go

@@ -14,7 +14,7 @@ public class Tests
 
         public void Connect(string ip, int port, string user, string password)
         {
-            string connectionString = $"Host={ip};Port={port};Username={user};Password={password};Database=postgres;";
+            string connectionString = $"Host={ip};Port={port};Username={user};Password={password};Database=postgres;Timeout=300;CommandTimeout=600;";
             try
             {
                 conn = new NpgsqlConnection(connectionString);

compatibility/pg/csharp/PGTest.cs

@@ -41,13 +41,11 @@ start_process() {
     start_process $BATS_TEST_DIRNAME/c/pg_test 127.0.0.1 5432 postgres "" $BATS_TEST_DIRNAME/test.data
 }
 
-# Failed because of the following error:
-# > Catalog Error: Table with name pg_range does not exist!
-# @test "pg-csharp" {
-#     set_custom_teardown "sudo pkill -f dotnet"
-#     start_process dotnet build $BATS_TEST_DIRNAME/csharp/PGTest.csproj -o $BATS_TEST_DIRNAME/csharp/bin
-#     start_process dotnet $BATS_TEST_DIRNAME/csharp/bin/PGTest.dll 127.0.0.1 5432 postgres "" $BATS_TEST_DIRNAME/test.data
-# }
+@test "pg-csharp" {
+    set_custom_teardown "sudo pkill -f dotnet"
+    start_process dotnet build $BATS_TEST_DIRNAME/csharp/PGTest.csproj -o $BATS_TEST_DIRNAME/csharp/bin
+    start_process dotnet $BATS_TEST_DIRNAME/csharp/bin/PGTest.dll 127.0.0.1 5432 postgres "" $BATS_TEST_DIRNAME/test.data
+}
 
 @test "pg-go" {
     start_process go build -o $BATS_TEST_DIRNAME/go/pg $BATS_TEST_DIRNAME/go/pg.go

compatibility/pg/test.bats

@@ -0,0 +1,18 @@
+package initialdata
+
+import _ "embed"
+
+//go:embed pg_class.csv
+var pgClassContent string
+
+//go:embed pg_proc.csv
+var pgProcContent string
+
+//go:embed pg_type.csv
+var pgTypeContent string
+
+var InitialTableDataMap = map[string]string{
+	"pg_class": pgClassContent,
+	"pg_proc":  pgProcContent,
+	"pg_type":  pgTypeContent,
+}

initialdata/file_content.go

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Start a PostgreSQL container and mount the current directory
+CONTAINER_ID=$(docker run --rm -d -e POSTGRES_PASSWORD=postgres -v "$(pwd):/data" postgres)
+sleep 5
+
+# Set file paths within the container
+PG_CLASS_FILE="/data/pg_class.csv"
+PG_PROC_FILE="/data/pg_proc.csv"
+PG_TYPE_FILE="/data/pg_type.csv"
+
+# Define SQL queries
+PG_CLASS_QUERY="SELECT oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound FROM pg_class"
+PG_PROC_QUERY="SELECT oid, proname, pronamespace, proowner, prolang, procost, prorows, provariadic, prosupport::regproc::oid, prokind, prosecdef, proleakproof, proisstrict, proretset, provolatile, proparallel, pronargs, pronargdefaults, prorettype, proargtypes, proallargtypes, proargmodes, proargnames, proargdefaults, protrftypes, prosrc, probin, prosqlbody, proconfig, proacl FROM pg_proc"
+PG_TYPE_QUERY="SELECT oid, typname, typnamespace, typowner, typlen, typbyval, typtype, typcategory, typispreferred, typisdefined, typdelim, typrelid, typsubscript::regproc::oid, typelem, typarray, typinput::regproc::oid, typoutput::regproc::oid, typreceive::regproc::oid, typsend::regproc::oid, typmodin::regproc::oid, typmodout::regproc::oid, typanalyze::regproc::oid, typalign, typstorage, typnotnull, typbasetype, typtypmod, typndims, typcollation, typdefaultbin, typdefault, typacl FROM pg_type"
+
+# Execute queries and export data to mounted files
+docker exec -i $CONTAINER_ID psql -U postgres -c "\COPY ($PG_CLASS_QUERY) TO '$PG_CLASS_FILE' WITH CSV HEADER"
+docker exec -i $CONTAINER_ID psql -U postgres -c "\COPY ($PG_PROC_QUERY) TO '$PG_PROC_FILE' WITH CSV HEADER"
+docker exec -i $CONTAINER_ID psql -U postgres -c "\COPY ($PG_TYPE_QUERY) TO '$PG_TYPE_FILE' WITH CSV HEADER"
+
+# Stop the container
+docker kill $CONTAINER_ID