feat: support automatic redirection to __sys__ (#296)

* feat: support automatic redirection to __sys__

Signed-off-by: Noy <[email protected]>

* ci: let clients compatibility tests recover

Signed-off-by: Noy <[email protected]>

---------

Signed-off-by: Noy <[email protected]>

Author: NoyException

.github/workflows/clients-compatibility.yml

@@ -33,8 +33,8 @@ jobs:
       - name: Install system packages
         uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          packages: bats cpanminus libmysqlclient-dev dotnet-sdk-8.0 dotnet-runtime-8.0 php-mysql r-base-core libblas-dev
-          version: 1.0
+          packages: bats cpanminus libmysqlclient-dev dotnet-sdk-8.0 dotnet-runtime-8.0 php-mysql r-base-core
+          version: 1.1
 
       - name: Install dependencies
         run: |
@@ -52,6 +52,7 @@ jobs:
           npm install mysql
           sudo cpanm --notest DBD::mysql
           pip3 install mysql-connector-python
+          sudo apt-get install -y libblas-dev
           sudo R -e "install.packages('RMySQL', repos='http://cran.r-project.org')"
           sudo gem install mysql2
 
@@ -90,8 +91,8 @@ jobs:
       - name: Install system packages
         uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          packages: bats cpanminus libpq-dev postgresql-client dotnet-sdk-8.0 dotnet-runtime-8.0 r-base-core libblas-dev
-          version: 1.0
+          packages: bats cpanminus libpq-dev postgresql-client dotnet-sdk-8.0 dotnet-runtime-8.0 r-base-core
+          version: 1.1
 
       - name: Install dependencies
         run: |
@@ -109,6 +110,7 @@ jobs:
           npm install pg
           sudo cpanm --notest DBD::Pg
           pip3 install "psycopg[binary]" pandas pyarrow polars
+          sudo apt-get install -y libblas-dev
           # TODO: Speed up the installation of RPostgres
           sudo R -e "install.packages('RPostgres', repos='http://cran.r-project.org')"
           sudo gem install pg

catalog/internal_tables.go

@@ -177,6 +177,7 @@ var InternalTables = struct {
 	//             Once we add 'pg_catalog' and support views for PG, replace this by a view.
 	//             https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-REPLICATION-VIEW
 	PGStatReplication InternalTable
+	PGRange           InternalTable
 }{
 	PersistentVariable: InternalTable{
 		Schema:       "__sys__",
@@ -287,6 +288,21 @@ var InternalTables = struct {
 		},
 		DDL: "pid INTEGER PRIMARY KEY, usesysid TEXT, usename TEXT, application_name TEXT, client_addr TEXT, client_hostname TEXT, client_port INTEGER, backend_start TIMESTAMP, backend_xmin INTEGER, state TEXT, sent_lsn TEXT, write_lsn TEXT, flush_lsn TEXT, replay_lsn TEXT, write_lag INTERVAL, flush_lag INTERVAL, replay_lag INTERVAL, sync_priority INTEGER, sync_state TEXT, reply_time TIMESTAMP",
 	},
+	PGRange: InternalTable{
+		Schema:       "__sys__",
+		Name:         "pg_range",
+		KeyColumns:   []string{"rngtypid"},
+		ValueColumns: []string{"rngsubtype", "rngmultitypid", "rngcollation", "rngsubopc", "rngcanonical", "rngsubdiff"},
+		DDL:          "rngtypid TEXT PRIMARY KEY, rngsubtype TEXT, rngmultitypid TEXT, rngcollation TEXT, rngsubopc TEXT, rngcanonical TEXT, rngsubdiff TEXT",
+		InitialData: [][]any{
+			{"3904", "23", "4451", "0", "1978", "int4range_canonical", "int4range_subdiff"},
+			{"3906", "1700", "4532", "0", "3125", "-", "numrange_subdiff"},
+			{"3908", "1114", "4533", "0", "3128", "-", "tsrange_subdiff"},
+			{"3910", "1184", "4534", "0", "3127", "-", "tstzrange_subdiff"},
+			{"3912", "1082", "4535", "0", "3122", "daterange_canonical", "daterange_subdiff"},
+			{"3926", "20", "4536", "0", "3124", "int8range_canonical", "int8range_subdiff"},
+		},
+	},
 }
 
 var internalTables = []InternalTable{
@@ -295,4 +311,9 @@ var internalTables = []InternalTable{
 	InternalTables.PgSubscription,
 	InternalTables.GlobalStatus,
 	InternalTables.PGStatReplication,
+	InternalTables.PGRange,
+}
+
+func GetInternalTables() []InternalTable {
+	return internalTables
 }

pgserver/duck_handler.go

@@ -424,7 +424,7 @@ func (h *DuckHandler) executeQuery(ctx *sql.Context, query string, parsed tree.S
 		}))
 
 	default:
-		rows, err = adapter.QueryCatalog(ctx, query)
+		rows, err = adapter.QueryCatalog(ctx, ConvertToSys(query))
 		if err != nil {
 			break
 		}

pgserver/pg_catalog_handler.go

@@ -25,9 +25,6 @@ var pgWALLSNRegex = regexp.MustCompile(`(?i)^\s*select\s+pg_catalog\.(pg_current
 // precompile a regex to match "select pg_catalog.current_setting('xxx');".
 var currentSettingRegex = regexp.MustCompile(`(?i)^\s*select\s+(pg_catalog.)?current_setting\(\s*'([^']+)'\s*\)\s*;?\s*$`)
 
-// precompile a regex to match any "from pg_catalog.pg_stat_replication" in the query.
-var pgCatalogRegex = regexp.MustCompile(`(?i)\s+from\s+pg_catalog\.(pg_stat_replication)`)
-
 // isInRecovery will get the count of
 func (h *ConnectionHandler) isInRecovery() (string, error) {
 	// Grab a sql.Context.
@@ -167,9 +164,8 @@ func (h *ConnectionHandler) handleCurrentSetting(query ConvertedStatement) (bool
 
 // handler for pgCatalog
 func (h *ConnectionHandler) handlePgCatalog(query ConvertedStatement) (bool, error) {
-	sql := RemoveComments(query.String)
 	return true, h.run(ConvertedStatement{
-		String: pgCatalogRegex.ReplaceAllString(sql, " FROM __sys__.$1"),
+		String: ConvertToSys(query.String),
 		Tag:    "SELECT",
 	})
 }
@@ -208,7 +204,7 @@ func isPgCurrentSetting(query ConvertedStatement) bool {
 
 func isSpecialPgCatalog(query ConvertedStatement) bool {
 	sql := RemoveComments(query.String)
-	return pgCatalogRegex.MatchString(sql)
+	return getPgCatalogRegex().MatchString(sql)
 }
 
 // The key is the statement tag of the query.

pgserver/stmt.go

@@ -2,7 +2,10 @@ package pgserver
 
 import (
 	"bytes"
+	"github.com/apecloud/myduckserver/catalog"
+	"regexp"
 	"strings"
+	"sync"
 	"unicode"
 
 	"github.com/dolthub/go-mysql-server/sql"
@@ -259,3 +262,27 @@ func RemoveComments(query string) string {
 
 	return buf.String()
 }
+
+var (
+	pgCatalogRegex     *regexp.Regexp
+	initPgCatalogRegex sync.Once
+)
+
+// get the regex to match any table in pg_catalog in the query.
+func getPgCatalogRegex() *regexp.Regexp {
+	initPgCatalogRegex.Do(func() {
+		var tableNames []string
+		for _, table := range catalog.GetInternalTables() {
+			if table.Schema != "__sys__" {
+				continue
+			}
+			tableNames = append(tableNames, table.Name)
+		}
+		pgCatalogRegex = regexp.MustCompile(`(?i)\b(?:FROM|JOIN)\s+(?:pg_catalog\.)?(` + strings.Join(tableNames, "|") + `)`)
+	})
+	return pgCatalogRegex
+}
+
+func ConvertToSys(sql string) string {
+	return getPgCatalogRegex().ReplaceAllString(RemoveComments(sql), " __sys__.$1")
+}