Fix DB.Close performance regression in large databases #434

example_usage.java

@@ -0,0 +1,50 @@
+import java.sql.Timestamp;
+import java.sql.Statement;
+
+// Method 1: Using a formatted string instead of Timestamp object
+private void insertUserWithFormattedTimestamp() throws SQLException {
+    final String INSERT_QUERY_USERS = "INSERT INTO USERS (username,created_at) VALUES (?,?)";
+    String urlForConnection = "jdbc:postgresql://"+url+":"+port+"/"+databaseName+"?sslmode=allow&preferQueryMode=simple";
+    var connection = DriverManager.getConnection(urlForConnection,"immudb", "immudb");
+    var client = connection.getClient();
+    PreparedStatement pstmt = client.prepareStatement(INSERT_QUERY_USERS);
+
+    Timestamp timestamp = new Timestamp(System.currentTimeMillis());
+
+    pstmt.setString(1, "TestUserName");
+    // Convert the timestamp to a string format immudb can handle
+    pstmt.setString(2, ImmudbTimestampHelper.formatTimestamp(timestamp));
+    pstmt.execute();
+}
+
+// Method 2: Using SQL CAST with a formatted timestamp
+private void insertUserWithTimestampCast() throws SQLException {
+    final String INSERT_QUERY_USERS =
+        "INSERT INTO USERS (username,created_at) VALUES (?, CAST(? AS TIMESTAMP))";
+    String urlForConnection = "jdbc:postgresql://"+url+":"+port+"/"+databaseName+"?sslmode=allow&preferQueryMode=simple";
+    var connection = DriverManager.getConnection(urlForConnection,"immudb", "immudb");
+    var client = connection.getClient();
+    PreparedStatement pstmt = client.prepareStatement(INSERT_QUERY_USERS);
+
+    Timestamp timestamp = new Timestamp(System.currentTimeMillis());
+
+    pstmt.setString(1, "TestUserName");
+    pstmt.setString(2, ImmudbTimestampHelper.formatTimestamp(timestamp));
+    pstmt.execute();
+}
+
+// Method 3: Using direct string concatenation (less secure, avoid with user input)
+private void insertUserWithDirectQuery() throws SQLException {
+    Timestamp timestamp = new Timestamp(System.currentTimeMillis());
+    String formattedTimestamp = ImmudbTimestampHelper.formatTimestamp(timestamp);
+
+    String query = String.format(
+        "INSERT INTO USERS (username,created_at) VALUES ('TestUserName', '%s')",
+        formattedTimestamp
+    );
+
+    String urlForConnection = "jdbc:postgresql://"+url+":"+port+"/"+databaseName+"?sslmode=allow&preferQueryMode=simple";
+    var connection = DriverManager.getConnection(urlForConnection,"immudb", "immudb");
+    Statement stmt = connection.createStatement();
+    stmt.execute(query);
+}

pkg/engine/timestamp_test.go

@@ -0,0 +1,92 @@
+package engine
+
+import (
+	"testing"
+	"time"
+
+	"github.com/codenotary/immudb/pkg/server"
+)
+
+func TestParseTimestamp(t *testing.T) {
+	now := time.Now().UTC()
+	formatted := now.Format("2006-01-02 15:04:05.999999")
+
+	// Test standard PostgreSQL timestamp format
+	t.Run("PostgreSQL format", func(t *testing.T) {
+		pgTimestamp := "2022-03-18 10:23:15"
+
+		// Test detection
+		if !server.IsPgTimestampString(pgTimestamp) {
+			t.Errorf("Expected %s to be detected as a timestamp", pgTimestamp)
+		}
+
+		// Test conversion
+		ts, err := server.ConvertPgTimestamp(pgTimestamp)
+		if err != nil {
+			t.Errorf("Failed to parse %s: %v", pgTimestamp, err)
+		}
+
+		expected := time.Date(2022, 3, 18, 10, 23, 15, 0, time.UTC)
+		if !ts.Equal(expected) {
+			t.Errorf("Expected %v, got %v", expected, ts)
+		}
+	})
+
+	// Test ISO 8601 format
+	t.Run("ISO 8601 format", func(t *testing.T) {
+		isoTimestamp := "2022-03-18T10:23:15Z"
+
+		// Test detection
+		if !server.IsPgTimestampString(isoTimestamp) {
+			t.Errorf("Expected %s to be detected as a timestamp", isoTimestamp)
+		}
+
+		// Test conversion
+		ts, err := server.ConvertPgTimestamp(isoTimestamp)
+		if err != nil {
+			t.Errorf("Failed to parse %s: %v", isoTimestamp, err)
+		}
+
+		expected := time.Date(2022, 3, 18, 10, 23, 15, 0, time.UTC)
+		if !ts.Equal(expected) {
+			t.Errorf("Expected %v, got %v", expected, ts)
+		}
+	})
+
+	// Test PostgreSQL format with fractional seconds
+	t.Run("PostgreSQL format with fractional seconds", func(t *testing.T) {
+		pgTimestamp := "2022-03-18 10:23:15.123456"
+
+		// Test detection
+		if !server.IsPgTimestampString(pgTimestamp) {
+			t.Errorf("Expected %s to be detected as a timestamp", pgTimestamp)
+		}
+
+		// Test conversion
+		ts, err := server.ConvertPgTimestamp(pgTimestamp)
+		if err != nil {
+			t.Errorf("Failed to parse %s: %v", pgTimestamp, err)
+		}
+
+		expected := time.Date(2022, 3, 18, 10, 23, 15, 123456000, time.UTC)
+		if !ts.Equal(expected) {
+			t.Errorf("Expected %v, got %v", expected, ts)
+		}
+	})
+
+	// Test invalid timestamp
+	t.Run("Invalid timestamp", func(t *testing.T) {
+		invalidTimestamp := "not-a-timestamp"
+
+		// Test detection
+		if server.IsPgTimestampString(invalidTimestamp) {
+			t.Errorf("Expected %s to NOT be detected as a timestamp", invalidTimestamp)
+		}
+
+		// Test conversion
+		_, err := server.ConvertPgTimestamp(invalidTimestamp)
+		if err == nil {
+			t.Errorf("Expected error when parsing %s", invalidTimestamp)
+		}
+	})
+}

pkg/pgsql/pgtime/pg_timestamp.go

@@ -0,0 +1,68 @@
+package pgtime
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+// PostgreSQL timestamp formats that might be received from clients
+var PgTimestampFormats = []string{
+	"2006-01-02 15:04:05",           // YYYY-MM-DD HH:MM:SS
+	"2006-01-02 15:04:05.999999",    // YYYY-MM-DD HH:MM:SS.SSSSSS
+	"2006-01-02T15:04:05Z",          // ISO 8601
+	"2006-01-02T15:04:05.999999Z",   // ISO 8601 with microseconds
+	"2006-01-02T15:04:05-07:00",     // ISO 8601 with timezone
+	"2006-01-02T15:04:05.999-07:00", // ISO 8601 with microseconds and timezone
+	time.RFC3339,                    // RFC3339
+	time.RFC3339Nano,                // RFC3339 with nanoseconds
+}
+
+// IsPgTimestampString checks if a string likely represents a PostgreSQL timestamp
+func IsPgTimestampString(s string) bool {
+	for _, format := range PgTimestampFormats {
+		if _, err := time.Parse(format, s); err == nil {
+			return true
+		}
+	}
+	return false
+}
+
+// ConvertPgTimestamp attempts to parse a PostgreSQL timestamp string into a time.Time
+func ConvertPgTimestamp(s string) (time.Time, error) {
+	// First handle any PostgreSQL-specific formatting
+	s = strings.TrimSpace(s)
+
+	// Remove type annotations
+	if strings.HasPrefix(s, "::timestamp") {
+		s = strings.TrimPrefix(s, "::timestamp")
+		s = strings.TrimSpace(s)
+	}
+
+	// Try all supported formats
+	for _, format := range PgTimestampFormats {
+		if t, err := time.Parse(format, s); err == nil {
+			return t, nil
+		}
+	}
+
+	return time.Time{}, fmt.Errorf("value is not a timestamp: invalid value provided")
+}
+
+// FormatTimestamp formats a time.Time as a PostgreSQL-compatible timestamp string
+func FormatTimestamp(t time.Time) string {
+	return t.Format("2006-01-02 15:04:05.999999")
+}
+
+// HandlePgTimestampLiterals processes SQL queries to handle PostgreSQL timestamp literals
+func HandlePgTimestampLiterals(query string) string {
+	// Remove PostgreSQL timestamp type casts
+	query = strings.ReplaceAll(query, "::timestamp", "")
+
+	// Handle CURRENT_TIMESTAMP and NOW() functions
+	now := FormatTimestamp(time.Now())
+	query = strings.ReplaceAll(query, "CURRENT_TIMESTAMP", fmt.Sprintf("'%s'", now))
+	query = strings.ReplaceAll(query, "NOW()", fmt.Sprintf("'%s'", now))
+
+	return query
+}

pkg/pgsql/pgtime/pg_timestamp_test.go

@@ -0,0 +1,107 @@
+package pgtime
+
+import (
+	"testing"
+	"time"
+)
+
+func TestPgTimestampHandling(t *testing.T) {
+	// Test standard PostgreSQL timestamp format
+	t.Run("PostgreSQL format", func(t *testing.T) {
+		pgTimestamp := "2022-03-18 10:23:15"
+
+		// Test detection
+		if !IsPgTimestampString(pgTimestamp) {
+			t.Errorf("Expected %s to be detected as a timestamp", pgTimestamp)
+		}
+
+		// Test conversion
+		ts, err := ConvertPgTimestamp(pgTimestamp)
+		if err != nil {
+			t.Errorf("Failed to parse %s: %v", pgTimestamp, err)
+		}
+
+		expected := time.Date(2022, 3, 18, 10, 23, 15, 0, time.UTC)
+		if !ts.Equal(expected) {
+			t.Errorf("Expected %v, got %v", expected, ts)
+		}
+	})
+
+	// Test ISO 8601 format
+	t.Run("ISO 8601 format", func(t *testing.T) {
+		isoTimestamp := "2022-03-18T10:23:15Z"
+
+		// Test detection
+		if !IsPgTimestampString(isoTimestamp) {
+			t.Errorf("Expected %s to be detected as a timestamp", isoTimestamp)
+		}
+
+		// Test conversion
+		ts, err := ConvertPgTimestamp(isoTimestamp)
+		if err != nil {
+			t.Errorf("Failed to parse %s: %v", isoTimestamp, err)
+		}
+
+		expected := time.Date(2022, 3, 18, 10, 23, 15, 0, time.UTC)
+		if !ts.Equal(expected) {
+			t.Errorf("Expected %v, got %v", expected, ts)
+		}
+	})
+
+	// Test PostgreSQL format with fractional seconds
+	t.Run("PostgreSQL format with fractional seconds", func(t *testing.T) {
+		pgTimestamp := "2022-03-18 10:23:15.123456"
+
+		// Test detection
+		if !IsPgTimestampString(pgTimestamp) {
+			t.Errorf("Expected %s to be detected as a timestamp", pgTimestamp)
+		}
+
+		// Test conversion
+		ts, err := ConvertPgTimestamp(pgTimestamp)
+		if err != nil {
+			t.Errorf("Failed to parse %s: %v", pgTimestamp, err)
+		}
+
+		expected := time.Date(2022, 3, 18, 10, 23, 15, 123456000, time.UTC)
+		if !ts.Equal(expected) {
+			t.Errorf("Expected %v, got %v", expected, ts)
+		}
+	})
+
+	// Test invalid timestamp
+	t.Run("Invalid timestamp", func(t *testing.T) {
+		invalidTimestamp := "not-a-timestamp"
+
+		// Test detection
+		if IsPgTimestampString(invalidTimestamp) {
+			t.Errorf("Expected %s to NOT be detected as a timestamp", invalidTimestamp)
+		}
+
+		// Test conversion
+		_, err := ConvertPgTimestamp(invalidTimestamp)
+		if err == nil {
+			t.Errorf("Expected error when parsing %s", invalidTimestamp)
+		}
+	})
+
+	// Test format timestamp
+	t.Run("Format timestamp", func(t *testing.T) {
+		ts := time.Date(2022, 3, 18, 10, 23, 15, 123456000, time.UTC)
+		formatted := FormatTimestamp(ts)
+		expected := "2022-03-18 10:23:15.123456"
+		if formatted != expected {
+			t.Errorf("Expected formatted timestamp %s, got %s", expected, formatted)
+		}
+	})
+
+	// Test timestamp literal handling
+	t.Run("Handle timestamp literals", func(t *testing.T) {
+		query := "SELECT * FROM table WHERE timestamp_col = '2022-03-18 10:23:15'::timestamp"
+		modified := HandlePgTimestampLiterals(query)
+		expected := "SELECT * FROM table WHERE timestamp_col = '2022-03-18 10:23:15'"
+		if modified != expected {
+			t.Errorf("Expected modified query %s, got %s", expected, modified)
+		}
+	})
+}

pkg/pgsql/server/pg_handler.go

@@ -0,0 +1,45 @@
+package server
+
+import (
+	"strings"
+	"github.com/codenotary/immudb/pkg/api/schema"
+	"github.com/codenotary/immudb/pkg/server"
+)
+
+// ProcessSQLQueryForPgCompat processes an SQL query for PostgreSQL compatibility
+// including handling timestamp literals in the query
+func ProcessSQLQueryForPgCompat(query string, params []*schema.NamedParam) (string, []*schema.NamedParam, error) {
+	// Process query for timestamp literal patterns
+	modifiedQuery := query
+
+	// Find timestamp literals like '2022-01-01 12:00:00'::timestamp
+	// and convert them to a format immudb understands
+
+	// Process parameters that might contain timestamp values
+	if params != nil {
+		modifiedParams := make([]*schema.NamedParam, len(params))
+		for i, param := range params {
+			if param.Value != nil {
+				// Check if this might be a string that represents a timestamp
+				if strVal, ok := param.Value.Value.(*schema.SQLValue_S); ok {
+					if server.IsPgTimestampString(strVal.S) {
+						tsValue, err := server.ConvertToImmudbTimestamp(strVal.S)
+						if err == nil {
+							modifiedParam := *param
+							modifiedParam.Value = tsValue
+							modifiedParams[i] = &modifiedParam
+							continue
+						}
+					}
+				}
+			}
+			modifiedParams[i] = param
+		}
+		params = modifiedParams
+	}
+
+	// Replace timestamp casting in query with immudb compatible syntax
+	modifiedQuery = strings.ReplaceAll(modifiedQuery, "::timestamp", "")
+
+	return modifiedQuery, params, nil
+}