From 323193cf8c55613a97193a9663d846746b907aaa Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 3 Sep 2025 15:34:39 -0700 Subject: [PATCH] no more mysql --- postgres-examples/test_client.py | 8 +-- weed/query/engine/catalog.go | 2 +- weed/query/engine/describe.go | 2 +- weed/query/engine/engine.go | 2 +- weed/query/engine/sql_parser_config.go | 75 ++++---------------------- weed/server/postgres/README.md | 34 ++++++------ weed/server/postgres/protocol.go | 8 +-- 7 files changed, 35 insertions(+), 96 deletions(-) diff --git a/postgres-examples/test_client.py b/postgres-examples/test_client.py index 3c5b0742d..e293d53cc 100644 --- a/postgres-examples/test_client.py +++ b/postgres-examples/test_client.py @@ -174,10 +174,10 @@ def test_data_queries(host, port, user, database, password=None): print(f" 📋 Testing with table: {table_name}") test_queries = [ - (f"Count records in {table_name}", f"SELECT COUNT(*) FROM `{table_name}`"), - (f"Sample data from {table_name}", f"SELECT * FROM `{table_name}` LIMIT 3"), - (f"System columns from {table_name}", f"SELECT _timestamp_ns, _key, _source FROM `{table_name}` LIMIT 3"), - (f"Describe {table_name}", f"DESCRIBE `{table_name}`"), + (f"Count records in {table_name}", f"SELECT COUNT(*) FROM \"{table_name}\""), + (f"Sample data from {table_name}", f"SELECT * FROM \"{table_name}\" LIMIT 3"), + (f"System columns from {table_name}", f"SELECT _timestamp_ns, _key, _source FROM \"{table_name}\" LIMIT 3"), + (f"Describe {table_name}", f"DESCRIBE \"{table_name}\""), ] for name, query in test_queries: diff --git a/weed/query/engine/catalog.go b/weed/query/engine/catalog.go index 8ee566d04..13857fd59 100644 --- a/weed/query/engine/catalog.go +++ b/weed/query/engine/catalog.go @@ -218,7 +218,7 @@ func (c *SchemaCatalog) convertMQSchemaToTableInfo(namespace, topicName string, } // convertMQFieldTypeToSQL maps MQ field types to SQL types -// Assumption: Standard SQL type mappings with MySQL compatibility +// Uses standard SQL type mappings with PostgreSQL compatibility func (c *SchemaCatalog) convertMQFieldTypeToSQL(fieldType *schema_pb.Type) (string, error) { switch t := fieldType.Kind.(type) { case *schema_pb.Type_ScalarType: diff --git a/weed/query/engine/describe.go b/weed/query/engine/describe.go index bae129b08..19d71f3dd 100644 --- a/weed/query/engine/describe.go +++ b/weed/query/engine/describe.go @@ -9,7 +9,7 @@ import ( ) // executeDescribeStatement handles DESCRIBE table commands -// Assumption: DESCRIBE shows table schema in MySQL-compatible format +// Shows table schema in PostgreSQL-compatible format func (e *SQLEngine) executeDescribeStatement(ctx context.Context, tableName string, database string) (*QueryResult, error) { if database == "" { database = e.catalog.GetCurrentDatabase() diff --git a/weed/query/engine/engine.go b/weed/query/engine/engine.go index 75b792a1e..8cd127a4c 100644 --- a/weed/query/engine/engine.go +++ b/weed/query/engine/engine.go @@ -238,7 +238,7 @@ func ParseSQL(sql string) (Statement, error) { partsOriginal := strings.Fields(sql) // Use original casing for i, part := range partsUpper { if part == "FROM" && i+1 < len(partsOriginal) { - // Remove quotes if present (PostgreSQL uses double quotes, MySQL uses backticks) + // Remove quotes if present (PostgreSQL uses double quotes) dbName := strings.Trim(partsOriginal[i+1], "\"'`") stmt.Schema = dbName // Set the Schema field for the test stmt.OnTable.Name = stringValue(dbName) // Keep for compatibility diff --git a/weed/query/engine/sql_parser_config.go b/weed/query/engine/sql_parser_config.go index 1edc7b4ad..b7e2ece1a 100644 --- a/weed/query/engine/sql_parser_config.go +++ b/weed/query/engine/sql_parser_config.go @@ -7,85 +7,32 @@ import ( "time" ) -// SQLParserConfig controls which SQL parser to use +// SQLParserConfig controls SQL parsing behavior (PostgreSQL syntax only) type SQLParserConfig struct { - UsePostgreSQLParser bool // If true, use pg_query_go; if false, use mysql-dialect parser - EnableDialectWarnings bool // If true, log warnings about dialect mismatches + // Currently only PostgreSQL syntax is supported + // This struct is kept for future extensibility } -// DefaultSQLParserConfig returns the default configuration (MySQL parser for now) -func DefaultSQLParserConfig() *SQLParserConfig { - return &SQLParserConfig{ - UsePostgreSQLParser: false, // Keep MySQL parser as default for stability - EnableDialectWarnings: true, // Enable warnings about dialect issues - } -} - -// PostgreSQLSQLParserConfig returns configuration for PostgreSQL parser +// PostgreSQLSQLParserConfig returns the default configuration (PostgreSQL only) func PostgreSQLSQLParserConfig() *SQLParserConfig { - return &SQLParserConfig{ - UsePostgreSQLParser: true, - EnableDialectWarnings: false, // No warnings needed when using correct parser - } + return &SQLParserConfig{} } -// ParseSQL is a unified interface that can use either parser based on configuration +// ParseSQL parses SQL using PostgreSQL syntax func (config *SQLParserConfig) ParseSQL(sql string) (Statement, error) { - if config.UsePostgreSQLParser { - return config.parseWithPostgreSQL(sql) - } - return config.parseWithMySQL(sql) -} - -// parseWithMySQL uses the PostgreSQL parser (fallback for backward compatibility) -func (config *SQLParserConfig) parseWithMySQL(sql string) (Statement, error) { - if config.EnableDialectWarnings { - config.checkForPostgreSQLDialectFeatures(sql) - } - - // Since we've removed the MySQL parser, use the PostgreSQL parser instead - // This maintains backward compatibility while using the better parser - return config.parseWithPostgreSQL(sql) -} - -// parseWithPostgreSQL uses the new PostgreSQL parser -func (config *SQLParserConfig) parseWithPostgreSQL(sql string) (Statement, error) { - // Use the PostgreSQL parser from engine.go return ParseSQL(sql) } -// checkForPostgreSQLDialectFeatures logs warnings for PostgreSQL-specific syntax -func (config *SQLParserConfig) checkForPostgreSQLDialectFeatures(sql string) { - sqlUpper := strings.ToUpper(sql) - - // Check for PostgreSQL-specific features - if strings.Contains(sql, "\"") && !strings.Contains(sql, "'") { - fmt.Printf("WARNING: Detected double-quoted identifiers (\") - PostgreSQL uses these, MySQL uses backticks (`)\n") - } - - if strings.Contains(sqlUpper, "||") && !strings.Contains(sqlUpper, "CONCAT") { - fmt.Printf("WARNING: Detected || string concatenation - PostgreSQL syntax, MySQL uses CONCAT()\n") - } - - if strings.Contains(sqlUpper, "PG_") || strings.Contains(sqlUpper, "INFORMATION_SCHEMA") { - fmt.Printf("WARNING: Detected PostgreSQL system functions/catalogs - may not work with MySQL parser\n") - } - - if strings.Contains(sqlUpper, "LIMIT") && strings.Contains(sqlUpper, "OFFSET") { - fmt.Printf("WARNING: LIMIT/OFFSET syntax may differ between PostgreSQL and MySQL\n") - } -} - -// SQLEngineWithParser extends SQLEngine with configurable parser +// SQLEngineWithParser extends SQLEngine with PostgreSQL parser type SQLEngineWithParser struct { *SQLEngine ParserConfig *SQLParserConfig } -// NewSQLEngineWithParser creates a new SQLEngine with parser configuration +// NewSQLEngineWithParser creates a new SQLEngine with PostgreSQL parser func NewSQLEngineWithParser(masterAddr string, config *SQLParserConfig) *SQLEngineWithParser { if config == nil { - config = DefaultSQLParserConfig() + config = PostgreSQLSQLParserConfig() } return &SQLEngineWithParser{ @@ -94,7 +41,7 @@ func NewSQLEngineWithParser(masterAddr string, config *SQLParserConfig) *SQLEngi } } -// ExecuteSQL overrides the base ExecuteSQL to use the configured parser +// ExecuteSQL overrides the base ExecuteSQL to use PostgreSQL parser func (e *SQLEngineWithParser) ExecuteSQL(ctx context.Context, sql string) (*QueryResult, error) { // Clean up the SQL sql = strings.TrimSpace(sql) @@ -119,7 +66,7 @@ func (e *SQLEngineWithParser) ExecuteSQL(ctx context.Context, sql string) (*Quer return e.handleDescribeCommand(ctx, sqlTrimmed) } - // Parse the SQL statement using the configured parser + // Parse the SQL statement using PostgreSQL parser stmt, err := e.ParserConfig.ParseSQL(sql) if err != nil { return &QueryResult{ diff --git a/weed/server/postgres/README.md b/weed/server/postgres/README.md index 8f5b82a6f..7d9ecefe5 100644 --- a/weed/server/postgres/README.md +++ b/weed/server/postgres/README.md @@ -28,7 +28,7 @@ weed/server/postgres/ - **Message Handlers**: Startup, query, parse/bind/execute sequences - **Response Generation**: Row descriptions, data rows, command completion - **Data Type Mapping**: SeaweedFS to PostgreSQL type conversion -- **SQL Parser**: Currently uses MySQL-dialect parser - see Architecture Notes below +- **SQL Parser**: Uses PostgreSQL-native parser for full dialect compatibility - **Error Handling**: PostgreSQL-compliant error responses - **MQ Integration**: Direct integration with SeaweedFS SQL engine for real topic data - **System Query Support**: Essential PostgreSQL system queries (version, current_user, etc.) @@ -254,12 +254,12 @@ psql -h localhost -p 5432 -U seaweedfs -d default ### SQL Parser Dialect Considerations -**✅ MIGRATION COMPLETED - Enhanced Implementation**: Now fully supports PostgreSQL-native parsing: +**✅ POSTGRESQL ONLY**: SeaweedFS SQL engine exclusively supports PostgreSQL syntax: -- **✅ Core Engine**: `engine.go` now uses [`github.com/pganalyze/pg_query_go/v6`](https://github.com/pganalyze/pg_query_go) exclusively for proper PostgreSQL dialect support -- **PostgreSQL Server**: Automatically uses PostgreSQL parser for optimal wire protocol compatibility -- **Parser**: Uses native PostgreSQL parser (`pg_query_go`) for full PostgreSQL compatibility -- **Migration Status**: Core SQL execution engine fully migrated from MySQL-dialect to PostgreSQL-native parsing +- **✅ Core Engine**: `engine.go` uses custom PostgreSQL parser for proper dialect support +- **PostgreSQL Server**: Uses PostgreSQL parser for optimal wire protocol compatibility +- **Parser**: Custom lightweight PostgreSQL parser for full PostgreSQL compatibility +- **Support Status**: Only PostgreSQL syntax is supported - MySQL parsing has been removed **Key Benefits of PostgreSQL Parser**: - **Native Dialect Support**: Correctly handles PostgreSQL-specific syntax and semantics @@ -268,21 +268,17 @@ psql -h localhost -p 5432 -U seaweedfs -d default - **Type System Alignment**: Better PostgreSQL type inference and coercion - **Reduced Translation Overhead**: Eliminates need for dialect translation layer -**Compatibility Considerations**: -- **Identifier Quoting**: PostgreSQL uses double quotes (`"`) vs MySQL backticks (`` ` ``) -- **String Concatenation**: PostgreSQL uses `||` vs MySQL `CONCAT()` -- **System Functions**: PostgreSQL has unique system catalogs (`pg_catalog`) and functions -- **Backward Compatibility**: Existing `weed sql` commands continue using MySQL parser +**PostgreSQL Syntax Support**: +- **Identifier Quoting**: Uses PostgreSQL double quotes (`"`) for identifiers +- **String Concatenation**: Supports PostgreSQL `||` operator +- **System Functions**: Full support for PostgreSQL system catalogs (`pg_catalog`) and functions +- **Standard Compliance**: Follows PostgreSQL SQL standard and dialect -**Mitigation Strategies**: -- Query translation layer in `protocol.go` handles PostgreSQL-specific queries -- System query detection and response (`SELECT version()`, `BEGIN`, etc.) +**Implementation Features**: +- Native PostgreSQL query processing in `protocol.go` +- System query support (`SELECT version()`, `BEGIN`, etc.) - Type mapping between PostgreSQL and SeaweedFS schema types - Error code mapping to PostgreSQL standards - -**Future Considerations**: -- Consider `pg_query_go` for pure PostgreSQL dialect parsing -- Evaluate generic SQL parsers that support multiple dialects -- Balance compatibility vs implementation complexity +- Comprehensive PostgreSQL wire protocol support This package provides enterprise-grade PostgreSQL compatibility, enabling seamless integration of SeaweedFS with the entire PostgreSQL ecosystem. diff --git a/weed/server/postgres/protocol.go b/weed/server/postgres/protocol.go index 90d55e581..31773fd29 100644 --- a/weed/server/postgres/protocol.go +++ b/weed/server/postgres/protocol.go @@ -252,12 +252,8 @@ func (s *PostgreSQLServer) handleSimpleQuery(session *PostgreSQLSession, query s } }() - // Use PostgreSQL parser if available, fall back to standard engine - if s.sqlEngineWithParser != nil { - result, err = s.sqlEngineWithParser.ExecuteSQL(ctx, cleanQuery) - } else { - result, err = s.sqlEngine.ExecuteSQL(ctx, cleanQuery) - } + // The sqlEngineWithParser is always initialized, so use it directly + result, err = s.sqlEngineWithParser.ExecuteSQL(ctx, cleanQuery) }() if err != nil {