Browse Source

no more mysql

pull/7185/head
chrislu 1 month ago
parent
commit
323193cf8c
  1. 8
      postgres-examples/test_client.py
  2. 2
      weed/query/engine/catalog.go
  3. 2
      weed/query/engine/describe.go
  4. 2
      weed/query/engine/engine.go
  5. 75
      weed/query/engine/sql_parser_config.go
  6. 34
      weed/server/postgres/README.md
  7. 8
      weed/server/postgres/protocol.go

8
postgres-examples/test_client.py

@ -174,10 +174,10 @@ def test_data_queries(host, port, user, database, password=None):
print(f" 📋 Testing with table: {table_name}")
test_queries = [
(f"Count records in {table_name}", f"SELECT COUNT(*) FROM `{table_name}`"),
(f"Sample data from {table_name}", f"SELECT * FROM `{table_name}` LIMIT 3"),
(f"System columns from {table_name}", f"SELECT _timestamp_ns, _key, _source FROM `{table_name}` LIMIT 3"),
(f"Describe {table_name}", f"DESCRIBE `{table_name}`"),
(f"Count records in {table_name}", f"SELECT COUNT(*) FROM \"{table_name}\""),
(f"Sample data from {table_name}", f"SELECT * FROM \"{table_name}\" LIMIT 3"),
(f"System columns from {table_name}", f"SELECT _timestamp_ns, _key, _source FROM \"{table_name}\" LIMIT 3"),
(f"Describe {table_name}", f"DESCRIBE \"{table_name}\""),
]
for name, query in test_queries:

2
weed/query/engine/catalog.go

@ -218,7 +218,7 @@ func (c *SchemaCatalog) convertMQSchemaToTableInfo(namespace, topicName string,
}
// convertMQFieldTypeToSQL maps MQ field types to SQL types
// Assumption: Standard SQL type mappings with MySQL compatibility
// Uses standard SQL type mappings with PostgreSQL compatibility
func (c *SchemaCatalog) convertMQFieldTypeToSQL(fieldType *schema_pb.Type) (string, error) {
switch t := fieldType.Kind.(type) {
case *schema_pb.Type_ScalarType:

2
weed/query/engine/describe.go

@ -9,7 +9,7 @@ import (
)
// executeDescribeStatement handles DESCRIBE table commands
// Assumption: DESCRIBE shows table schema in MySQL-compatible format
// Shows table schema in PostgreSQL-compatible format
func (e *SQLEngine) executeDescribeStatement(ctx context.Context, tableName string, database string) (*QueryResult, error) {
if database == "" {
database = e.catalog.GetCurrentDatabase()

2
weed/query/engine/engine.go

@ -238,7 +238,7 @@ func ParseSQL(sql string) (Statement, error) {
partsOriginal := strings.Fields(sql) // Use original casing
for i, part := range partsUpper {
if part == "FROM" && i+1 < len(partsOriginal) {
// Remove quotes if present (PostgreSQL uses double quotes, MySQL uses backticks)
// Remove quotes if present (PostgreSQL uses double quotes)
dbName := strings.Trim(partsOriginal[i+1], "\"'`")
stmt.Schema = dbName // Set the Schema field for the test
stmt.OnTable.Name = stringValue(dbName) // Keep for compatibility

75
weed/query/engine/sql_parser_config.go

@ -7,85 +7,32 @@ import (
"time"
)
// SQLParserConfig controls which SQL parser to use
// SQLParserConfig controls SQL parsing behavior (PostgreSQL syntax only)
type SQLParserConfig struct {
UsePostgreSQLParser bool // If true, use pg_query_go; if false, use mysql-dialect parser
EnableDialectWarnings bool // If true, log warnings about dialect mismatches
// Currently only PostgreSQL syntax is supported
// This struct is kept for future extensibility
}
// DefaultSQLParserConfig returns the default configuration (MySQL parser for now)
func DefaultSQLParserConfig() *SQLParserConfig {
return &SQLParserConfig{
UsePostgreSQLParser: false, // Keep MySQL parser as default for stability
EnableDialectWarnings: true, // Enable warnings about dialect issues
}
}
// PostgreSQLSQLParserConfig returns configuration for PostgreSQL parser
// PostgreSQLSQLParserConfig returns the default configuration (PostgreSQL only)
func PostgreSQLSQLParserConfig() *SQLParserConfig {
return &SQLParserConfig{
UsePostgreSQLParser: true,
EnableDialectWarnings: false, // No warnings needed when using correct parser
}
return &SQLParserConfig{}
}
// ParseSQL is a unified interface that can use either parser based on configuration
// ParseSQL parses SQL using PostgreSQL syntax
func (config *SQLParserConfig) ParseSQL(sql string) (Statement, error) {
if config.UsePostgreSQLParser {
return config.parseWithPostgreSQL(sql)
}
return config.parseWithMySQL(sql)
}
// parseWithMySQL uses the PostgreSQL parser (fallback for backward compatibility)
func (config *SQLParserConfig) parseWithMySQL(sql string) (Statement, error) {
if config.EnableDialectWarnings {
config.checkForPostgreSQLDialectFeatures(sql)
}
// Since we've removed the MySQL parser, use the PostgreSQL parser instead
// This maintains backward compatibility while using the better parser
return config.parseWithPostgreSQL(sql)
}
// parseWithPostgreSQL uses the new PostgreSQL parser
func (config *SQLParserConfig) parseWithPostgreSQL(sql string) (Statement, error) {
// Use the PostgreSQL parser from engine.go
return ParseSQL(sql)
}
// checkForPostgreSQLDialectFeatures logs warnings for PostgreSQL-specific syntax
func (config *SQLParserConfig) checkForPostgreSQLDialectFeatures(sql string) {
sqlUpper := strings.ToUpper(sql)
// Check for PostgreSQL-specific features
if strings.Contains(sql, "\"") && !strings.Contains(sql, "'") {
fmt.Printf("WARNING: Detected double-quoted identifiers (\") - PostgreSQL uses these, MySQL uses backticks (`)\n")
}
if strings.Contains(sqlUpper, "||") && !strings.Contains(sqlUpper, "CONCAT") {
fmt.Printf("WARNING: Detected || string concatenation - PostgreSQL syntax, MySQL uses CONCAT()\n")
}
if strings.Contains(sqlUpper, "PG_") || strings.Contains(sqlUpper, "INFORMATION_SCHEMA") {
fmt.Printf("WARNING: Detected PostgreSQL system functions/catalogs - may not work with MySQL parser\n")
}
if strings.Contains(sqlUpper, "LIMIT") && strings.Contains(sqlUpper, "OFFSET") {
fmt.Printf("WARNING: LIMIT/OFFSET syntax may differ between PostgreSQL and MySQL\n")
}
}
// SQLEngineWithParser extends SQLEngine with configurable parser
// SQLEngineWithParser extends SQLEngine with PostgreSQL parser
type SQLEngineWithParser struct {
*SQLEngine
ParserConfig *SQLParserConfig
}
// NewSQLEngineWithParser creates a new SQLEngine with parser configuration
// NewSQLEngineWithParser creates a new SQLEngine with PostgreSQL parser
func NewSQLEngineWithParser(masterAddr string, config *SQLParserConfig) *SQLEngineWithParser {
if config == nil {
config = DefaultSQLParserConfig()
config = PostgreSQLSQLParserConfig()
}
return &SQLEngineWithParser{
@ -94,7 +41,7 @@ func NewSQLEngineWithParser(masterAddr string, config *SQLParserConfig) *SQLEngi
}
}
// ExecuteSQL overrides the base ExecuteSQL to use the configured parser
// ExecuteSQL overrides the base ExecuteSQL to use PostgreSQL parser
func (e *SQLEngineWithParser) ExecuteSQL(ctx context.Context, sql string) (*QueryResult, error) {
// Clean up the SQL
sql = strings.TrimSpace(sql)
@ -119,7 +66,7 @@ func (e *SQLEngineWithParser) ExecuteSQL(ctx context.Context, sql string) (*Quer
return e.handleDescribeCommand(ctx, sqlTrimmed)
}
// Parse the SQL statement using the configured parser
// Parse the SQL statement using PostgreSQL parser
stmt, err := e.ParserConfig.ParseSQL(sql)
if err != nil {
return &QueryResult{

34
weed/server/postgres/README.md

@ -28,7 +28,7 @@ weed/server/postgres/
- **Message Handlers**: Startup, query, parse/bind/execute sequences
- **Response Generation**: Row descriptions, data rows, command completion
- **Data Type Mapping**: SeaweedFS to PostgreSQL type conversion
- **SQL Parser**: Currently uses MySQL-dialect parser - see Architecture Notes below
- **SQL Parser**: Uses PostgreSQL-native parser for full dialect compatibility
- **Error Handling**: PostgreSQL-compliant error responses
- **MQ Integration**: Direct integration with SeaweedFS SQL engine for real topic data
- **System Query Support**: Essential PostgreSQL system queries (version, current_user, etc.)
@ -254,12 +254,12 @@ psql -h localhost -p 5432 -U seaweedfs -d default
### SQL Parser Dialect Considerations
**✅ MIGRATION COMPLETED - Enhanced Implementation**: Now fully supports PostgreSQL-native parsing:
**✅ POSTGRESQL ONLY**: SeaweedFS SQL engine exclusively supports PostgreSQL syntax:
- **✅ Core Engine**: `engine.go` now uses [`github.com/pganalyze/pg_query_go/v6`](https://github.com/pganalyze/pg_query_go) exclusively for proper PostgreSQL dialect support
- **PostgreSQL Server**: Automatically uses PostgreSQL parser for optimal wire protocol compatibility
- **Parser**: Uses native PostgreSQL parser (`pg_query_go`) for full PostgreSQL compatibility
- **Migration Status**: Core SQL execution engine fully migrated from MySQL-dialect to PostgreSQL-native parsing
- **✅ Core Engine**: `engine.go` uses custom PostgreSQL parser for proper dialect support
- **PostgreSQL Server**: Uses PostgreSQL parser for optimal wire protocol compatibility
- **Parser**: Custom lightweight PostgreSQL parser for full PostgreSQL compatibility
- **Support Status**: Only PostgreSQL syntax is supported - MySQL parsing has been removed
**Key Benefits of PostgreSQL Parser**:
- **Native Dialect Support**: Correctly handles PostgreSQL-specific syntax and semantics
@ -268,21 +268,17 @@ psql -h localhost -p 5432 -U seaweedfs -d default
- **Type System Alignment**: Better PostgreSQL type inference and coercion
- **Reduced Translation Overhead**: Eliminates need for dialect translation layer
**Compatibility Considerations**:
- **Identifier Quoting**: PostgreSQL uses double quotes (`"`) vs MySQL backticks (`` ` ``)
- **String Concatenation**: PostgreSQL uses `||` vs MySQL `CONCAT()`
- **System Functions**: PostgreSQL has unique system catalogs (`pg_catalog`) and functions
- **Backward Compatibility**: Existing `weed sql` commands continue using MySQL parser
**PostgreSQL Syntax Support**:
- **Identifier Quoting**: Uses PostgreSQL double quotes (`"`) for identifiers
- **String Concatenation**: Supports PostgreSQL `||` operator
- **System Functions**: Full support for PostgreSQL system catalogs (`pg_catalog`) and functions
- **Standard Compliance**: Follows PostgreSQL SQL standard and dialect
**Mitigation Strategies**:
- Query translation layer in `protocol.go` handles PostgreSQL-specific queries
- System query detection and response (`SELECT version()`, `BEGIN`, etc.)
**Implementation Features**:
- Native PostgreSQL query processing in `protocol.go`
- System query support (`SELECT version()`, `BEGIN`, etc.)
- Type mapping between PostgreSQL and SeaweedFS schema types
- Error code mapping to PostgreSQL standards
**Future Considerations**:
- Consider `pg_query_go` for pure PostgreSQL dialect parsing
- Evaluate generic SQL parsers that support multiple dialects
- Balance compatibility vs implementation complexity
- Comprehensive PostgreSQL wire protocol support
This package provides enterprise-grade PostgreSQL compatibility, enabling seamless integration of SeaweedFS with the entire PostgreSQL ecosystem.

8
weed/server/postgres/protocol.go

@ -252,12 +252,8 @@ func (s *PostgreSQLServer) handleSimpleQuery(session *PostgreSQLSession, query s
}
}()
// Use PostgreSQL parser if available, fall back to standard engine
if s.sqlEngineWithParser != nil {
result, err = s.sqlEngineWithParser.ExecuteSQL(ctx, cleanQuery)
} else {
result, err = s.sqlEngine.ExecuteSQL(ctx, cleanQuery)
}
// The sqlEngineWithParser is always initialized, so use it directly
result, err = s.sqlEngineWithParser.ExecuteSQL(ctx, cleanQuery)
}()
if err != nil {

Loading…
Cancel
Save