8 changed files with 3322 additions and 0 deletions
-
415postgres-examples/README.md
-
374postgres-examples/test_client.py
-
379weed/command/postgres.go
-
389weed/server/postgres/DESIGN.md
-
240weed/server/postgres/README.md
-
529weed/server/postgres/protocol.go
-
640weed/server/postgres/server.go
-
356weed/server/postgres/translator.go
@ -0,0 +1,415 @@ |
|||||
|
# SeaweedFS PostgreSQL Protocol Examples |
||||
|
|
||||
|
This directory contains examples demonstrating how to connect to SeaweedFS using the PostgreSQL wire protocol. |
||||
|
|
||||
|
## Starting the PostgreSQL Server |
||||
|
|
||||
|
```bash |
||||
|
# Start with trust authentication (no password required) |
||||
|
weed postgres -port=5432 -master=localhost:9333 |
||||
|
|
||||
|
# Start with password authentication |
||||
|
weed postgres -port=5432 -auth=password -users="admin:secret,readonly:view123" |
||||
|
|
||||
|
# Start with MD5 authentication (more secure) |
||||
|
weed postgres -port=5432 -auth=md5 -users="user1:pass1,user2:pass2" |
||||
|
|
||||
|
# Start with TLS encryption |
||||
|
weed postgres -port=5432 -tls-cert=server.crt -tls-key=server.key |
||||
|
|
||||
|
# Allow connections from any host |
||||
|
weed postgres -host=0.0.0.0 -port=5432 |
||||
|
``` |
||||
|
|
||||
|
## Client Connections |
||||
|
|
||||
|
### psql Command Line |
||||
|
|
||||
|
```bash |
||||
|
# Basic connection (trust auth) |
||||
|
psql -h localhost -p 5432 -U seaweedfs -d default |
||||
|
|
||||
|
# With password |
||||
|
PGPASSWORD=secret psql -h localhost -p 5432 -U admin -d default |
||||
|
|
||||
|
# Connection string format |
||||
|
psql "postgresql://admin:secret@localhost:5432/default" |
||||
|
|
||||
|
# Connection string with parameters |
||||
|
psql "host=localhost port=5432 dbname=default user=admin password=secret" |
||||
|
``` |
||||
|
|
||||
|
### Programming Languages |
||||
|
|
||||
|
#### Python (psycopg2) |
||||
|
```python |
||||
|
import psycopg2 |
||||
|
|
||||
|
# Connect to SeaweedFS |
||||
|
conn = psycopg2.connect( |
||||
|
host="localhost", |
||||
|
port=5432, |
||||
|
user="seaweedfs", |
||||
|
database="default" |
||||
|
) |
||||
|
|
||||
|
# Execute queries |
||||
|
cursor = conn.cursor() |
||||
|
cursor.execute("SELECT * FROM my_topic LIMIT 10") |
||||
|
|
||||
|
for row in cursor.fetchall(): |
||||
|
print(row) |
||||
|
|
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
``` |
||||
|
|
||||
|
#### Java JDBC |
||||
|
```java |
||||
|
import java.sql.*; |
||||
|
|
||||
|
public class SeaweedFSExample { |
||||
|
public static void main(String[] args) throws SQLException { |
||||
|
String url = "jdbc:postgresql://localhost:5432/default"; |
||||
|
|
||||
|
Connection conn = DriverManager.getConnection(url, "seaweedfs", ""); |
||||
|
Statement stmt = conn.createStatement(); |
||||
|
|
||||
|
ResultSet rs = stmt.executeQuery("SELECT * FROM my_topic LIMIT 10"); |
||||
|
while (rs.next()) { |
||||
|
System.out.println("ID: " + rs.getLong("id")); |
||||
|
System.out.println("Message: " + rs.getString("message")); |
||||
|
} |
||||
|
|
||||
|
rs.close(); |
||||
|
stmt.close(); |
||||
|
conn.close(); |
||||
|
} |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
#### Go (lib/pq) |
||||
|
```go |
||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"database/sql" |
||||
|
"fmt" |
||||
|
_ "github.com/lib/pq" |
||||
|
) |
||||
|
|
||||
|
func main() { |
||||
|
db, err := sql.Open("postgres", |
||||
|
"host=localhost port=5432 user=seaweedfs dbname=default sslmode=disable") |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
defer db.Close() |
||||
|
|
||||
|
rows, err := db.Query("SELECT * FROM my_topic LIMIT 10") |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
defer rows.Close() |
||||
|
|
||||
|
for rows.Next() { |
||||
|
var id int64 |
||||
|
var message string |
||||
|
err := rows.Scan(&id, &message) |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
fmt.Printf("ID: %d, Message: %s\n", id, message) |
||||
|
} |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
#### Node.js (pg) |
||||
|
```javascript |
||||
|
const { Client } = require('pg'); |
||||
|
|
||||
|
const client = new Client({ |
||||
|
host: 'localhost', |
||||
|
port: 5432, |
||||
|
user: 'seaweedfs', |
||||
|
database: 'default', |
||||
|
}); |
||||
|
|
||||
|
async function query() { |
||||
|
await client.connect(); |
||||
|
|
||||
|
const result = await client.query('SELECT * FROM my_topic LIMIT 10'); |
||||
|
console.log(result.rows); |
||||
|
|
||||
|
await client.end(); |
||||
|
} |
||||
|
|
||||
|
query().catch(console.error); |
||||
|
``` |
||||
|
|
||||
|
## SQL Operations |
||||
|
|
||||
|
### Basic Queries |
||||
|
```sql |
||||
|
-- List databases |
||||
|
SHOW DATABASES; |
||||
|
|
||||
|
-- List tables (topics) |
||||
|
SHOW TABLES; |
||||
|
|
||||
|
-- Describe table structure |
||||
|
DESCRIBE my_topic; |
||||
|
-- or |
||||
|
DESC my_topic; |
||||
|
|
||||
|
-- Basic select |
||||
|
SELECT * FROM my_topic; |
||||
|
|
||||
|
-- With WHERE clause |
||||
|
SELECT id, message FROM my_topic WHERE id > 1000; |
||||
|
|
||||
|
-- With LIMIT |
||||
|
SELECT * FROM my_topic ORDER BY _timestamp_ns DESC LIMIT 100; |
||||
|
``` |
||||
|
|
||||
|
### Aggregations |
||||
|
```sql |
||||
|
-- Count records |
||||
|
SELECT COUNT(*) FROM my_topic; |
||||
|
|
||||
|
-- Multiple aggregations |
||||
|
SELECT |
||||
|
COUNT(*) as total_messages, |
||||
|
MIN(id) as min_id, |
||||
|
MAX(id) as max_id, |
||||
|
AVG(amount) as avg_amount |
||||
|
FROM my_topic; |
||||
|
|
||||
|
-- Aggregations with WHERE |
||||
|
SELECT COUNT(*) FROM my_topic WHERE status = 'active'; |
||||
|
``` |
||||
|
|
||||
|
### System Columns |
||||
|
```sql |
||||
|
-- Access system columns |
||||
|
SELECT |
||||
|
id, |
||||
|
message, |
||||
|
_timestamp_ns as timestamp, |
||||
|
_key as partition_key, |
||||
|
_source as data_source |
||||
|
FROM my_topic; |
||||
|
|
||||
|
-- Filter by timestamp |
||||
|
SELECT * FROM my_topic |
||||
|
WHERE _timestamp_ns > 1640995200000000000 |
||||
|
LIMIT 10; |
||||
|
``` |
||||
|
|
||||
|
### PostgreSQL System Queries |
||||
|
```sql |
||||
|
-- Version information |
||||
|
SELECT version(); |
||||
|
|
||||
|
-- Current database |
||||
|
SELECT current_database(); |
||||
|
|
||||
|
-- Current user |
||||
|
SELECT current_user; |
||||
|
|
||||
|
-- Server settings |
||||
|
SELECT current_setting('server_version'); |
||||
|
SELECT current_setting('server_encoding'); |
||||
|
``` |
||||
|
|
||||
|
## psql Meta-Commands |
||||
|
|
||||
|
```sql |
||||
|
-- List tables |
||||
|
\d |
||||
|
\dt |
||||
|
|
||||
|
-- List databases |
||||
|
\l |
||||
|
|
||||
|
-- Describe specific table |
||||
|
\d my_topic |
||||
|
\dt my_topic |
||||
|
|
||||
|
-- List schemas |
||||
|
\dn |
||||
|
|
||||
|
-- Help |
||||
|
\h |
||||
|
\? |
||||
|
|
||||
|
-- Quit |
||||
|
\q |
||||
|
``` |
||||
|
|
||||
|
## Database Tools Integration |
||||
|
|
||||
|
### DBeaver |
||||
|
1. Create New Connection → PostgreSQL |
||||
|
2. Settings: |
||||
|
- **Host**: localhost |
||||
|
- **Port**: 5432 |
||||
|
- **Database**: default |
||||
|
- **Username**: seaweedfs (or configured user) |
||||
|
- **Password**: (if using password auth) |
||||
|
|
||||
|
### pgAdmin |
||||
|
1. Add New Server |
||||
|
2. Connection tab: |
||||
|
- **Host**: localhost |
||||
|
- **Port**: 5432 |
||||
|
- **Username**: seaweedfs |
||||
|
- **Database**: default |
||||
|
|
||||
|
### DataGrip |
||||
|
1. New Data Source → PostgreSQL |
||||
|
2. Configure: |
||||
|
- **Host**: localhost |
||||
|
- **Port**: 5432 |
||||
|
- **User**: seaweedfs |
||||
|
- **Database**: default |
||||
|
|
||||
|
### Grafana |
||||
|
1. Add Data Source → PostgreSQL |
||||
|
2. Configuration: |
||||
|
- **Host**: localhost:5432 |
||||
|
- **Database**: default |
||||
|
- **User**: seaweedfs |
||||
|
- **SSL Mode**: disable |
||||
|
|
||||
|
## BI Tools |
||||
|
|
||||
|
### Tableau |
||||
|
1. Connect to Data → PostgreSQL |
||||
|
2. Server: localhost |
||||
|
3. Port: 5432 |
||||
|
4. Database: default |
||||
|
5. Username: seaweedfs |
||||
|
|
||||
|
### Power BI |
||||
|
1. Get Data → Database → PostgreSQL |
||||
|
2. Server: localhost |
||||
|
3. Database: default |
||||
|
4. Username: seaweedfs |
||||
|
|
||||
|
## Connection Pooling |
||||
|
|
||||
|
### Java (HikariCP) |
||||
|
```java |
||||
|
HikariConfig config = new HikariConfig(); |
||||
|
config.setJdbcUrl("jdbc:postgresql://localhost:5432/default"); |
||||
|
config.setUsername("seaweedfs"); |
||||
|
config.setMaximumPoolSize(10); |
||||
|
|
||||
|
HikariDataSource dataSource = new HikariDataSource(config); |
||||
|
``` |
||||
|
|
||||
|
### Python (connection pooling) |
||||
|
```python |
||||
|
from psycopg2 import pool |
||||
|
|
||||
|
connection_pool = psycopg2.pool.SimpleConnectionPool( |
||||
|
1, 20, |
||||
|
host="localhost", |
||||
|
port=5432, |
||||
|
user="seaweedfs", |
||||
|
database="default" |
||||
|
) |
||||
|
|
||||
|
conn = connection_pool.getconn() |
||||
|
# Use connection |
||||
|
connection_pool.putconn(conn) |
||||
|
``` |
||||
|
|
||||
|
## Security Best Practices |
||||
|
|
||||
|
### Use TLS Encryption |
||||
|
```bash |
||||
|
# Generate self-signed certificate for testing |
||||
|
openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt -days 365 -nodes |
||||
|
|
||||
|
# Start with TLS |
||||
|
weed postgres -tls-cert=server.crt -tls-key=server.key |
||||
|
``` |
||||
|
|
||||
|
### Use MD5 Authentication |
||||
|
```bash |
||||
|
# More secure than password auth |
||||
|
weed postgres -auth=md5 -users="admin:secret123,readonly:view456" |
||||
|
``` |
||||
|
|
||||
|
### Limit Connections |
||||
|
```bash |
||||
|
# Limit concurrent connections |
||||
|
weed postgres -max-connections=50 -idle-timeout=30m |
||||
|
``` |
||||
|
|
||||
|
## Troubleshooting |
||||
|
|
||||
|
### Connection Issues |
||||
|
```bash |
||||
|
# Test connectivity |
||||
|
telnet localhost 5432 |
||||
|
|
||||
|
# Check if server is running |
||||
|
ps aux | grep "weed postgres" |
||||
|
|
||||
|
# Check logs for errors |
||||
|
tail -f /var/log/seaweedfs/postgres.log |
||||
|
``` |
||||
|
|
||||
|
### Common Errors |
||||
|
|
||||
|
**"Connection refused"** |
||||
|
- Ensure PostgreSQL server is running |
||||
|
- Check host/port configuration |
||||
|
- Verify firewall settings |
||||
|
|
||||
|
**"Authentication failed"** |
||||
|
- Check username/password |
||||
|
- Verify auth method configuration |
||||
|
- Ensure user is configured in server |
||||
|
|
||||
|
**"Database does not exist"** |
||||
|
- Use correct database name (default: 'default') |
||||
|
- Check available databases: `SHOW DATABASES` |
||||
|
|
||||
|
**"Permission denied"** |
||||
|
- Check user permissions |
||||
|
- Verify authentication method |
||||
|
- Use correct credentials |
||||
|
|
||||
|
## Performance Tips |
||||
|
|
||||
|
1. **Use LIMIT clauses** for large result sets |
||||
|
2. **Filter with WHERE clauses** to reduce data transfer |
||||
|
3. **Use connection pooling** for multi-threaded applications |
||||
|
4. **Close resources properly** (connections, statements, result sets) |
||||
|
5. **Use prepared statements** for repeated queries |
||||
|
|
||||
|
## Monitoring |
||||
|
|
||||
|
### Connection Statistics |
||||
|
```sql |
||||
|
-- Current connections (if supported) |
||||
|
SELECT COUNT(*) FROM pg_stat_activity; |
||||
|
|
||||
|
-- Server version |
||||
|
SELECT version(); |
||||
|
|
||||
|
-- Current settings |
||||
|
SELECT name, setting FROM pg_settings WHERE name LIKE '%connection%'; |
||||
|
``` |
||||
|
|
||||
|
### Query Performance |
||||
|
```sql |
||||
|
-- Use EXPLAIN for query plans (if supported) |
||||
|
EXPLAIN SELECT * FROM my_topic WHERE id > 1000; |
||||
|
``` |
||||
|
|
||||
|
This PostgreSQL protocol support makes SeaweedFS accessible to the entire PostgreSQL ecosystem, enabling seamless integration with existing tools, applications, and workflows. |
@ -0,0 +1,374 @@ |
|||||
|
#!/usr/bin/env python3 |
||||
|
""" |
||||
|
Test client for SeaweedFS PostgreSQL protocol support. |
||||
|
|
||||
|
This script demonstrates how to connect to SeaweedFS using standard PostgreSQL |
||||
|
libraries and execute various types of queries. |
||||
|
|
||||
|
Requirements: |
||||
|
pip install psycopg2-binary |
||||
|
|
||||
|
Usage: |
||||
|
python test_client.py |
||||
|
python test_client.py --host localhost --port 5432 --user seaweedfs --database default |
||||
|
""" |
||||
|
|
||||
|
import sys |
||||
|
import argparse |
||||
|
import time |
||||
|
import traceback |
||||
|
|
||||
|
try: |
||||
|
import psycopg2 |
||||
|
import psycopg2.extras |
||||
|
except ImportError: |
||||
|
print("Error: psycopg2 not found. Install with: pip install psycopg2-binary") |
||||
|
sys.exit(1) |
||||
|
|
||||
|
|
||||
|
def test_connection(host, port, user, database, password=None): |
||||
|
"""Test basic connection to SeaweedFS PostgreSQL server.""" |
||||
|
print(f"🔗 Testing connection to {host}:{port}/{database} as user '{user}'") |
||||
|
|
||||
|
try: |
||||
|
conn_params = { |
||||
|
'host': host, |
||||
|
'port': port, |
||||
|
'user': user, |
||||
|
'database': database, |
||||
|
'connect_timeout': 10 |
||||
|
} |
||||
|
|
||||
|
if password: |
||||
|
conn_params['password'] = password |
||||
|
|
||||
|
conn = psycopg2.connect(**conn_params) |
||||
|
print("✅ Connection successful!") |
||||
|
|
||||
|
# Test basic query |
||||
|
cursor = conn.cursor() |
||||
|
cursor.execute("SELECT 1 as test") |
||||
|
result = cursor.fetchone() |
||||
|
print(f"✅ Basic query successful: {result}") |
||||
|
|
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
return True |
||||
|
|
||||
|
except Exception as e: |
||||
|
print(f"❌ Connection failed: {e}") |
||||
|
return False |
||||
|
|
||||
|
|
||||
|
def test_system_queries(host, port, user, database, password=None): |
||||
|
"""Test PostgreSQL system queries.""" |
||||
|
print("\n🔧 Testing PostgreSQL system queries...") |
||||
|
|
||||
|
try: |
||||
|
conn_params = { |
||||
|
'host': host, |
||||
|
'port': port, |
||||
|
'user': user, |
||||
|
'database': database |
||||
|
} |
||||
|
if password: |
||||
|
conn_params['password'] = password |
||||
|
|
||||
|
conn = psycopg2.connect(**conn_params) |
||||
|
cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) |
||||
|
|
||||
|
system_queries = [ |
||||
|
("Version", "SELECT version()"), |
||||
|
("Current Database", "SELECT current_database()"), |
||||
|
("Current User", "SELECT current_user"), |
||||
|
("Server Encoding", "SELECT current_setting('server_encoding')"), |
||||
|
("Client Encoding", "SELECT current_setting('client_encoding')"), |
||||
|
] |
||||
|
|
||||
|
for name, query in system_queries: |
||||
|
try: |
||||
|
cursor.execute(query) |
||||
|
result = cursor.fetchone() |
||||
|
print(f" ✅ {name}: {result[0]}") |
||||
|
except Exception as e: |
||||
|
print(f" ❌ {name}: {e}") |
||||
|
|
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
|
||||
|
except Exception as e: |
||||
|
print(f"❌ System queries failed: {e}") |
||||
|
|
||||
|
|
||||
|
def test_schema_queries(host, port, user, database, password=None): |
||||
|
"""Test schema and metadata queries.""" |
||||
|
print("\n📊 Testing schema queries...") |
||||
|
|
||||
|
try: |
||||
|
conn_params = { |
||||
|
'host': host, |
||||
|
'port': port, |
||||
|
'user': user, |
||||
|
'database': database |
||||
|
} |
||||
|
if password: |
||||
|
conn_params['password'] = password |
||||
|
|
||||
|
conn = psycopg2.connect(**conn_params) |
||||
|
cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) |
||||
|
|
||||
|
schema_queries = [ |
||||
|
("Show Databases", "SHOW DATABASES"), |
||||
|
("Show Tables", "SHOW TABLES"), |
||||
|
("List Schemas", "SELECT 'public' as schema_name"), |
||||
|
] |
||||
|
|
||||
|
for name, query in schema_queries: |
||||
|
try: |
||||
|
cursor.execute(query) |
||||
|
results = cursor.fetchall() |
||||
|
print(f" ✅ {name}: Found {len(results)} items") |
||||
|
for row in results[:3]: # Show first 3 results |
||||
|
print(f" - {dict(row)}") |
||||
|
if len(results) > 3: |
||||
|
print(f" ... and {len(results) - 3} more") |
||||
|
except Exception as e: |
||||
|
print(f" ❌ {name}: {e}") |
||||
|
|
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
|
||||
|
except Exception as e: |
||||
|
print(f"❌ Schema queries failed: {e}") |
||||
|
|
||||
|
|
||||
|
def test_data_queries(host, port, user, database, password=None): |
||||
|
"""Test data queries on actual topics.""" |
||||
|
print("\n📝 Testing data queries...") |
||||
|
|
||||
|
try: |
||||
|
conn_params = { |
||||
|
'host': host, |
||||
|
'port': port, |
||||
|
'user': user, |
||||
|
'database': database |
||||
|
} |
||||
|
if password: |
||||
|
conn_params['password'] = password |
||||
|
|
||||
|
conn = psycopg2.connect(**conn_params) |
||||
|
cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) |
||||
|
|
||||
|
# First, try to get available tables/topics |
||||
|
cursor.execute("SHOW TABLES") |
||||
|
tables = cursor.fetchall() |
||||
|
|
||||
|
if not tables: |
||||
|
print(" ℹ️ No tables/topics found for data testing") |
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
return |
||||
|
|
||||
|
# Test with first available table |
||||
|
table_name = tables[0][0] if tables[0] else 'test_topic' |
||||
|
print(f" 📋 Testing with table: {table_name}") |
||||
|
|
||||
|
test_queries = [ |
||||
|
(f"Count records in {table_name}", f"SELECT COUNT(*) FROM `{table_name}`"), |
||||
|
(f"Sample data from {table_name}", f"SELECT * FROM `{table_name}` LIMIT 3"), |
||||
|
(f"System columns from {table_name}", f"SELECT _timestamp_ns, _key, _source FROM `{table_name}` LIMIT 3"), |
||||
|
(f"Describe {table_name}", f"DESCRIBE `{table_name}`"), |
||||
|
] |
||||
|
|
||||
|
for name, query in test_queries: |
||||
|
try: |
||||
|
cursor.execute(query) |
||||
|
results = cursor.fetchall() |
||||
|
|
||||
|
if "COUNT" in query.upper(): |
||||
|
count = results[0][0] if results else 0 |
||||
|
print(f" ✅ {name}: {count} records") |
||||
|
elif "DESCRIBE" in query.upper(): |
||||
|
print(f" ✅ {name}: {len(results)} columns") |
||||
|
for row in results[:5]: # Show first 5 columns |
||||
|
print(f" - {dict(row)}") |
||||
|
else: |
||||
|
print(f" ✅ {name}: {len(results)} rows") |
||||
|
for row in results: |
||||
|
print(f" - {dict(row)}") |
||||
|
|
||||
|
except Exception as e: |
||||
|
print(f" ❌ {name}: {e}") |
||||
|
|
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
|
||||
|
except Exception as e: |
||||
|
print(f"❌ Data queries failed: {e}") |
||||
|
|
||||
|
|
||||
|
def test_prepared_statements(host, port, user, database, password=None): |
||||
|
"""Test prepared statements.""" |
||||
|
print("\n📝 Testing prepared statements...") |
||||
|
|
||||
|
try: |
||||
|
conn_params = { |
||||
|
'host': host, |
||||
|
'port': port, |
||||
|
'user': user, |
||||
|
'database': database |
||||
|
} |
||||
|
if password: |
||||
|
conn_params['password'] = password |
||||
|
|
||||
|
conn = psycopg2.connect(**conn_params) |
||||
|
cursor = conn.cursor() |
||||
|
|
||||
|
# Test parameterized query |
||||
|
try: |
||||
|
cursor.execute("SELECT %s as param1, %s as param2", ("hello", 42)) |
||||
|
result = cursor.fetchone() |
||||
|
print(f" ✅ Prepared statement: {result}") |
||||
|
except Exception as e: |
||||
|
print(f" ❌ Prepared statement: {e}") |
||||
|
|
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
|
||||
|
except Exception as e: |
||||
|
print(f"❌ Prepared statements test failed: {e}") |
||||
|
|
||||
|
|
||||
|
def test_transaction_support(host, port, user, database, password=None): |
||||
|
"""Test transaction support (should be no-op for read-only).""" |
||||
|
print("\n🔄 Testing transaction support...") |
||||
|
|
||||
|
try: |
||||
|
conn_params = { |
||||
|
'host': host, |
||||
|
'port': port, |
||||
|
'user': user, |
||||
|
'database': database |
||||
|
} |
||||
|
if password: |
||||
|
conn_params['password'] = password |
||||
|
|
||||
|
conn = psycopg2.connect(**conn_params) |
||||
|
cursor = conn.cursor() |
||||
|
|
||||
|
transaction_commands = [ |
||||
|
"BEGIN", |
||||
|
"SELECT 1 as in_transaction", |
||||
|
"COMMIT", |
||||
|
"SELECT 1 as after_commit", |
||||
|
] |
||||
|
|
||||
|
for cmd in transaction_commands: |
||||
|
try: |
||||
|
cursor.execute(cmd) |
||||
|
if "SELECT" in cmd: |
||||
|
result = cursor.fetchone() |
||||
|
print(f" ✅ {cmd}: {result}") |
||||
|
else: |
||||
|
print(f" ✅ {cmd}: OK") |
||||
|
except Exception as e: |
||||
|
print(f" ❌ {cmd}: {e}") |
||||
|
|
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
|
||||
|
except Exception as e: |
||||
|
print(f"❌ Transaction test failed: {e}") |
||||
|
|
||||
|
|
||||
|
def test_performance(host, port, user, database, password=None, iterations=10): |
||||
|
"""Test query performance.""" |
||||
|
print(f"\n⚡ Testing performance ({iterations} iterations)...") |
||||
|
|
||||
|
try: |
||||
|
conn_params = { |
||||
|
'host': host, |
||||
|
'port': port, |
||||
|
'user': user, |
||||
|
'database': database |
||||
|
} |
||||
|
if password: |
||||
|
conn_params['password'] = password |
||||
|
|
||||
|
times = [] |
||||
|
|
||||
|
for i in range(iterations): |
||||
|
start_time = time.time() |
||||
|
|
||||
|
conn = psycopg2.connect(**conn_params) |
||||
|
cursor = conn.cursor() |
||||
|
cursor.execute("SELECT 1") |
||||
|
result = cursor.fetchone() |
||||
|
cursor.close() |
||||
|
conn.close() |
||||
|
|
||||
|
elapsed = time.time() - start_time |
||||
|
times.append(elapsed) |
||||
|
|
||||
|
if i < 3: # Show first 3 iterations |
||||
|
print(f" Iteration {i+1}: {elapsed:.3f}s") |
||||
|
|
||||
|
avg_time = sum(times) / len(times) |
||||
|
min_time = min(times) |
||||
|
max_time = max(times) |
||||
|
|
||||
|
print(f" ✅ Performance results:") |
||||
|
print(f" - Average: {avg_time:.3f}s") |
||||
|
print(f" - Min: {min_time:.3f}s") |
||||
|
print(f" - Max: {max_time:.3f}s") |
||||
|
|
||||
|
except Exception as e: |
||||
|
print(f"❌ Performance test failed: {e}") |
||||
|
|
||||
|
|
||||
|
def main(): |
||||
|
parser = argparse.ArgumentParser(description="Test SeaweedFS PostgreSQL Protocol") |
||||
|
parser.add_argument("--host", default="localhost", help="PostgreSQL server host") |
||||
|
parser.add_argument("--port", type=int, default=5432, help="PostgreSQL server port") |
||||
|
parser.add_argument("--user", default="seaweedfs", help="PostgreSQL username") |
||||
|
parser.add_argument("--password", help="PostgreSQL password") |
||||
|
parser.add_argument("--database", default="default", help="PostgreSQL database") |
||||
|
parser.add_argument("--skip-performance", action="store_true", help="Skip performance tests") |
||||
|
|
||||
|
args = parser.parse_args() |
||||
|
|
||||
|
print("🧪 SeaweedFS PostgreSQL Protocol Test Client") |
||||
|
print("=" * 50) |
||||
|
|
||||
|
# Test basic connection first |
||||
|
if not test_connection(args.host, args.port, args.user, args.database, args.password): |
||||
|
print("\n❌ Basic connection failed. Cannot continue with other tests.") |
||||
|
sys.exit(1) |
||||
|
|
||||
|
# Run all tests |
||||
|
try: |
||||
|
test_system_queries(args.host, args.port, args.user, args.database, args.password) |
||||
|
test_schema_queries(args.host, args.port, args.user, args.database, args.password) |
||||
|
test_data_queries(args.host, args.port, args.user, args.database, args.password) |
||||
|
test_prepared_statements(args.host, args.port, args.user, args.database, args.password) |
||||
|
test_transaction_support(args.host, args.port, args.user, args.database, args.password) |
||||
|
|
||||
|
if not args.skip_performance: |
||||
|
test_performance(args.host, args.port, args.user, args.database, args.password) |
||||
|
|
||||
|
except KeyboardInterrupt: |
||||
|
print("\n\n⚠️ Tests interrupted by user") |
||||
|
sys.exit(0) |
||||
|
except Exception as e: |
||||
|
print(f"\n❌ Unexpected error during testing: {e}") |
||||
|
traceback.print_exc() |
||||
|
sys.exit(1) |
||||
|
|
||||
|
print("\n🎉 All tests completed!") |
||||
|
print("\nTo use SeaweedFS with PostgreSQL tools:") |
||||
|
print(f" psql -h {args.host} -p {args.port} -U {args.user} -d {args.database}") |
||||
|
print(f" Connection string: postgresql://{args.user}@{args.host}:{args.port}/{args.database}") |
||||
|
|
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
main() |
@ -0,0 +1,379 @@ |
|||||
|
package command |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"crypto/tls" |
||||
|
"fmt" |
||||
|
"os" |
||||
|
"os/signal" |
||||
|
"strconv" |
||||
|
"strings" |
||||
|
"syscall" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/server/postgres" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
postgresOptions PostgresOptions |
||||
|
) |
||||
|
|
||||
|
type PostgresOptions struct { |
||||
|
host *string |
||||
|
port *int |
||||
|
masterAddr *string |
||||
|
authMethod *string |
||||
|
users *string |
||||
|
database *string |
||||
|
maxConns *int |
||||
|
idleTimeout *string |
||||
|
tlsCert *string |
||||
|
tlsKey *string |
||||
|
} |
||||
|
|
||||
|
func init() { |
||||
|
cmdPostgres.Run = runPostgres // break init cycle
|
||||
|
postgresOptions.host = cmdPostgres.Flag.String("host", "localhost", "PostgreSQL server host") |
||||
|
postgresOptions.port = cmdPostgres.Flag.Int("port", 5432, "PostgreSQL server port") |
||||
|
postgresOptions.masterAddr = cmdPostgres.Flag.String("master", "localhost:9333", "SeaweedFS master server address") |
||||
|
postgresOptions.authMethod = cmdPostgres.Flag.String("auth", "trust", "Authentication method: trust, password, md5") |
||||
|
postgresOptions.users = cmdPostgres.Flag.String("users", "", "User credentials for auth (format: user1:pass1,user2:pass2)") |
||||
|
postgresOptions.database = cmdPostgres.Flag.String("database", "default", "Default database name") |
||||
|
postgresOptions.maxConns = cmdPostgres.Flag.Int("max-connections", 100, "Maximum concurrent connections") |
||||
|
postgresOptions.idleTimeout = cmdPostgres.Flag.String("idle-timeout", "1h", "Connection idle timeout") |
||||
|
postgresOptions.tlsCert = cmdPostgres.Flag.String("tls-cert", "", "TLS certificate file path") |
||||
|
postgresOptions.tlsKey = cmdPostgres.Flag.String("tls-key", "", "TLS private key file path") |
||||
|
} |
||||
|
|
||||
|
var cmdPostgres = &Command{ |
||||
|
UsageLine: "postgres -port=5432 -master=<master_server>", |
||||
|
Short: "start a PostgreSQL-compatible server for SQL queries", |
||||
|
Long: `Start a PostgreSQL wire protocol compatible server that provides SQL query access to SeaweedFS. |
||||
|
|
||||
|
This PostgreSQL server enables any PostgreSQL client, tool, or application to connect to SeaweedFS |
||||
|
and execute SQL queries against MQ topics. It implements the PostgreSQL wire protocol for maximum |
||||
|
compatibility with the existing PostgreSQL ecosystem. |
||||
|
|
||||
|
Examples: |
||||
|
|
||||
|
# Start PostgreSQL server on default port 5432 |
||||
|
weed postgres |
||||
|
|
||||
|
# Start with password authentication |
||||
|
weed postgres -auth=password -users="admin:secret,readonly:view123" |
||||
|
|
||||
|
# Start with MD5 authentication |
||||
|
weed postgres -auth=md5 -users="user1:pass1,user2:pass2" |
||||
|
|
||||
|
# Start with custom port and master |
||||
|
weed postgres -port=5433 -master=master1:9333 |
||||
|
|
||||
|
# Allow connections from any host |
||||
|
weed postgres -host=0.0.0.0 -port=5432 |
||||
|
|
||||
|
# Start with TLS encryption |
||||
|
weed postgres -tls-cert=server.crt -tls-key=server.key |
||||
|
|
||||
|
Client Connection Examples: |
||||
|
|
||||
|
# psql command line client |
||||
|
psql "host=localhost port=5432 dbname=default user=seaweedfs" |
||||
|
psql -h localhost -p 5432 -U seaweedfs -d default |
||||
|
|
||||
|
# With password |
||||
|
PGPASSWORD=secret psql -h localhost -p 5432 -U admin -d default |
||||
|
|
||||
|
# Connection string |
||||
|
psql "postgresql://admin:secret@localhost:5432/default" |
||||
|
|
||||
|
Programming Language Examples: |
||||
|
|
||||
|
# Python (psycopg2) |
||||
|
import psycopg2 |
||||
|
conn = psycopg2.connect( |
||||
|
host="localhost", port=5432, |
||||
|
user="seaweedfs", database="default" |
||||
|
) |
||||
|
|
||||
|
# Java JDBC |
||||
|
String url = "jdbc:postgresql://localhost:5432/default"; |
||||
|
Connection conn = DriverManager.getConnection(url, "seaweedfs", ""); |
||||
|
|
||||
|
# Go (lib/pq) |
||||
|
db, err := sql.Open("postgres", "host=localhost port=5432 user=seaweedfs dbname=default sslmode=disable") |
||||
|
|
||||
|
# Node.js (pg) |
||||
|
const client = new Client({ |
||||
|
host: 'localhost', port: 5432, |
||||
|
user: 'seaweedfs', database: 'default' |
||||
|
}); |
||||
|
|
||||
|
Supported SQL Operations: |
||||
|
- SELECT queries on MQ topics |
||||
|
- DESCRIBE/DESC table_name commands |
||||
|
- SHOW DATABASES/TABLES commands |
||||
|
- Aggregation functions (COUNT, SUM, AVG, MIN, MAX) |
||||
|
- WHERE clauses with filtering |
||||
|
- System columns (_timestamp_ns, _key, _source) |
||||
|
- PostgreSQL system queries (version(), current_database(), etc.) |
||||
|
- psql meta-commands (\d, \dt, \l, etc.) |
||||
|
|
||||
|
Authentication Methods: |
||||
|
- trust: No authentication required (default) |
||||
|
- password: Clear text password authentication |
||||
|
- md5: MD5 password authentication (more secure) |
||||
|
|
||||
|
Compatible Tools: |
||||
|
- psql (PostgreSQL command line client) |
||||
|
- pgAdmin (PostgreSQL admin tool) |
||||
|
- DBeaver (universal database tool) |
||||
|
- DataGrip (JetBrains database IDE) |
||||
|
- Grafana (PostgreSQL data source) |
||||
|
- Superset (PostgreSQL connector) |
||||
|
- Tableau (PostgreSQL native connector) |
||||
|
- Any PostgreSQL JDBC/ODBC compatible tool |
||||
|
|
||||
|
Security Features: |
||||
|
- Multiple authentication methods |
||||
|
- TLS encryption support |
||||
|
- User access control |
||||
|
- Connection limits |
||||
|
- Read-only access (no data modification) |
||||
|
|
||||
|
Performance Features: |
||||
|
- Connection pooling |
||||
|
- Configurable connection limits |
||||
|
- Idle connection timeout |
||||
|
- Efficient wire protocol |
||||
|
- Query result streaming |
||||
|
|
||||
|
`, |
||||
|
} |
||||
|
|
||||
|
func runPostgres(cmd *Command, args []string) bool { |
||||
|
|
||||
|
util.LoadConfiguration("security", false) |
||||
|
|
||||
|
// Validate options
|
||||
|
if *postgresOptions.masterAddr == "" { |
||||
|
fmt.Fprintf(os.Stderr, "Error: master address is required\n") |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
// Parse authentication method
|
||||
|
authMethod, err := parseAuthMethod(*postgresOptions.authMethod) |
||||
|
if err != nil { |
||||
|
fmt.Fprintf(os.Stderr, "Error: %v\n", err) |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
// Parse user credentials
|
||||
|
users, err := parseUsers(*postgresOptions.users, authMethod) |
||||
|
if err != nil { |
||||
|
fmt.Fprintf(os.Stderr, "Error: %v\n", err) |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
// Parse idle timeout
|
||||
|
idleTimeout, err := time.ParseDuration(*postgresOptions.idleTimeout) |
||||
|
if err != nil { |
||||
|
fmt.Fprintf(os.Stderr, "Error parsing idle timeout: %v\n", err) |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
// Setup TLS if requested
|
||||
|
var tlsConfig *tls.Config |
||||
|
if *postgresOptions.tlsCert != "" && *postgresOptions.tlsKey != "" { |
||||
|
cert, err := tls.LoadX509KeyPair(*postgresOptions.tlsCert, *postgresOptions.tlsKey) |
||||
|
if err != nil { |
||||
|
fmt.Fprintf(os.Stderr, "Error loading TLS certificates: %v\n", err) |
||||
|
return false |
||||
|
} |
||||
|
tlsConfig = &tls.Config{ |
||||
|
Certificates: []tls.Certificate{cert}, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Create server configuration
|
||||
|
config := &postgres.PostgreSQLServerConfig{ |
||||
|
Host: *postgresOptions.host, |
||||
|
Port: *postgresOptions.port, |
||||
|
AuthMethod: authMethod, |
||||
|
Users: users, |
||||
|
Database: *postgresOptions.database, |
||||
|
MaxConns: *postgresOptions.maxConns, |
||||
|
IdleTimeout: idleTimeout, |
||||
|
TLSConfig: tlsConfig, |
||||
|
} |
||||
|
|
||||
|
// Create PostgreSQL server
|
||||
|
postgresServer, err := postgres.NewPostgreSQLServer(config, *postgresOptions.masterAddr) |
||||
|
if err != nil { |
||||
|
fmt.Fprintf(os.Stderr, "Error creating PostgreSQL server: %v\n", err) |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
// Print startup information
|
||||
|
fmt.Printf("Starting SeaweedFS PostgreSQL Server...\n") |
||||
|
fmt.Printf("Host: %s\n", *postgresOptions.host) |
||||
|
fmt.Printf("Port: %d\n", *postgresOptions.port) |
||||
|
fmt.Printf("Master: %s\n", *postgresOptions.masterAddr) |
||||
|
fmt.Printf("Database: %s\n", *postgresOptions.database) |
||||
|
fmt.Printf("Auth Method: %s\n", *postgresOptions.authMethod) |
||||
|
fmt.Printf("Max Connections: %d\n", *postgresOptions.maxConns) |
||||
|
fmt.Printf("Idle Timeout: %s\n", *postgresOptions.idleTimeout) |
||||
|
if tlsConfig != nil { |
||||
|
fmt.Printf("TLS: Enabled\n") |
||||
|
} else { |
||||
|
fmt.Printf("TLS: Disabled\n") |
||||
|
} |
||||
|
if len(users) > 0 { |
||||
|
fmt.Printf("Users: %d configured\n", len(users)) |
||||
|
} |
||||
|
|
||||
|
fmt.Printf("\nPostgreSQL Connection Examples:\n") |
||||
|
fmt.Printf(" psql -h %s -p %d -U seaweedfs -d %s\n", *postgresOptions.host, *postgresOptions.port, *postgresOptions.database) |
||||
|
if len(users) > 0 { |
||||
|
// Show first user as example
|
||||
|
for username := range users { |
||||
|
fmt.Printf(" psql -h %s -p %d -U %s -d %s\n", *postgresOptions.host, *postgresOptions.port, username, *postgresOptions.database) |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
fmt.Printf(" postgresql://%s:%d/%s\n", *postgresOptions.host, *postgresOptions.port, *postgresOptions.database) |
||||
|
|
||||
|
fmt.Printf("\nSupported Operations:\n") |
||||
|
fmt.Printf(" - SELECT queries on MQ topics\n") |
||||
|
fmt.Printf(" - DESCRIBE/DESC table_name\n") |
||||
|
fmt.Printf(" - SHOW DATABASES/TABLES\n") |
||||
|
fmt.Printf(" - Aggregations: COUNT, SUM, AVG, MIN, MAX\n") |
||||
|
fmt.Printf(" - System columns: _timestamp_ns, _key, _source\n") |
||||
|
fmt.Printf(" - psql commands: \\d, \\dt, \\l, \\q\n") |
||||
|
|
||||
|
fmt.Printf("\nReady for PostgreSQL connections!\n\n") |
||||
|
|
||||
|
// Start the server
|
||||
|
err = postgresServer.Start() |
||||
|
if err != nil { |
||||
|
fmt.Fprintf(os.Stderr, "Error starting PostgreSQL server: %v\n", err) |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
// Set up signal handling for graceful shutdown
|
||||
|
sigChan := make(chan os.Signal, 1) |
||||
|
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) |
||||
|
|
||||
|
// Wait for shutdown signal
|
||||
|
<-sigChan |
||||
|
fmt.Printf("\nReceived shutdown signal, stopping PostgreSQL server...\n") |
||||
|
|
||||
|
// Create context with timeout for graceful shutdown
|
||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
||||
|
defer cancel() |
||||
|
|
||||
|
// Stop the server with timeout
|
||||
|
done := make(chan error, 1) |
||||
|
go func() { |
||||
|
done <- postgresServer.Stop() |
||||
|
}() |
||||
|
|
||||
|
select { |
||||
|
case err := <-done: |
||||
|
if err != nil { |
||||
|
fmt.Fprintf(os.Stderr, "Error stopping PostgreSQL server: %v\n", err) |
||||
|
return false |
||||
|
} |
||||
|
fmt.Printf("PostgreSQL server stopped successfully\n") |
||||
|
case <-ctx.Done(): |
||||
|
fmt.Fprintf(os.Stderr, "Timeout waiting for PostgreSQL server to stop\n") |
||||
|
return false |
||||
|
} |
||||
|
|
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
// parseAuthMethod parses the authentication method string
|
||||
|
func parseAuthMethod(method string) (postgres.AuthMethod, error) { |
||||
|
switch strings.ToLower(method) { |
||||
|
case "trust": |
||||
|
return postgres.AuthTrust, nil |
||||
|
case "password": |
||||
|
return postgres.AuthPassword, nil |
||||
|
case "md5": |
||||
|
return postgres.AuthMD5, nil |
||||
|
default: |
||||
|
return postgres.AuthTrust, fmt.Errorf("unsupported auth method '%s'. Supported: trust, password, md5", method) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// parseUsers parses the user credentials string
|
||||
|
func parseUsers(usersStr string, authMethod postgres.AuthMethod) (map[string]string, error) { |
||||
|
users := make(map[string]string) |
||||
|
|
||||
|
if usersStr == "" { |
||||
|
// No users specified
|
||||
|
if authMethod != postgres.AuthTrust { |
||||
|
return nil, fmt.Errorf("users must be specified when auth method is not 'trust'") |
||||
|
} |
||||
|
return users, nil |
||||
|
} |
||||
|
|
||||
|
// Parse user:password pairs
|
||||
|
pairs := strings.Split(usersStr, ",") |
||||
|
for _, pair := range pairs { |
||||
|
pair = strings.TrimSpace(pair) |
||||
|
if pair == "" { |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
parts := strings.SplitN(pair, ":", 2) |
||||
|
if len(parts) != 2 { |
||||
|
return nil, fmt.Errorf("invalid user format '%s'. Expected 'username:password'", pair) |
||||
|
} |
||||
|
|
||||
|
username := strings.TrimSpace(parts[0]) |
||||
|
password := strings.TrimSpace(parts[1]) |
||||
|
|
||||
|
if username == "" { |
||||
|
return nil, fmt.Errorf("empty username in user specification") |
||||
|
} |
||||
|
|
||||
|
if authMethod != postgres.AuthTrust && password == "" { |
||||
|
return nil, fmt.Errorf("empty password for user '%s' with auth method", username) |
||||
|
} |
||||
|
|
||||
|
users[username] = password |
||||
|
} |
||||
|
|
||||
|
return users, nil |
||||
|
} |
||||
|
|
||||
|
// validatePortNumber validates that the port number is reasonable
|
||||
|
func validatePortNumber(port int) error { |
||||
|
if port < 1 || port > 65535 { |
||||
|
return fmt.Errorf("port number must be between 1 and 65535, got %d", port) |
||||
|
} |
||||
|
if port < 1024 { |
||||
|
return fmt.Errorf("port number %d may require root privileges", port) |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// parseConnectionLimit parses and validates the connection limit
|
||||
|
func parseConnectionLimit(limitStr string) (int, error) { |
||||
|
limit, err := strconv.Atoi(limitStr) |
||||
|
if err != nil { |
||||
|
return 0, fmt.Errorf("invalid connection limit '%s': %v", limitStr, err) |
||||
|
} |
||||
|
|
||||
|
if limit < 1 { |
||||
|
return 0, fmt.Errorf("connection limit must be at least 1, got %d", limit) |
||||
|
} |
||||
|
|
||||
|
if limit > 10000 { |
||||
|
return 0, fmt.Errorf("connection limit too high (%d), maximum is 10000", limit) |
||||
|
} |
||||
|
|
||||
|
return limit, nil |
||||
|
} |
@ -0,0 +1,389 @@ |
|||||
|
# PostgreSQL Wire Protocol Support for SeaweedFS |
||||
|
|
||||
|
## Overview |
||||
|
|
||||
|
This design adds native PostgreSQL wire protocol support to SeaweedFS, enabling compatibility with all PostgreSQL clients, tools, and drivers without requiring custom implementations. |
||||
|
|
||||
|
## Benefits |
||||
|
|
||||
|
### Universal Compatibility |
||||
|
- **Standard PostgreSQL Clients**: psql, pgAdmin, Adminer, etc. |
||||
|
- **JDBC/ODBC Drivers**: Use standard PostgreSQL drivers |
||||
|
- **BI Tools**: Tableau, Power BI, Grafana, Superset with native PostgreSQL connectors |
||||
|
- **ORMs**: Hibernate, ActiveRecord, Django ORM, etc. |
||||
|
- **Programming Languages**: Native PostgreSQL libraries in Python (psycopg2), Node.js (pg), Go (lib/pq), etc. |
||||
|
|
||||
|
### Enterprise Integration |
||||
|
- **Existing Infrastructure**: Drop-in replacement for PostgreSQL in read-only scenarios |
||||
|
- **Migration Path**: Easy transition from PostgreSQL-based analytics |
||||
|
- **Tool Ecosystem**: Leverage entire PostgreSQL ecosystem |
||||
|
|
||||
|
## Architecture |
||||
|
|
||||
|
``` |
||||
|
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ |
||||
|
│ PostgreSQL │ │ PostgreSQL │ │ SeaweedFS │ |
||||
|
│ Clients │◄──►│ Protocol │◄──►│ SQL Engine │ |
||||
|
│ (psql, etc.) │ │ Server │ │ │ |
||||
|
└─────────────────┘ └──────────────────┘ └─────────────────┘ |
||||
|
│ |
||||
|
▼ |
||||
|
┌──────────────────┐ |
||||
|
│ Authentication │ |
||||
|
│ & Session Mgmt │ |
||||
|
└──────────────────┘ |
||||
|
``` |
||||
|
|
||||
|
## Core Components |
||||
|
|
||||
|
### 1. PostgreSQL Wire Protocol Handler |
||||
|
|
||||
|
```go |
||||
|
// PostgreSQL message types |
||||
|
const ( |
||||
|
PG_MSG_STARTUP = 0x00 // Startup message |
||||
|
PG_MSG_QUERY = 'Q' // Simple query |
||||
|
PG_MSG_PARSE = 'P' // Parse (prepared statement) |
||||
|
PG_MSG_BIND = 'B' // Bind parameters |
||||
|
PG_MSG_EXECUTE = 'E' // Execute prepared statement |
||||
|
PG_MSG_DESCRIBE = 'D' // Describe statement/portal |
||||
|
PG_MSG_CLOSE = 'C' // Close statement/portal |
||||
|
PG_MSG_FLUSH = 'H' // Flush |
||||
|
PG_MSG_SYNC = 'S' // Sync |
||||
|
PG_MSG_TERMINATE = 'X' // Terminate connection |
||||
|
PG_MSG_PASSWORD = 'p' // Password message |
||||
|
) |
||||
|
|
||||
|
// PostgreSQL response types |
||||
|
const ( |
||||
|
PG_RESP_AUTH_OK = 'R' // Authentication OK |
||||
|
PG_RESP_AUTH_REQ = 'R' // Authentication request |
||||
|
PG_RESP_BACKEND_KEY = 'K' // Backend key data |
||||
|
PG_RESP_PARAMETER = 'S' // Parameter status |
||||
|
PG_RESP_READY = 'Z' // Ready for query |
||||
|
PG_RESP_COMMAND = 'C' // Command complete |
||||
|
PG_RESP_DATA_ROW = 'D' // Data row |
||||
|
PG_RESP_ROW_DESC = 'T' // Row description |
||||
|
PG_RESP_PARSE_COMPLETE = '1' // Parse complete |
||||
|
PG_RESP_BIND_COMPLETE = '2' // Bind complete |
||||
|
PG_RESP_CLOSE_COMPLETE = '3' // Close complete |
||||
|
PG_RESP_ERROR = 'E' // Error response |
||||
|
PG_RESP_NOTICE = 'N' // Notice response |
||||
|
) |
||||
|
``` |
||||
|
|
||||
|
### 2. Session Management |
||||
|
|
||||
|
```go |
||||
|
type PostgreSQLSession struct { |
||||
|
conn net.Conn |
||||
|
reader *bufio.Reader |
||||
|
writer *bufio.Writer |
||||
|
authenticated bool |
||||
|
username string |
||||
|
database string |
||||
|
parameters map[string]string |
||||
|
preparedStmts map[string]*PreparedStatement |
||||
|
portals map[string]*Portal |
||||
|
transactionState TransactionState |
||||
|
processID uint32 |
||||
|
secretKey uint32 |
||||
|
} |
||||
|
|
||||
|
type PreparedStatement struct { |
||||
|
name string |
||||
|
query string |
||||
|
paramTypes []uint32 |
||||
|
fields []FieldDescription |
||||
|
} |
||||
|
|
||||
|
type Portal struct { |
||||
|
name string |
||||
|
statement string |
||||
|
parameters [][]byte |
||||
|
suspended bool |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
### 3. SQL Translation Layer |
||||
|
|
||||
|
```go |
||||
|
type PostgreSQLTranslator struct { |
||||
|
dialectMap map[string]string |
||||
|
} |
||||
|
|
||||
|
// Translates PostgreSQL-specific SQL to SeaweedFS SQL |
||||
|
func (t *PostgreSQLTranslator) TranslateQuery(pgSQL string) (string, error) { |
||||
|
// Handle PostgreSQL-specific syntax: |
||||
|
// - SELECT version() -> SELECT 'SeaweedFS 1.0' |
||||
|
// - SELECT current_database() -> SELECT 'default' |
||||
|
// - SELECT current_user -> SELECT 'seaweedfs' |
||||
|
// - \d commands -> SHOW TABLES/DESCRIBE equivalents |
||||
|
// - PostgreSQL system catalogs -> SeaweedFS equivalents |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
### 4. Data Type Mapping |
||||
|
|
||||
|
```go |
||||
|
var PostgreSQLTypeMap = map[string]uint32{ |
||||
|
"TEXT": 25, // PostgreSQL TEXT type |
||||
|
"VARCHAR": 1043, // PostgreSQL VARCHAR type |
||||
|
"INTEGER": 23, // PostgreSQL INTEGER type |
||||
|
"BIGINT": 20, // PostgreSQL BIGINT type |
||||
|
"FLOAT": 701, // PostgreSQL FLOAT8 type |
||||
|
"BOOLEAN": 16, // PostgreSQL BOOLEAN type |
||||
|
"TIMESTAMP": 1114, // PostgreSQL TIMESTAMP type |
||||
|
"JSON": 114, // PostgreSQL JSON type |
||||
|
} |
||||
|
|
||||
|
func SeaweedToPostgreSQLType(seaweedType string) uint32 { |
||||
|
if pgType, exists := PostgreSQLTypeMap[strings.ToUpper(seaweedType)]; exists { |
||||
|
return pgType |
||||
|
} |
||||
|
return 25 // Default to TEXT |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
## Protocol Implementation |
||||
|
|
||||
|
### 1. Connection Flow |
||||
|
|
||||
|
``` |
||||
|
Client Server |
||||
|
│ │ |
||||
|
├─ StartupMessage ────────────►│ |
||||
|
│ ├─ AuthenticationOk |
||||
|
│ ├─ ParameterStatus (multiple) |
||||
|
│ ├─ BackendKeyData |
||||
|
│ └─ ReadyForQuery |
||||
|
│ │ |
||||
|
├─ Query('SELECT 1') ─────────►│ |
||||
|
│ ├─ RowDescription |
||||
|
│ ├─ DataRow |
||||
|
│ ├─ CommandComplete |
||||
|
│ └─ ReadyForQuery |
||||
|
│ │ |
||||
|
├─ Parse('stmt1', 'SELECT $1')►│ |
||||
|
│ └─ ParseComplete |
||||
|
├─ Bind('portal1', 'stmt1')───►│ |
||||
|
│ └─ BindComplete |
||||
|
├─ Execute('portal1')─────────►│ |
||||
|
│ ├─ DataRow (multiple) |
||||
|
│ └─ CommandComplete |
||||
|
├─ Sync ──────────────────────►│ |
||||
|
│ └─ ReadyForQuery |
||||
|
│ │ |
||||
|
├─ Terminate ─────────────────►│ |
||||
|
│ └─ [Connection closed] |
||||
|
``` |
||||
|
|
||||
|
### 2. Authentication |
||||
|
|
||||
|
```go |
||||
|
type AuthMethod int |
||||
|
|
||||
|
const ( |
||||
|
AuthTrust AuthMethod = iota |
||||
|
AuthPassword |
||||
|
AuthMD5 |
||||
|
AuthSASL |
||||
|
) |
||||
|
|
||||
|
func (s *PostgreSQLServer) handleAuthentication(session *PostgreSQLSession) error { |
||||
|
switch s.authMethod { |
||||
|
case AuthTrust: |
||||
|
return s.sendAuthenticationOk(session) |
||||
|
case AuthPassword: |
||||
|
return s.handlePasswordAuth(session) |
||||
|
case AuthMD5: |
||||
|
return s.handleMD5Auth(session) |
||||
|
default: |
||||
|
return fmt.Errorf("unsupported auth method") |
||||
|
} |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
### 3. Query Processing |
||||
|
|
||||
|
```go |
||||
|
func (s *PostgreSQLServer) handleSimpleQuery(session *PostgreSQLSession, query string) error { |
||||
|
// 1. Translate PostgreSQL SQL to SeaweedFS SQL |
||||
|
translatedQuery, err := s.translator.TranslateQuery(query) |
||||
|
if err != nil { |
||||
|
return s.sendError(session, err) |
||||
|
} |
||||
|
|
||||
|
// 2. Execute using existing SQL engine |
||||
|
result, err := s.sqlEngine.ExecuteSQL(context.Background(), translatedQuery) |
||||
|
if err != nil { |
||||
|
return s.sendError(session, err) |
||||
|
} |
||||
|
|
||||
|
// 3. Send results in PostgreSQL format |
||||
|
err = s.sendRowDescription(session, result.Columns) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
for _, row := range result.Rows { |
||||
|
err = s.sendDataRow(session, row) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return s.sendCommandComplete(session, fmt.Sprintf("SELECT %d", len(result.Rows))) |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
## System Catalogs Support |
||||
|
|
||||
|
PostgreSQL clients expect certain system catalogs. We'll implement views for key ones: |
||||
|
|
||||
|
```sql |
||||
|
-- pg_tables equivalent |
||||
|
SELECT |
||||
|
'default' as schemaname, |
||||
|
table_name as tablename, |
||||
|
'seaweedfs' as tableowner, |
||||
|
NULL as tablespace, |
||||
|
false as hasindexes, |
||||
|
false as hasrules, |
||||
|
false as hastriggers |
||||
|
FROM information_schema.tables; |
||||
|
|
||||
|
-- pg_database equivalent |
||||
|
SELECT |
||||
|
database_name as datname, |
||||
|
'seaweedfs' as datdba, |
||||
|
'UTF8' as encoding, |
||||
|
'C' as datcollate, |
||||
|
'C' as datctype |
||||
|
FROM information_schema.schemata; |
||||
|
|
||||
|
-- pg_version equivalent |
||||
|
SELECT 'SeaweedFS 1.0 (PostgreSQL 14.0 compatible)' as version; |
||||
|
``` |
||||
|
|
||||
|
## Configuration |
||||
|
|
||||
|
### Server Configuration |
||||
|
```go |
||||
|
type PostgreSQLServerConfig struct { |
||||
|
Host string |
||||
|
Port int |
||||
|
Database string |
||||
|
AuthMethod AuthMethod |
||||
|
Users map[string]string // username -> password |
||||
|
TLSConfig *tls.Config |
||||
|
MaxConns int |
||||
|
IdleTimeout time.Duration |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
### Client Connection String |
||||
|
```bash |
||||
|
# Standard PostgreSQL connection strings work |
||||
|
psql "host=localhost port=5432 dbname=default user=seaweedfs" |
||||
|
PGPASSWORD=secret psql -h localhost -p 5432 -U seaweedfs -d default |
||||
|
|
||||
|
# JDBC URL |
||||
|
jdbc:postgresql://localhost:5432/default?user=seaweedfs&password=secret |
||||
|
``` |
||||
|
|
||||
|
## Command Line Interface |
||||
|
|
||||
|
```bash |
||||
|
# Start PostgreSQL protocol server |
||||
|
weed postgres -port=5432 -auth=trust |
||||
|
weed postgres -port=5432 -auth=password -users="admin:secret,readonly:pass" |
||||
|
weed postgres -port=5432 -tls-cert=server.crt -tls-key=server.key |
||||
|
|
||||
|
# Configuration options |
||||
|
-host=localhost # Listen host |
||||
|
-port=5432 # PostgreSQL standard port |
||||
|
-auth=trust|password|md5 # Authentication method |
||||
|
-users=user:pass,user2:pass2 # User credentials (password/md5 auth) |
||||
|
-database=default # Default database name |
||||
|
-max-connections=100 # Maximum concurrent connections |
||||
|
-idle-timeout=1h # Connection idle timeout |
||||
|
-tls-cert="" # TLS certificate file |
||||
|
-tls-key="" # TLS private key file |
||||
|
``` |
||||
|
|
||||
|
## Client Compatibility Testing |
||||
|
|
||||
|
### Essential Clients |
||||
|
- **psql**: PostgreSQL command line client |
||||
|
- **pgAdmin**: Web-based administration tool |
||||
|
- **DBeaver**: Universal database tool |
||||
|
- **DataGrip**: JetBrains database IDE |
||||
|
|
||||
|
### Programming Language Drivers |
||||
|
- **Python**: psycopg2, asyncpg |
||||
|
- **Java**: PostgreSQL JDBC driver |
||||
|
- **Node.js**: pg, node-postgres |
||||
|
- **Go**: lib/pq, pgx |
||||
|
- **.NET**: Npgsql |
||||
|
|
||||
|
### BI Tools |
||||
|
- **Grafana**: PostgreSQL data source |
||||
|
- **Superset**: PostgreSQL connector |
||||
|
- **Tableau**: PostgreSQL native connector |
||||
|
- **Power BI**: PostgreSQL connector |
||||
|
|
||||
|
## Implementation Plan |
||||
|
|
||||
|
1. **Phase 1**: Basic wire protocol and simple queries |
||||
|
2. **Phase 2**: Extended query protocol (prepared statements) |
||||
|
3. **Phase 3**: System catalog views |
||||
|
4. **Phase 4**: Advanced features (transactions, notifications) |
||||
|
5. **Phase 5**: Performance optimization and caching |
||||
|
|
||||
|
## Limitations |
||||
|
|
||||
|
### Read-Only Access |
||||
|
- INSERT/UPDATE/DELETE operations not supported |
||||
|
- Returns appropriate error messages for write operations |
||||
|
|
||||
|
### Partial SQL Compatibility |
||||
|
- Subset of PostgreSQL SQL features |
||||
|
- SeaweedFS-specific limitations apply |
||||
|
|
||||
|
### System Features |
||||
|
- No stored procedures/functions |
||||
|
- No triggers or constraints |
||||
|
- No user-defined types |
||||
|
- Limited transaction support (mostly no-op) |
||||
|
|
||||
|
## Security Considerations |
||||
|
|
||||
|
### Authentication |
||||
|
- Support for trust, password, and MD5 authentication |
||||
|
- TLS encryption support |
||||
|
- User access control |
||||
|
|
||||
|
### SQL Injection Prevention |
||||
|
- Prepared statements with parameter binding |
||||
|
- Input validation and sanitization |
||||
|
- Query complexity limits |
||||
|
|
||||
|
## Performance Optimizations |
||||
|
|
||||
|
### Connection Pooling |
||||
|
- Configurable maximum connections |
||||
|
- Connection reuse and idle timeout |
||||
|
- Memory efficient session management |
||||
|
|
||||
|
### Query Caching |
||||
|
- Prepared statement caching |
||||
|
- Result set caching for repeated queries |
||||
|
- Metadata caching |
||||
|
|
||||
|
### Protocol Efficiency |
||||
|
- Binary result format support |
||||
|
- Batch query processing |
||||
|
- Streaming large result sets |
||||
|
|
||||
|
This design provides a comprehensive PostgreSQL wire protocol implementation that makes SeaweedFS accessible to the entire PostgreSQL ecosystem while maintaining compatibility and performance. |
@ -0,0 +1,240 @@ |
|||||
|
# PostgreSQL Wire Protocol Package |
||||
|
|
||||
|
This package implements PostgreSQL wire protocol support for SeaweedFS, enabling universal compatibility with PostgreSQL clients, tools, and applications. |
||||
|
|
||||
|
## Package Structure |
||||
|
|
||||
|
``` |
||||
|
weed/server/postgres/ |
||||
|
├── README.md # This documentation |
||||
|
├── server.go # Main PostgreSQL server implementation |
||||
|
├── protocol.go # Wire protocol message handlers |
||||
|
├── translator.go # SQL translation layer |
||||
|
├── DESIGN.md # Architecture and design documentation |
||||
|
└── IMPLEMENTATION.md # Complete implementation guide |
||||
|
``` |
||||
|
|
||||
|
## Core Components |
||||
|
|
||||
|
### `server.go` |
||||
|
- **PostgreSQLServer**: Main server structure with connection management |
||||
|
- **PostgreSQLSession**: Individual client session handling |
||||
|
- **PostgreSQLServerConfig**: Server configuration options |
||||
|
- **Authentication System**: Trust, password, and MD5 authentication |
||||
|
- **TLS Support**: Encrypted connections with custom certificates |
||||
|
- **Connection Pooling**: Resource management and cleanup |
||||
|
|
||||
|
### `protocol.go` |
||||
|
- **Wire Protocol Implementation**: Full PostgreSQL 3.0 protocol support |
||||
|
- **Message Handlers**: Startup, query, parse/bind/execute sequences |
||||
|
- **Response Generation**: Row descriptions, data rows, command completion |
||||
|
- **Data Type Mapping**: SeaweedFS to PostgreSQL type conversion |
||||
|
- **Error Handling**: PostgreSQL-compliant error responses |
||||
|
|
||||
|
### `translator.go` |
||||
|
- **SQL Translation**: PostgreSQL to SeaweedFS SQL conversion |
||||
|
- **System Query Emulation**: version(), current_database(), current_user |
||||
|
- **Meta-Command Support**: psql commands (\d, \dt, \l, \q) |
||||
|
- **System Catalog Emulation**: pg_tables, pg_database, information_schema |
||||
|
- **Transaction Commands**: BEGIN/COMMIT/ROLLBACK (no-op for read-only) |
||||
|
|
||||
|
## Usage |
||||
|
|
||||
|
### Import the Package |
||||
|
```go |
||||
|
import "github.com/seaweedfs/seaweedfs/weed/server/postgres" |
||||
|
``` |
||||
|
|
||||
|
### Create and Start Server |
||||
|
```go |
||||
|
config := &postgres.PostgreSQLServerConfig{ |
||||
|
Host: "localhost", |
||||
|
Port: 5432, |
||||
|
AuthMethod: postgres.AuthMD5, |
||||
|
Users: map[string]string{"admin": "secret"}, |
||||
|
Database: "default", |
||||
|
MaxConns: 100, |
||||
|
IdleTimeout: time.Hour, |
||||
|
} |
||||
|
|
||||
|
server, err := postgres.NewPostgreSQLServer(config, "localhost:9333") |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
err = server.Start() |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Server is now accepting PostgreSQL connections |
||||
|
``` |
||||
|
|
||||
|
## Authentication Methods |
||||
|
|
||||
|
The package supports three authentication methods: |
||||
|
|
||||
|
### Trust Authentication |
||||
|
```go |
||||
|
AuthMethod: postgres.AuthTrust |
||||
|
``` |
||||
|
- No password required |
||||
|
- Suitable for development/testing |
||||
|
- Not recommended for production |
||||
|
|
||||
|
### Password Authentication |
||||
|
```go |
||||
|
AuthMethod: postgres.AuthPassword, |
||||
|
Users: map[string]string{"user": "password"} |
||||
|
``` |
||||
|
- Clear text password transmission |
||||
|
- Simple but less secure |
||||
|
- Requires TLS for production use |
||||
|
|
||||
|
### MD5 Authentication |
||||
|
```go |
||||
|
AuthMethod: postgres.AuthMD5, |
||||
|
Users: map[string]string{"user": "password"} |
||||
|
``` |
||||
|
- Secure hashed authentication with salt |
||||
|
- **Recommended for production** |
||||
|
- Compatible with all PostgreSQL clients |
||||
|
|
||||
|
## TLS Configuration |
||||
|
|
||||
|
Enable TLS encryption for secure connections: |
||||
|
|
||||
|
```go |
||||
|
cert, err := tls.LoadX509KeyPair("server.crt", "server.key") |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
config.TLSConfig = &tls.Config{ |
||||
|
Certificates: []tls.Certificate{cert}, |
||||
|
} |
||||
|
``` |
||||
|
|
||||
|
## Client Compatibility |
||||
|
|
||||
|
This implementation is compatible with: |
||||
|
|
||||
|
### Command Line Tools |
||||
|
- `psql` - PostgreSQL command line client |
||||
|
- `pgcli` - Enhanced command line with auto-completion |
||||
|
- Database IDEs (DataGrip, DBeaver) |
||||
|
|
||||
|
### Programming Languages |
||||
|
- **Python**: psycopg2, asyncpg |
||||
|
- **Java**: PostgreSQL JDBC driver |
||||
|
- **JavaScript**: pg (node-postgres) |
||||
|
- **Go**: lib/pq, pgx |
||||
|
- **.NET**: Npgsql |
||||
|
- **PHP**: pdo_pgsql |
||||
|
- **Ruby**: pg gem |
||||
|
|
||||
|
### BI Tools |
||||
|
- Tableau (native PostgreSQL connector) |
||||
|
- Power BI (PostgreSQL data source) |
||||
|
- Grafana (PostgreSQL plugin) |
||||
|
- Apache Superset |
||||
|
|
||||
|
## Supported SQL Operations |
||||
|
|
||||
|
### Data Queries |
||||
|
```sql |
||||
|
SELECT * FROM topic_name; |
||||
|
SELECT id, message FROM topic_name WHERE condition; |
||||
|
SELECT COUNT(*) FROM topic_name; |
||||
|
SELECT MIN(id), MAX(id), AVG(amount) FROM topic_name; |
||||
|
``` |
||||
|
|
||||
|
### Schema Information |
||||
|
```sql |
||||
|
SHOW DATABASES; |
||||
|
SHOW TABLES; |
||||
|
DESCRIBE topic_name; |
||||
|
DESC topic_name; |
||||
|
``` |
||||
|
|
||||
|
### System Information |
||||
|
```sql |
||||
|
SELECT version(); |
||||
|
SELECT current_database(); |
||||
|
SELECT current_user; |
||||
|
``` |
||||
|
|
||||
|
### System Columns |
||||
|
```sql |
||||
|
SELECT id, message, _timestamp_ns, _key, _source FROM topic_name; |
||||
|
``` |
||||
|
|
||||
|
## Configuration Options |
||||
|
|
||||
|
### Server Configuration |
||||
|
- **Host/Port**: Server binding address and port |
||||
|
- **Authentication**: Method and user credentials |
||||
|
- **Database**: Default database/namespace name |
||||
|
- **Connections**: Maximum concurrent connections |
||||
|
- **Timeouts**: Idle connection timeout |
||||
|
- **TLS**: Certificate and encryption settings |
||||
|
|
||||
|
### Performance Tuning |
||||
|
- **Connection Limits**: Prevent resource exhaustion |
||||
|
- **Idle Timeout**: Automatic cleanup of unused connections |
||||
|
- **Memory Management**: Efficient session handling |
||||
|
- **Query Streaming**: Large result set support |
||||
|
|
||||
|
## Error Handling |
||||
|
|
||||
|
The package provides PostgreSQL-compliant error responses: |
||||
|
|
||||
|
- **Connection Errors**: Authentication failures, network issues |
||||
|
- **SQL Errors**: Invalid syntax, missing tables |
||||
|
- **Resource Errors**: Connection limits, timeouts |
||||
|
- **Security Errors**: Permission denied, invalid credentials |
||||
|
|
||||
|
## Development and Testing |
||||
|
|
||||
|
### Unit Tests |
||||
|
Run PostgreSQL package tests: |
||||
|
```bash |
||||
|
go test ./weed/server/postgres |
||||
|
``` |
||||
|
|
||||
|
### Integration Testing |
||||
|
Use the provided Python test client: |
||||
|
```bash |
||||
|
python postgres-examples/test_client.py --host localhost --port 5432 |
||||
|
``` |
||||
|
|
||||
|
### Manual Testing |
||||
|
Connect with psql: |
||||
|
```bash |
||||
|
psql -h localhost -p 5432 -U seaweedfs -d default |
||||
|
``` |
||||
|
|
||||
|
## Documentation |
||||
|
|
||||
|
- **DESIGN.md**: Complete architecture and design overview |
||||
|
- **IMPLEMENTATION.md**: Detailed implementation guide |
||||
|
- **postgres-examples/**: Client examples and test scripts |
||||
|
- **Command Documentation**: `weed postgres -help` |
||||
|
|
||||
|
## Security Considerations |
||||
|
|
||||
|
### Production Deployment |
||||
|
- Use MD5 or stronger authentication |
||||
|
- Enable TLS encryption |
||||
|
- Configure appropriate connection limits |
||||
|
- Monitor for suspicious activity |
||||
|
- Use strong passwords |
||||
|
- Implement proper firewall rules |
||||
|
|
||||
|
### Access Control |
||||
|
- Create dedicated read-only users |
||||
|
- Use principle of least privilege |
||||
|
- Monitor connection patterns |
||||
|
- Log authentication attempts |
||||
|
|
||||
|
This package provides enterprise-grade PostgreSQL compatibility, enabling seamless integration of SeaweedFS with the entire PostgreSQL ecosystem. |
@ -0,0 +1,529 @@ |
|||||
|
package postgres |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"encoding/binary" |
||||
|
"fmt" |
||||
|
"io" |
||||
|
"strconv" |
||||
|
"strings" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/glog" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/query/sqltypes" |
||||
|
) |
||||
|
|
||||
|
// handleMessage processes a single PostgreSQL protocol message
|
||||
|
func (s *PostgreSQLServer) handleMessage(session *PostgreSQLSession) error { |
||||
|
// Read message type
|
||||
|
msgType := make([]byte, 1) |
||||
|
_, err := io.ReadFull(session.reader, msgType) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Read message length
|
||||
|
length := make([]byte, 4) |
||||
|
_, err = io.ReadFull(session.reader, length) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
msgLength := binary.BigEndian.Uint32(length) - 4 |
||||
|
msgBody := make([]byte, msgLength) |
||||
|
if msgLength > 0 { |
||||
|
_, err = io.ReadFull(session.reader, msgBody) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Process message based on type
|
||||
|
switch msgType[0] { |
||||
|
case PG_MSG_QUERY: |
||||
|
return s.handleSimpleQuery(session, string(msgBody[:len(msgBody)-1])) // Remove null terminator
|
||||
|
case PG_MSG_PARSE: |
||||
|
return s.handleParse(session, msgBody) |
||||
|
case PG_MSG_BIND: |
||||
|
return s.handleBind(session, msgBody) |
||||
|
case PG_MSG_EXECUTE: |
||||
|
return s.handleExecute(session, msgBody) |
||||
|
case PG_MSG_DESCRIBE: |
||||
|
return s.handleDescribe(session, msgBody) |
||||
|
case PG_MSG_CLOSE: |
||||
|
return s.handleClose(session, msgBody) |
||||
|
case PG_MSG_FLUSH: |
||||
|
return s.handleFlush(session) |
||||
|
case PG_MSG_SYNC: |
||||
|
return s.handleSync(session) |
||||
|
case PG_MSG_TERMINATE: |
||||
|
return io.EOF // Signal connection termination
|
||||
|
default: |
||||
|
return s.sendError(session, "08P01", fmt.Sprintf("unknown message type: %c", msgType[0])) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// handleSimpleQuery processes a simple query message
|
||||
|
func (s *PostgreSQLServer) handleSimpleQuery(session *PostgreSQLSession, query string) error { |
||||
|
glog.V(2).Infof("PostgreSQL Query (ID: %d): %s", session.processID, query) |
||||
|
|
||||
|
// Translate PostgreSQL SQL to SeaweedFS SQL
|
||||
|
translatedQuery, err := s.translator.TranslateQuery(query) |
||||
|
if err != nil { |
||||
|
return s.sendError(session, "42601", err.Error()) |
||||
|
} |
||||
|
|
||||
|
// Execute using SQL engine
|
||||
|
ctx := context.Background() |
||||
|
result, err := s.sqlEngine.ExecuteSQL(ctx, translatedQuery) |
||||
|
if err != nil { |
||||
|
return s.sendError(session, "42000", err.Error()) |
||||
|
} |
||||
|
|
||||
|
if result.Error != nil { |
||||
|
return s.sendError(session, "42000", result.Error.Error()) |
||||
|
} |
||||
|
|
||||
|
// Send results
|
||||
|
if len(result.Columns) > 0 { |
||||
|
// Send row description
|
||||
|
err = s.sendRowDescription(session, result.Columns, result.Rows) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Send data rows
|
||||
|
for _, row := range result.Rows { |
||||
|
err = s.sendDataRow(session, row) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Send command complete
|
||||
|
tag := s.getCommandTag(query, len(result.Rows)) |
||||
|
err = s.sendCommandComplete(session, tag) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Send ready for query
|
||||
|
return s.sendReadyForQuery(session) |
||||
|
} |
||||
|
|
||||
|
// handleParse processes a Parse message (prepared statement)
|
||||
|
func (s *PostgreSQLServer) handleParse(session *PostgreSQLSession, msgBody []byte) error { |
||||
|
// Parse message format: statement_name\0query\0param_count(int16)[param_type(int32)...]
|
||||
|
parts := strings.Split(string(msgBody), "\x00") |
||||
|
if len(parts) < 2 { |
||||
|
return s.sendError(session, "08P01", "invalid Parse message format") |
||||
|
} |
||||
|
|
||||
|
stmtName := parts[0] |
||||
|
query := parts[1] |
||||
|
|
||||
|
// Create prepared statement
|
||||
|
stmt := &PreparedStatement{ |
||||
|
Name: stmtName, |
||||
|
Query: query, |
||||
|
ParamTypes: []uint32{}, |
||||
|
Fields: []FieldDescription{}, |
||||
|
} |
||||
|
|
||||
|
session.preparedStmts[stmtName] = stmt |
||||
|
|
||||
|
// Send parse complete
|
||||
|
return s.sendParseComplete(session) |
||||
|
} |
||||
|
|
||||
|
// handleBind processes a Bind message
|
||||
|
func (s *PostgreSQLServer) handleBind(session *PostgreSQLSession, msgBody []byte) error { |
||||
|
// For now, simple implementation
|
||||
|
// In full implementation, would parse parameters and create portal
|
||||
|
|
||||
|
// Send bind complete
|
||||
|
return s.sendBindComplete(session) |
||||
|
} |
||||
|
|
||||
|
// handleExecute processes an Execute message
|
||||
|
func (s *PostgreSQLServer) handleExecute(session *PostgreSQLSession, msgBody []byte) error { |
||||
|
// Parse portal name
|
||||
|
parts := strings.Split(string(msgBody), "\x00") |
||||
|
if len(parts) == 0 { |
||||
|
return s.sendError(session, "08P01", "invalid Execute message format") |
||||
|
} |
||||
|
|
||||
|
portalName := parts[0] |
||||
|
|
||||
|
// For now, execute as simple query
|
||||
|
// In full implementation, would use portal with parameters
|
||||
|
glog.V(2).Infof("PostgreSQL Execute portal (ID: %d): %s", session.processID, portalName) |
||||
|
|
||||
|
// Send command complete
|
||||
|
err := s.sendCommandComplete(session, "SELECT 0") |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// handleDescribe processes a Describe message
|
||||
|
func (s *PostgreSQLServer) handleDescribe(session *PostgreSQLSession, msgBody []byte) error { |
||||
|
if len(msgBody) < 2 { |
||||
|
return s.sendError(session, "08P01", "invalid Describe message format") |
||||
|
} |
||||
|
|
||||
|
objectType := msgBody[0] // 'S' for statement, 'P' for portal
|
||||
|
objectName := string(msgBody[1:]) |
||||
|
|
||||
|
glog.V(2).Infof("PostgreSQL Describe %c (ID: %d): %s", objectType, session.processID, objectName) |
||||
|
|
||||
|
// For now, send empty row description
|
||||
|
return s.sendRowDescription(session, []string{}, [][]sqltypes.Value{}) |
||||
|
} |
||||
|
|
||||
|
// handleClose processes a Close message
|
||||
|
func (s *PostgreSQLServer) handleClose(session *PostgreSQLSession, msgBody []byte) error { |
||||
|
if len(msgBody) < 2 { |
||||
|
return s.sendError(session, "08P01", "invalid Close message format") |
||||
|
} |
||||
|
|
||||
|
objectType := msgBody[0] // 'S' for statement, 'P' for portal
|
||||
|
objectName := string(msgBody[1:]) |
||||
|
|
||||
|
switch objectType { |
||||
|
case 'S': |
||||
|
delete(session.preparedStmts, objectName) |
||||
|
case 'P': |
||||
|
delete(session.portals, objectName) |
||||
|
} |
||||
|
|
||||
|
// Send close complete
|
||||
|
return s.sendCloseComplete(session) |
||||
|
} |
||||
|
|
||||
|
// handleFlush processes a Flush message
|
||||
|
func (s *PostgreSQLServer) handleFlush(session *PostgreSQLSession) error { |
||||
|
return session.writer.Flush() |
||||
|
} |
||||
|
|
||||
|
// handleSync processes a Sync message
|
||||
|
func (s *PostgreSQLServer) handleSync(session *PostgreSQLSession) error { |
||||
|
// Reset transaction state if needed
|
||||
|
session.transactionState = PG_TRANS_IDLE |
||||
|
|
||||
|
// Send ready for query
|
||||
|
return s.sendReadyForQuery(session) |
||||
|
} |
||||
|
|
||||
|
// sendParameterStatus sends a parameter status message
|
||||
|
func (s *PostgreSQLServer) sendParameterStatus(session *PostgreSQLSession, name, value string) error { |
||||
|
msg := make([]byte, 0) |
||||
|
msg = append(msg, PG_RESP_PARAMETER) |
||||
|
|
||||
|
// Calculate length
|
||||
|
length := 4 + len(name) + 1 + len(value) + 1 |
||||
|
lengthBytes := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(lengthBytes, uint32(length)) |
||||
|
msg = append(msg, lengthBytes...) |
||||
|
|
||||
|
// Add name and value
|
||||
|
msg = append(msg, []byte(name)...) |
||||
|
msg = append(msg, 0) // null terminator
|
||||
|
msg = append(msg, []byte(value)...) |
||||
|
msg = append(msg, 0) // null terminator
|
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendBackendKeyData sends backend key data
|
||||
|
func (s *PostgreSQLServer) sendBackendKeyData(session *PostgreSQLSession) error { |
||||
|
msg := make([]byte, 12) |
||||
|
msg[0] = PG_RESP_BACKEND_KEY |
||||
|
binary.BigEndian.PutUint32(msg[1:5], 12) |
||||
|
binary.BigEndian.PutUint32(msg[5:9], session.processID) |
||||
|
binary.BigEndian.PutUint32(msg[9:13], session.secretKey) |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendReadyForQuery sends ready for query message
|
||||
|
func (s *PostgreSQLServer) sendReadyForQuery(session *PostgreSQLSession) error { |
||||
|
msg := make([]byte, 5) |
||||
|
msg[0] = PG_RESP_READY |
||||
|
binary.BigEndian.PutUint32(msg[1:5], 5) |
||||
|
msg[5] = session.transactionState |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendRowDescription sends row description message
|
||||
|
func (s *PostgreSQLServer) sendRowDescription(session *PostgreSQLSession, columns []string, rows [][]sqltypes.Value) error { |
||||
|
msg := make([]byte, 0) |
||||
|
msg = append(msg, PG_RESP_ROW_DESC) |
||||
|
|
||||
|
// Calculate message length
|
||||
|
length := 4 + 2 // length + field count
|
||||
|
for _, col := range columns { |
||||
|
length += len(col) + 1 + 4 + 2 + 4 + 2 + 4 + 2 // name + null + tableOID + attrNum + typeOID + typeSize + typeMod + format
|
||||
|
} |
||||
|
|
||||
|
lengthBytes := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(lengthBytes, uint32(length)) |
||||
|
msg = append(msg, lengthBytes...) |
||||
|
|
||||
|
// Field count
|
||||
|
fieldCountBytes := make([]byte, 2) |
||||
|
binary.BigEndian.PutUint16(fieldCountBytes, uint16(len(columns))) |
||||
|
msg = append(msg, fieldCountBytes...) |
||||
|
|
||||
|
// Field descriptions
|
||||
|
for i, col := range columns { |
||||
|
// Field name
|
||||
|
msg = append(msg, []byte(col)...) |
||||
|
msg = append(msg, 0) // null terminator
|
||||
|
|
||||
|
// Table OID (0 for no table)
|
||||
|
tableOID := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(tableOID, 0) |
||||
|
msg = append(msg, tableOID...) |
||||
|
|
||||
|
// Attribute number
|
||||
|
attrNum := make([]byte, 2) |
||||
|
binary.BigEndian.PutUint16(attrNum, uint16(i+1)) |
||||
|
msg = append(msg, attrNum...) |
||||
|
|
||||
|
// Type OID (determine from data)
|
||||
|
typeOID := s.getPostgreSQLType(columns, rows, i) |
||||
|
typeOIDBytes := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(typeOIDBytes, typeOID) |
||||
|
msg = append(msg, typeOIDBytes...) |
||||
|
|
||||
|
// Type size (-1 for variable length)
|
||||
|
typeSize := make([]byte, 2) |
||||
|
binary.BigEndian.PutUint16(typeSize, 0xFFFF) // -1 as uint16
|
||||
|
msg = append(msg, typeSize...) |
||||
|
|
||||
|
// Type modifier (-1 for default)
|
||||
|
typeMod := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(typeMod, 0xFFFFFFFF) // -1 as uint32
|
||||
|
msg = append(msg, typeMod...) |
||||
|
|
||||
|
// Format (0 for text)
|
||||
|
format := make([]byte, 2) |
||||
|
binary.BigEndian.PutUint16(format, 0) |
||||
|
msg = append(msg, format...) |
||||
|
} |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendDataRow sends a data row message
|
||||
|
func (s *PostgreSQLServer) sendDataRow(session *PostgreSQLSession, row []sqltypes.Value) error { |
||||
|
msg := make([]byte, 0) |
||||
|
msg = append(msg, PG_RESP_DATA_ROW) |
||||
|
|
||||
|
// Calculate message length
|
||||
|
length := 4 + 2 // length + field count
|
||||
|
for _, value := range row { |
||||
|
if value.IsNull() { |
||||
|
length += 4 // null value length (-1)
|
||||
|
} else { |
||||
|
valueStr := value.ToString() |
||||
|
length += 4 + len(valueStr) // field length + data
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
lengthBytes := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(lengthBytes, uint32(length)) |
||||
|
msg = append(msg, lengthBytes...) |
||||
|
|
||||
|
// Field count
|
||||
|
fieldCountBytes := make([]byte, 2) |
||||
|
binary.BigEndian.PutUint16(fieldCountBytes, uint16(len(row))) |
||||
|
msg = append(msg, fieldCountBytes...) |
||||
|
|
||||
|
// Field values
|
||||
|
for _, value := range row { |
||||
|
if value.IsNull() { |
||||
|
// Null value
|
||||
|
nullLength := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(nullLength, 0xFFFFFFFF) // -1 as uint32
|
||||
|
msg = append(msg, nullLength...) |
||||
|
} else { |
||||
|
valueStr := value.ToString() |
||||
|
valueLength := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(valueLength, uint32(len(valueStr))) |
||||
|
msg = append(msg, valueLength...) |
||||
|
msg = append(msg, []byte(valueStr)...) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendCommandComplete sends command complete message
|
||||
|
func (s *PostgreSQLServer) sendCommandComplete(session *PostgreSQLSession, tag string) error { |
||||
|
msg := make([]byte, 0) |
||||
|
msg = append(msg, PG_RESP_COMMAND) |
||||
|
|
||||
|
length := 4 + len(tag) + 1 |
||||
|
lengthBytes := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(lengthBytes, uint32(length)) |
||||
|
msg = append(msg, lengthBytes...) |
||||
|
|
||||
|
msg = append(msg, []byte(tag)...) |
||||
|
msg = append(msg, 0) // null terminator
|
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendParseComplete sends parse complete message
|
||||
|
func (s *PostgreSQLServer) sendParseComplete(session *PostgreSQLSession) error { |
||||
|
msg := make([]byte, 5) |
||||
|
msg[0] = PG_RESP_PARSE_COMPLETE |
||||
|
binary.BigEndian.PutUint32(msg[1:5], 4) |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendBindComplete sends bind complete message
|
||||
|
func (s *PostgreSQLServer) sendBindComplete(session *PostgreSQLSession) error { |
||||
|
msg := make([]byte, 5) |
||||
|
msg[0] = PG_RESP_BIND_COMPLETE |
||||
|
binary.BigEndian.PutUint32(msg[1:5], 4) |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendCloseComplete sends close complete message
|
||||
|
func (s *PostgreSQLServer) sendCloseComplete(session *PostgreSQLSession) error { |
||||
|
msg := make([]byte, 5) |
||||
|
msg[0] = PG_RESP_CLOSE_COMPLETE |
||||
|
binary.BigEndian.PutUint32(msg[1:5], 4) |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// sendError sends an error message
|
||||
|
func (s *PostgreSQLServer) sendError(session *PostgreSQLSession, code, message string) error { |
||||
|
msg := make([]byte, 0) |
||||
|
msg = append(msg, PG_RESP_ERROR) |
||||
|
|
||||
|
// Build error fields
|
||||
|
fields := fmt.Sprintf("S%s\x00C%s\x00M%s\x00\x00", "ERROR", code, message) |
||||
|
length := 4 + len(fields) |
||||
|
|
||||
|
lengthBytes := make([]byte, 4) |
||||
|
binary.BigEndian.PutUint32(lengthBytes, uint32(length)) |
||||
|
msg = append(msg, lengthBytes...) |
||||
|
msg = append(msg, []byte(fields)...) |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// getCommandTag generates appropriate command tag for query
|
||||
|
func (s *PostgreSQLServer) getCommandTag(query string, rowCount int) string { |
||||
|
queryUpper := strings.ToUpper(strings.TrimSpace(query)) |
||||
|
|
||||
|
if strings.HasPrefix(queryUpper, "SELECT") { |
||||
|
return fmt.Sprintf("SELECT %d", rowCount) |
||||
|
} else if strings.HasPrefix(queryUpper, "INSERT") { |
||||
|
return fmt.Sprintf("INSERT 0 %d", rowCount) |
||||
|
} else if strings.HasPrefix(queryUpper, "UPDATE") { |
||||
|
return fmt.Sprintf("UPDATE %d", rowCount) |
||||
|
} else if strings.HasPrefix(queryUpper, "DELETE") { |
||||
|
return fmt.Sprintf("DELETE %d", rowCount) |
||||
|
} else if strings.HasPrefix(queryUpper, "SHOW") { |
||||
|
return fmt.Sprintf("SELECT %d", rowCount) |
||||
|
} else if strings.HasPrefix(queryUpper, "DESCRIBE") || strings.HasPrefix(queryUpper, "DESC") { |
||||
|
return fmt.Sprintf("SELECT %d", rowCount) |
||||
|
} |
||||
|
|
||||
|
return "SELECT 0" |
||||
|
} |
||||
|
|
||||
|
// getPostgreSQLType determines PostgreSQL type OID from data
|
||||
|
func (s *PostgreSQLServer) getPostgreSQLType(columns []string, rows [][]sqltypes.Value, colIndex int) uint32 { |
||||
|
if len(rows) == 0 || colIndex >= len(rows[0]) { |
||||
|
return PG_TYPE_TEXT // Default to text
|
||||
|
} |
||||
|
|
||||
|
// Sample first non-null value to determine type
|
||||
|
for _, row := range rows { |
||||
|
if colIndex < len(row) && !row[colIndex].IsNull() { |
||||
|
value := row[colIndex] |
||||
|
switch value.Type() { |
||||
|
case sqltypes.Int8, sqltypes.Int16, sqltypes.Int32: |
||||
|
return PG_TYPE_INT4 |
||||
|
case sqltypes.Int64: |
||||
|
return PG_TYPE_INT8 |
||||
|
case sqltypes.Float32, sqltypes.Float64: |
||||
|
return PG_TYPE_FLOAT8 |
||||
|
case sqltypes.Bit: |
||||
|
return PG_TYPE_BOOL |
||||
|
case sqltypes.Timestamp, sqltypes.Datetime: |
||||
|
return PG_TYPE_TIMESTAMP |
||||
|
default: |
||||
|
// Try to infer from string content
|
||||
|
valueStr := value.ToString() |
||||
|
if _, err := strconv.ParseInt(valueStr, 10, 32); err == nil { |
||||
|
return PG_TYPE_INT4 |
||||
|
} |
||||
|
if _, err := strconv.ParseInt(valueStr, 10, 64); err == nil { |
||||
|
return PG_TYPE_INT8 |
||||
|
} |
||||
|
if _, err := strconv.ParseFloat(valueStr, 64); err == nil { |
||||
|
return PG_TYPE_FLOAT8 |
||||
|
} |
||||
|
if valueStr == "true" || valueStr == "false" { |
||||
|
return PG_TYPE_BOOL |
||||
|
} |
||||
|
return PG_TYPE_TEXT |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return PG_TYPE_TEXT // Default to text
|
||||
|
} |
@ -0,0 +1,640 @@ |
|||||
|
package postgres |
||||
|
|
||||
|
import ( |
||||
|
"bufio" |
||||
|
"crypto/md5" |
||||
|
"crypto/rand" |
||||
|
"crypto/tls" |
||||
|
"encoding/binary" |
||||
|
"fmt" |
||||
|
"io" |
||||
|
"net" |
||||
|
"strings" |
||||
|
"sync" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/glog" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/query/engine" |
||||
|
) |
||||
|
|
||||
|
// PostgreSQL protocol constants
|
||||
|
const ( |
||||
|
// Message types from client
|
||||
|
PG_MSG_STARTUP = 0x00 |
||||
|
PG_MSG_QUERY = 'Q' |
||||
|
PG_MSG_PARSE = 'P' |
||||
|
PG_MSG_BIND = 'B' |
||||
|
PG_MSG_EXECUTE = 'E' |
||||
|
PG_MSG_DESCRIBE = 'D' |
||||
|
PG_MSG_CLOSE = 'C' |
||||
|
PG_MSG_FLUSH = 'H' |
||||
|
PG_MSG_SYNC = 'S' |
||||
|
PG_MSG_TERMINATE = 'X' |
||||
|
PG_MSG_PASSWORD = 'p' |
||||
|
|
||||
|
// Response types to client
|
||||
|
PG_RESP_AUTH_OK = 'R' |
||||
|
PG_RESP_BACKEND_KEY = 'K' |
||||
|
PG_RESP_PARAMETER = 'S' |
||||
|
PG_RESP_READY = 'Z' |
||||
|
PG_RESP_COMMAND = 'C' |
||||
|
PG_RESP_DATA_ROW = 'D' |
||||
|
PG_RESP_ROW_DESC = 'T' |
||||
|
PG_RESP_PARSE_COMPLETE = '1' |
||||
|
PG_RESP_BIND_COMPLETE = '2' |
||||
|
PG_RESP_CLOSE_COMPLETE = '3' |
||||
|
PG_RESP_ERROR = 'E' |
||||
|
PG_RESP_NOTICE = 'N' |
||||
|
|
||||
|
// Transaction states
|
||||
|
PG_TRANS_IDLE = 'I' |
||||
|
PG_TRANS_INTRANS = 'T' |
||||
|
PG_TRANS_ERROR = 'E' |
||||
|
|
||||
|
// Authentication methods
|
||||
|
AUTH_OK = 0 |
||||
|
AUTH_CLEAR = 3 |
||||
|
AUTH_MD5 = 5 |
||||
|
AUTH_TRUST = 10 |
||||
|
|
||||
|
// PostgreSQL data types
|
||||
|
PG_TYPE_BOOL = 16 |
||||
|
PG_TYPE_INT8 = 20 |
||||
|
PG_TYPE_INT4 = 23 |
||||
|
PG_TYPE_TEXT = 25 |
||||
|
PG_TYPE_FLOAT8 = 701 |
||||
|
PG_TYPE_VARCHAR = 1043 |
||||
|
PG_TYPE_TIMESTAMP = 1114 |
||||
|
PG_TYPE_JSON = 114 |
||||
|
|
||||
|
// Default values
|
||||
|
DEFAULT_POSTGRES_PORT = 5432 |
||||
|
) |
||||
|
|
||||
|
// Authentication method type
|
||||
|
type AuthMethod int |
||||
|
|
||||
|
const ( |
||||
|
AuthTrust AuthMethod = iota |
||||
|
AuthPassword |
||||
|
AuthMD5 |
||||
|
) |
||||
|
|
||||
|
// PostgreSQL server configuration
|
||||
|
type PostgreSQLServerConfig struct { |
||||
|
Host string |
||||
|
Port int |
||||
|
AuthMethod AuthMethod |
||||
|
Users map[string]string |
||||
|
TLSConfig *tls.Config |
||||
|
MaxConns int |
||||
|
IdleTimeout time.Duration |
||||
|
Database string |
||||
|
} |
||||
|
|
||||
|
// PostgreSQL server
|
||||
|
type PostgreSQLServer struct { |
||||
|
config *PostgreSQLServerConfig |
||||
|
listener net.Listener |
||||
|
sqlEngine *engine.SQLEngine |
||||
|
sessions map[uint32]*PostgreSQLSession |
||||
|
sessionMux sync.RWMutex |
||||
|
shutdown chan struct{} |
||||
|
wg sync.WaitGroup |
||||
|
translator *PostgreSQLTranslator |
||||
|
nextConnID uint32 |
||||
|
} |
||||
|
|
||||
|
// PostgreSQL session
|
||||
|
type PostgreSQLSession struct { |
||||
|
conn net.Conn |
||||
|
reader *bufio.Reader |
||||
|
writer *bufio.Writer |
||||
|
authenticated bool |
||||
|
username string |
||||
|
database string |
||||
|
parameters map[string]string |
||||
|
preparedStmts map[string]*PreparedStatement |
||||
|
portals map[string]*Portal |
||||
|
transactionState byte |
||||
|
processID uint32 |
||||
|
secretKey uint32 |
||||
|
created time.Time |
||||
|
lastActivity time.Time |
||||
|
mutex sync.Mutex |
||||
|
} |
||||
|
|
||||
|
// Prepared statement
|
||||
|
type PreparedStatement struct { |
||||
|
Name string |
||||
|
Query string |
||||
|
ParamTypes []uint32 |
||||
|
Fields []FieldDescription |
||||
|
} |
||||
|
|
||||
|
// Portal (cursor)
|
||||
|
type Portal struct { |
||||
|
Name string |
||||
|
Statement string |
||||
|
Parameters [][]byte |
||||
|
Suspended bool |
||||
|
} |
||||
|
|
||||
|
// Field description
|
||||
|
type FieldDescription struct { |
||||
|
Name string |
||||
|
TableOID uint32 |
||||
|
AttrNum int16 |
||||
|
TypeOID uint32 |
||||
|
TypeSize int16 |
||||
|
TypeMod int32 |
||||
|
Format int16 |
||||
|
} |
||||
|
|
||||
|
// NewPostgreSQLServer creates a new PostgreSQL protocol server
|
||||
|
func NewPostgreSQLServer(config *PostgreSQLServerConfig, masterAddr string) (*PostgreSQLServer, error) { |
||||
|
if config.Port <= 0 { |
||||
|
config.Port = DEFAULT_POSTGRES_PORT |
||||
|
} |
||||
|
if config.Host == "" { |
||||
|
config.Host = "localhost" |
||||
|
} |
||||
|
if config.Database == "" { |
||||
|
config.Database = "default" |
||||
|
} |
||||
|
if config.MaxConns <= 0 { |
||||
|
config.MaxConns = 100 |
||||
|
} |
||||
|
if config.IdleTimeout <= 0 { |
||||
|
config.IdleTimeout = time.Hour |
||||
|
} |
||||
|
|
||||
|
// Create SQL engine
|
||||
|
sqlEngine := engine.NewSQLEngine(masterAddr) |
||||
|
|
||||
|
// Initialize translator
|
||||
|
translator := &PostgreSQLTranslator{ |
||||
|
systemQueries: make(map[string]string), |
||||
|
} |
||||
|
translator.initSystemQueries() |
||||
|
|
||||
|
server := &PostgreSQLServer{ |
||||
|
config: config, |
||||
|
sqlEngine: sqlEngine, |
||||
|
sessions: make(map[uint32]*PostgreSQLSession), |
||||
|
shutdown: make(chan struct{}), |
||||
|
translator: translator, |
||||
|
nextConnID: 1, |
||||
|
} |
||||
|
|
||||
|
return server, nil |
||||
|
} |
||||
|
|
||||
|
// Start begins listening for PostgreSQL connections
|
||||
|
func (s *PostgreSQLServer) Start() error { |
||||
|
addr := fmt.Sprintf("%s:%d", s.config.Host, s.config.Port) |
||||
|
|
||||
|
var listener net.Listener |
||||
|
var err error |
||||
|
|
||||
|
if s.config.TLSConfig != nil { |
||||
|
listener, err = tls.Listen("tcp", addr, s.config.TLSConfig) |
||||
|
glog.Infof("PostgreSQL Server with TLS listening on %s", addr) |
||||
|
} else { |
||||
|
listener, err = net.Listen("tcp", addr) |
||||
|
glog.Infof("PostgreSQL Server listening on %s", addr) |
||||
|
} |
||||
|
|
||||
|
if err != nil { |
||||
|
return fmt.Errorf("failed to start PostgreSQL server on %s: %v", addr, err) |
||||
|
} |
||||
|
|
||||
|
s.listener = listener |
||||
|
|
||||
|
// Start accepting connections
|
||||
|
s.wg.Add(1) |
||||
|
go s.acceptConnections() |
||||
|
|
||||
|
// Start cleanup routine
|
||||
|
s.wg.Add(1) |
||||
|
go s.cleanupSessions() |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// Stop gracefully shuts down the PostgreSQL server
|
||||
|
func (s *PostgreSQLServer) Stop() error { |
||||
|
close(s.shutdown) |
||||
|
|
||||
|
if s.listener != nil { |
||||
|
s.listener.Close() |
||||
|
} |
||||
|
|
||||
|
// Close all sessions
|
||||
|
s.sessionMux.Lock() |
||||
|
for _, session := range s.sessions { |
||||
|
session.close() |
||||
|
} |
||||
|
s.sessions = make(map[uint32]*PostgreSQLSession) |
||||
|
s.sessionMux.Unlock() |
||||
|
|
||||
|
s.wg.Wait() |
||||
|
glog.Infof("PostgreSQL Server stopped") |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// acceptConnections handles incoming PostgreSQL connections
|
||||
|
func (s *PostgreSQLServer) acceptConnections() { |
||||
|
defer s.wg.Done() |
||||
|
|
||||
|
for { |
||||
|
select { |
||||
|
case <-s.shutdown: |
||||
|
return |
||||
|
default: |
||||
|
} |
||||
|
|
||||
|
conn, err := s.listener.Accept() |
||||
|
if err != nil { |
||||
|
select { |
||||
|
case <-s.shutdown: |
||||
|
return |
||||
|
default: |
||||
|
glog.Errorf("Failed to accept PostgreSQL connection: %v", err) |
||||
|
continue |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Check connection limit
|
||||
|
s.sessionMux.RLock() |
||||
|
sessionCount := len(s.sessions) |
||||
|
s.sessionMux.RUnlock() |
||||
|
|
||||
|
if sessionCount >= s.config.MaxConns { |
||||
|
glog.Warningf("Maximum connections reached (%d), rejecting connection from %s", |
||||
|
s.config.MaxConns, conn.RemoteAddr()) |
||||
|
conn.Close() |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
s.wg.Add(1) |
||||
|
go s.handleConnection(conn) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// handleConnection processes a single PostgreSQL connection
|
||||
|
func (s *PostgreSQLServer) handleConnection(conn net.Conn) { |
||||
|
defer s.wg.Done() |
||||
|
defer conn.Close() |
||||
|
|
||||
|
// Generate unique connection ID
|
||||
|
connID := s.generateConnectionID() |
||||
|
secretKey := s.generateSecretKey() |
||||
|
|
||||
|
// Create session
|
||||
|
session := &PostgreSQLSession{ |
||||
|
conn: conn, |
||||
|
reader: bufio.NewReader(conn), |
||||
|
writer: bufio.NewWriter(conn), |
||||
|
authenticated: false, |
||||
|
database: s.config.Database, |
||||
|
parameters: make(map[string]string), |
||||
|
preparedStmts: make(map[string]*PreparedStatement), |
||||
|
portals: make(map[string]*Portal), |
||||
|
transactionState: PG_TRANS_IDLE, |
||||
|
processID: connID, |
||||
|
secretKey: secretKey, |
||||
|
created: time.Now(), |
||||
|
lastActivity: time.Now(), |
||||
|
} |
||||
|
|
||||
|
// Register session
|
||||
|
s.sessionMux.Lock() |
||||
|
s.sessions[connID] = session |
||||
|
s.sessionMux.Unlock() |
||||
|
|
||||
|
// Clean up on exit
|
||||
|
defer func() { |
||||
|
s.sessionMux.Lock() |
||||
|
delete(s.sessions, connID) |
||||
|
s.sessionMux.Unlock() |
||||
|
}() |
||||
|
|
||||
|
glog.Infof("New PostgreSQL connection from %s (ID: %d)", conn.RemoteAddr(), connID) |
||||
|
|
||||
|
// Handle startup
|
||||
|
err := s.handleStartup(session) |
||||
|
if err != nil { |
||||
|
glog.Errorf("Startup failed for connection %d: %v", connID, err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// Handle messages
|
||||
|
for { |
||||
|
select { |
||||
|
case <-s.shutdown: |
||||
|
return |
||||
|
default: |
||||
|
} |
||||
|
|
||||
|
// Set read timeout
|
||||
|
conn.SetReadDeadline(time.Now().Add(30 * time.Second)) |
||||
|
|
||||
|
err := s.handleMessage(session) |
||||
|
if err != nil { |
||||
|
if err == io.EOF { |
||||
|
glog.Infof("PostgreSQL client disconnected (ID: %d)", connID) |
||||
|
} else { |
||||
|
glog.Errorf("Error handling PostgreSQL message (ID: %d): %v", connID, err) |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
session.lastActivity = time.Now() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// handleStartup processes the PostgreSQL startup sequence
|
||||
|
func (s *PostgreSQLServer) handleStartup(session *PostgreSQLSession) error { |
||||
|
// Read startup message
|
||||
|
length := make([]byte, 4) |
||||
|
_, err := io.ReadFull(session.reader, length) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
msgLength := binary.BigEndian.Uint32(length) - 4 |
||||
|
msg := make([]byte, msgLength) |
||||
|
_, err = io.ReadFull(session.reader, msg) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Parse startup message
|
||||
|
protocolVersion := binary.BigEndian.Uint32(msg[0:4]) |
||||
|
if protocolVersion != 196608 { // PostgreSQL protocol version 3.0
|
||||
|
return fmt.Errorf("unsupported protocol version: %d", protocolVersion) |
||||
|
} |
||||
|
|
||||
|
// Parse parameters
|
||||
|
params := strings.Split(string(msg[4:]), "\x00") |
||||
|
for i := 0; i < len(params)-1; i += 2 { |
||||
|
if params[i] == "user" { |
||||
|
session.username = params[i+1] |
||||
|
} else if params[i] == "database" { |
||||
|
session.database = params[i+1] |
||||
|
} |
||||
|
session.parameters[params[i]] = params[i+1] |
||||
|
} |
||||
|
|
||||
|
// Handle authentication
|
||||
|
err = s.handleAuthentication(session) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Send parameter status messages
|
||||
|
err = s.sendParameterStatus(session, "server_version", "14.0 (SeaweedFS)") |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
err = s.sendParameterStatus(session, "server_encoding", "UTF8") |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
err = s.sendParameterStatus(session, "client_encoding", "UTF8") |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
err = s.sendParameterStatus(session, "DateStyle", "ISO, MDY") |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
err = s.sendParameterStatus(session, "integer_datetimes", "on") |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Send backend key data
|
||||
|
err = s.sendBackendKeyData(session) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Send ready for query
|
||||
|
err = s.sendReadyForQuery(session) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
session.authenticated = true |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// handleAuthentication processes authentication
|
||||
|
func (s *PostgreSQLServer) handleAuthentication(session *PostgreSQLSession) error { |
||||
|
switch s.config.AuthMethod { |
||||
|
case AuthTrust: |
||||
|
return s.sendAuthenticationOk(session) |
||||
|
case AuthPassword: |
||||
|
return s.handlePasswordAuth(session) |
||||
|
case AuthMD5: |
||||
|
return s.handleMD5Auth(session) |
||||
|
default: |
||||
|
return fmt.Errorf("unsupported authentication method") |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// sendAuthenticationOk sends authentication OK message
|
||||
|
func (s *PostgreSQLServer) sendAuthenticationOk(session *PostgreSQLSession) error { |
||||
|
msg := make([]byte, 8) |
||||
|
msg[0] = PG_RESP_AUTH_OK |
||||
|
binary.BigEndian.PutUint32(msg[1:5], 8) |
||||
|
binary.BigEndian.PutUint32(msg[5:9], AUTH_OK) |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err == nil { |
||||
|
err = session.writer.Flush() |
||||
|
} |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// handlePasswordAuth handles clear password authentication
|
||||
|
func (s *PostgreSQLServer) handlePasswordAuth(session *PostgreSQLSession) error { |
||||
|
// Send password request
|
||||
|
msg := make([]byte, 8) |
||||
|
msg[0] = PG_RESP_AUTH_OK |
||||
|
binary.BigEndian.PutUint32(msg[1:5], 8) |
||||
|
binary.BigEndian.PutUint32(msg[5:9], AUTH_CLEAR) |
||||
|
|
||||
|
_, err := session.writer.Write(msg) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
err = session.writer.Flush() |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Read password response
|
||||
|
msgType := make([]byte, 1) |
||||
|
_, err = io.ReadFull(session.reader, msgType) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
if msgType[0] != PG_MSG_PASSWORD { |
||||
|
return fmt.Errorf("expected password message, got %c", msgType[0]) |
||||
|
} |
||||
|
|
||||
|
length := make([]byte, 4) |
||||
|
_, err = io.ReadFull(session.reader, length) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
msgLength := binary.BigEndian.Uint32(length) - 4 |
||||
|
password := make([]byte, msgLength) |
||||
|
_, err = io.ReadFull(session.reader, password) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Verify password
|
||||
|
expectedPassword, exists := s.config.Users[session.username] |
||||
|
if !exists || string(password[:len(password)-1]) != expectedPassword { // Remove null terminator
|
||||
|
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"") |
||||
|
} |
||||
|
|
||||
|
return s.sendAuthenticationOk(session) |
||||
|
} |
||||
|
|
||||
|
// handleMD5Auth handles MD5 password authentication
|
||||
|
func (s *PostgreSQLServer) handleMD5Auth(session *PostgreSQLSession) error { |
||||
|
// Generate salt
|
||||
|
salt := make([]byte, 4) |
||||
|
_, err := rand.Read(salt) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Send MD5 request
|
||||
|
msg := make([]byte, 12) |
||||
|
msg[0] = PG_RESP_AUTH_OK |
||||
|
binary.BigEndian.PutUint32(msg[1:5], 12) |
||||
|
binary.BigEndian.PutUint32(msg[5:9], AUTH_MD5) |
||||
|
copy(msg[9:13], salt) |
||||
|
|
||||
|
_, err = session.writer.Write(msg) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
err = session.writer.Flush() |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Read password response
|
||||
|
msgType := make([]byte, 1) |
||||
|
_, err = io.ReadFull(session.reader, msgType) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
if msgType[0] != PG_MSG_PASSWORD { |
||||
|
return fmt.Errorf("expected password message, got %c", msgType[0]) |
||||
|
} |
||||
|
|
||||
|
length := make([]byte, 4) |
||||
|
_, err = io.ReadFull(session.reader, length) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
msgLength := binary.BigEndian.Uint32(length) - 4 |
||||
|
response := make([]byte, msgLength) |
||||
|
_, err = io.ReadFull(session.reader, response) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
// Verify MD5 hash
|
||||
|
expectedPassword, exists := s.config.Users[session.username] |
||||
|
if !exists { |
||||
|
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"") |
||||
|
} |
||||
|
|
||||
|
// Calculate expected hash: md5(md5(password + username) + salt)
|
||||
|
inner := md5.Sum([]byte(expectedPassword + session.username)) |
||||
|
expected := fmt.Sprintf("md5%x", md5.Sum(append([]byte(fmt.Sprintf("%x", inner)), salt...))) |
||||
|
|
||||
|
if string(response[:len(response)-1]) != expected { // Remove null terminator
|
||||
|
return s.sendError(session, "28P01", "authentication failed for user \""+session.username+"\"") |
||||
|
} |
||||
|
|
||||
|
return s.sendAuthenticationOk(session) |
||||
|
} |
||||
|
|
||||
|
// generateConnectionID generates a unique connection ID
|
||||
|
func (s *PostgreSQLServer) generateConnectionID() uint32 { |
||||
|
s.sessionMux.Lock() |
||||
|
defer s.sessionMux.Unlock() |
||||
|
id := s.nextConnID |
||||
|
s.nextConnID++ |
||||
|
return id |
||||
|
} |
||||
|
|
||||
|
// generateSecretKey generates a secret key for the connection
|
||||
|
func (s *PostgreSQLServer) generateSecretKey() uint32 { |
||||
|
key := make([]byte, 4) |
||||
|
rand.Read(key) |
||||
|
return binary.BigEndian.Uint32(key) |
||||
|
} |
||||
|
|
||||
|
// close marks the session as closed
|
||||
|
func (s *PostgreSQLSession) close() { |
||||
|
s.mutex.Lock() |
||||
|
defer s.mutex.Unlock() |
||||
|
if s.conn != nil { |
||||
|
s.conn.Close() |
||||
|
s.conn = nil |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// cleanupSessions periodically cleans up idle sessions
|
||||
|
func (s *PostgreSQLServer) cleanupSessions() { |
||||
|
defer s.wg.Done() |
||||
|
|
||||
|
ticker := time.NewTicker(time.Minute) |
||||
|
defer ticker.Stop() |
||||
|
|
||||
|
for { |
||||
|
select { |
||||
|
case <-s.shutdown: |
||||
|
return |
||||
|
case <-ticker.C: |
||||
|
s.cleanupIdleSessions() |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// cleanupIdleSessions removes sessions that have been idle too long
|
||||
|
func (s *PostgreSQLServer) cleanupIdleSessions() { |
||||
|
now := time.Now() |
||||
|
|
||||
|
s.sessionMux.Lock() |
||||
|
defer s.sessionMux.Unlock() |
||||
|
|
||||
|
for id, session := range s.sessions { |
||||
|
if now.Sub(session.lastActivity) > s.config.IdleTimeout { |
||||
|
glog.Infof("Closing idle PostgreSQL session %d", id) |
||||
|
session.close() |
||||
|
delete(s.sessions, id) |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// GetAddress returns the server address
|
||||
|
func (s *PostgreSQLServer) GetAddress() string { |
||||
|
return fmt.Sprintf("%s:%d", s.config.Host, s.config.Port) |
||||
|
} |
@ -0,0 +1,356 @@ |
|||||
|
package postgres |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"regexp" |
||||
|
"strings" |
||||
|
) |
||||
|
|
||||
|
// PostgreSQL to SeaweedFS SQL translator
|
||||
|
type PostgreSQLTranslator struct { |
||||
|
systemQueries map[string]string |
||||
|
patterns map[*regexp.Regexp]string |
||||
|
} |
||||
|
|
||||
|
// initSystemQueries initializes the system query mappings
|
||||
|
func (t *PostgreSQLTranslator) initSystemQueries() { |
||||
|
t.systemQueries = map[string]string{ |
||||
|
// Version queries
|
||||
|
"SELECT version()": "SELECT 'SeaweedFS 1.0 (PostgreSQL 14.0 compatible)' as version", |
||||
|
"SELECT version() AS version": "SELECT 'SeaweedFS 1.0 (PostgreSQL 14.0 compatible)' as version", |
||||
|
"select version()": "SELECT 'SeaweedFS 1.0 (PostgreSQL 14.0 compatible)' as version", |
||||
|
|
||||
|
// Current database
|
||||
|
"SELECT current_database()": "SELECT 'default' as current_database", |
||||
|
"select current_database()": "SELECT 'default' as current_database", |
||||
|
"SELECT current_database() AS current_database": "SELECT 'default' as current_database", |
||||
|
|
||||
|
// Current user
|
||||
|
"SELECT current_user": "SELECT 'seaweedfs' as current_user", |
||||
|
"select current_user": "SELECT 'seaweedfs' as current_user", |
||||
|
"SELECT current_user AS current_user": "SELECT 'seaweedfs' as current_user", |
||||
|
"SELECT user": "SELECT 'seaweedfs' as user", |
||||
|
|
||||
|
// Session info
|
||||
|
"SELECT session_user": "SELECT 'seaweedfs' as session_user", |
||||
|
"SELECT current_setting('server_version')": "SELECT '14.0' as server_version", |
||||
|
"SELECT current_setting('server_encoding')": "SELECT 'UTF8' as server_encoding", |
||||
|
"SELECT current_setting('client_encoding')": "SELECT 'UTF8' as client_encoding", |
||||
|
|
||||
|
// Simple system queries
|
||||
|
"SELECT 1": "SELECT 1", |
||||
|
"select 1": "SELECT 1", |
||||
|
"SELECT 1 AS test": "SELECT 1 AS test", |
||||
|
|
||||
|
// Database listing
|
||||
|
"SELECT datname FROM pg_database": "SHOW DATABASES", |
||||
|
"SELECT datname FROM pg_database ORDER BY datname": "SHOW DATABASES", |
||||
|
|
||||
|
// Table listing
|
||||
|
"SELECT tablename FROM pg_tables": "SHOW TABLES", |
||||
|
"SELECT schemaname, tablename FROM pg_tables": "SHOW TABLES", |
||||
|
"SELECT table_name FROM information_schema.tables": "SHOW TABLES", |
||||
|
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'": "SHOW TABLES", |
||||
|
|
||||
|
// Schema queries
|
||||
|
"SELECT schema_name FROM information_schema.schemata": "SELECT 'public' as schema_name", |
||||
|
"SELECT nspname FROM pg_namespace": "SELECT 'public' as nspname", |
||||
|
|
||||
|
// Connection info
|
||||
|
"SELECT inet_client_addr()": "SELECT '127.0.0.1' as inet_client_addr", |
||||
|
"SELECT inet_client_port()": "SELECT 0 as inet_client_port", |
||||
|
"SELECT pg_backend_pid()": "SELECT 1 as pg_backend_pid", |
||||
|
|
||||
|
// Transaction info
|
||||
|
"SELECT txid_current()": "SELECT 1 as txid_current", |
||||
|
"SELECT pg_is_in_recovery()": "SELECT false as pg_is_in_recovery", |
||||
|
|
||||
|
// Statistics
|
||||
|
"SELECT COUNT(*) FROM pg_stat_user_tables": "SELECT 0 as count", |
||||
|
|
||||
|
// Empty system tables
|
||||
|
"SELECT * FROM pg_settings LIMIT 0": "SELECT 'name' as name, 'setting' as setting, 'unit' as unit, 'category' as category, 'short_desc' as short_desc, 'extra_desc' as extra_desc, 'context' as context, 'vartype' as vartype, 'source' as source, 'min_val' as min_val, 'max_val' as max_val, 'enumvals' as enumvals, 'boot_val' as boot_val, 'reset_val' as reset_val, 'sourcefile' as sourcefile, 'sourceline' as sourceline, 'pending_restart' as pending_restart WHERE 1=0", |
||||
|
|
||||
|
"SELECT * FROM pg_type LIMIT 0": "SELECT 'oid' as oid, 'typname' as typname, 'typlen' as typlen WHERE 1=0", |
||||
|
|
||||
|
"SELECT * FROM pg_class LIMIT 0": "SELECT 'oid' as oid, 'relname' as relname, 'relkind' as relkind WHERE 1=0", |
||||
|
} |
||||
|
|
||||
|
// Initialize regex patterns for more complex queries
|
||||
|
t.patterns = map[*regexp.Regexp]string{ |
||||
|
// \d commands (psql describe commands)
|
||||
|
regexp.MustCompile(`(?i)\\d\+?\s*$`): "SHOW TABLES", |
||||
|
regexp.MustCompile(`(?i)\\dt\+?\s*$`): "SHOW TABLES", |
||||
|
regexp.MustCompile(`(?i)\\dn\+?\s*$`): "SELECT 'public' as name, 'seaweedfs' as owner", |
||||
|
regexp.MustCompile(`(?i)\\l\+?\s*$`): "SHOW DATABASES", |
||||
|
regexp.MustCompile(`(?i)\\d\+?\s+(\w+)$`): "DESCRIBE $1", |
||||
|
regexp.MustCompile(`(?i)\\dt\+?\s+(\w+)$`): "DESCRIBE $1", |
||||
|
|
||||
|
// pg_catalog queries
|
||||
|
regexp.MustCompile(`(?i)SELECT\s+.*\s+FROM\s+pg_catalog\.pg_tables`): "SHOW TABLES", |
||||
|
regexp.MustCompile(`(?i)SELECT\s+.*\s+FROM\s+pg_tables`): "SHOW TABLES", |
||||
|
regexp.MustCompile(`(?i)SELECT\s+.*\s+FROM\s+pg_database`): "SHOW DATABASES", |
||||
|
|
||||
|
// SHOW commands (already supported but normalize)
|
||||
|
regexp.MustCompile(`(?i)SHOW\s+DATABASES?\s*;?\s*$`): "SHOW DATABASES", |
||||
|
regexp.MustCompile(`(?i)SHOW\s+TABLES?\s*;?\s*$`): "SHOW TABLES", |
||||
|
regexp.MustCompile(`(?i)SHOW\s+SCHEMAS?\s*;?\s*$`): "SELECT 'public' as schema_name", |
||||
|
|
||||
|
// BEGIN/COMMIT/ROLLBACK (no-op for read-only)
|
||||
|
regexp.MustCompile(`(?i)BEGIN\s*;?\s*$`): "SELECT 'BEGIN' as status", |
||||
|
regexp.MustCompile(`(?i)START\s+TRANSACTION\s*;?\s*$`): "SELECT 'BEGIN' as status", |
||||
|
regexp.MustCompile(`(?i)COMMIT\s*;?\s*$`): "SELECT 'COMMIT' as status", |
||||
|
regexp.MustCompile(`(?i)ROLLBACK\s*;?\s*$`): "SELECT 'ROLLBACK' as status", |
||||
|
|
||||
|
// SET commands (mostly no-op)
|
||||
|
regexp.MustCompile(`(?i)SET\s+.*\s*;?\s*$`): "SELECT 'SET' as status", |
||||
|
|
||||
|
// Column information queries
|
||||
|
regexp.MustCompile(`(?i)SELECT\s+.*\s+FROM\s+information_schema\.columns\s+WHERE\s+table_name\s*=\s*'(\w+)'`): "DESCRIBE $1", |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// TranslateQuery translates a PostgreSQL query to SeaweedFS SQL
|
||||
|
func (t *PostgreSQLTranslator) TranslateQuery(pgSQL string) (string, error) { |
||||
|
// Trim whitespace and semicolons
|
||||
|
query := strings.TrimSpace(pgSQL) |
||||
|
query = strings.TrimSuffix(query, ";") |
||||
|
|
||||
|
// Check for exact matches first
|
||||
|
if seaweedSQL, exists := t.systemQueries[query]; exists { |
||||
|
return seaweedSQL, nil |
||||
|
} |
||||
|
|
||||
|
// Check case-insensitive exact matches
|
||||
|
queryLower := strings.ToLower(query) |
||||
|
for pgQuery, seaweedSQL := range t.systemQueries { |
||||
|
if strings.ToLower(pgQuery) == queryLower { |
||||
|
return seaweedSQL, nil |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Check regex patterns
|
||||
|
for pattern, replacement := range t.patterns { |
||||
|
if pattern.MatchString(query) { |
||||
|
// Handle replacements with capture groups
|
||||
|
if strings.Contains(replacement, "$") { |
||||
|
return pattern.ReplaceAllString(query, replacement), nil |
||||
|
} |
||||
|
return replacement, nil |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Handle psql meta-commands
|
||||
|
if strings.HasPrefix(query, "\\") { |
||||
|
return t.translateMetaCommand(query) |
||||
|
} |
||||
|
|
||||
|
// Handle information_schema queries
|
||||
|
if strings.Contains(strings.ToLower(query), "information_schema") { |
||||
|
return t.translateInformationSchema(query) |
||||
|
} |
||||
|
|
||||
|
// Handle pg_catalog queries
|
||||
|
if strings.Contains(strings.ToLower(query), "pg_catalog") || strings.Contains(strings.ToLower(query), "pg_") { |
||||
|
return t.translatePgCatalog(query) |
||||
|
} |
||||
|
|
||||
|
// For regular queries, pass through as-is
|
||||
|
// The SeaweedFS SQL engine will handle standard SQL
|
||||
|
return query, nil |
||||
|
} |
||||
|
|
||||
|
// translateMetaCommand translates psql meta-commands
|
||||
|
func (t *PostgreSQLTranslator) translateMetaCommand(cmd string) (string, error) { |
||||
|
cmd = strings.TrimSpace(cmd) |
||||
|
|
||||
|
switch { |
||||
|
case cmd == "\\d" || cmd == "\\dt": |
||||
|
return "SHOW TABLES", nil |
||||
|
case cmd == "\\l": |
||||
|
return "SHOW DATABASES", nil |
||||
|
case cmd == "\\dn": |
||||
|
return "SELECT 'public' as schema_name, 'seaweedfs' as owner", nil |
||||
|
case cmd == "\\du": |
||||
|
return "SELECT 'seaweedfs' as rolname, true as rolsuper, true as rolcreaterole, true as rolcreatedb", nil |
||||
|
case strings.HasPrefix(cmd, "\\d "): |
||||
|
// Describe table
|
||||
|
tableName := strings.TrimSpace(cmd[3:]) |
||||
|
return fmt.Sprintf("DESCRIBE %s", tableName), nil |
||||
|
case strings.HasPrefix(cmd, "\\dt "): |
||||
|
// Describe table (table-specific)
|
||||
|
tableName := strings.TrimSpace(cmd[4:]) |
||||
|
return fmt.Sprintf("DESCRIBE %s", tableName), nil |
||||
|
case cmd == "\\q": |
||||
|
return "SELECT 'quit' as status", fmt.Errorf("client requested quit") |
||||
|
case cmd == "\\h" || cmd == "\\help": |
||||
|
return "SELECT 'SeaweedFS PostgreSQL Interface - Limited command support' as help", nil |
||||
|
case cmd == "\\?": |
||||
|
return "SELECT 'Available: \\d (tables), \\l (databases), \\q (quit)' as commands", nil |
||||
|
default: |
||||
|
return "SELECT 'Unsupported meta-command' as error", fmt.Errorf("unsupported meta-command: %s", cmd) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// translateInformationSchema translates INFORMATION_SCHEMA queries
|
||||
|
func (t *PostgreSQLTranslator) translateInformationSchema(query string) (string, error) { |
||||
|
queryLower := strings.ToLower(query) |
||||
|
|
||||
|
if strings.Contains(queryLower, "information_schema.tables") { |
||||
|
return "SHOW TABLES", nil |
||||
|
} |
||||
|
|
||||
|
if strings.Contains(queryLower, "information_schema.columns") { |
||||
|
// Extract table name if present
|
||||
|
re := regexp.MustCompile(`(?i)table_name\s*=\s*'(\w+)'`) |
||||
|
matches := re.FindStringSubmatch(query) |
||||
|
if len(matches) > 1 { |
||||
|
return fmt.Sprintf("DESCRIBE %s", matches[1]), nil |
||||
|
} |
||||
|
return "SHOW TABLES", nil // Return tables if no specific table
|
||||
|
} |
||||
|
|
||||
|
if strings.Contains(queryLower, "information_schema.schemata") { |
||||
|
return "SELECT 'public' as schema_name, 'seaweedfs' as schema_owner", nil |
||||
|
} |
||||
|
|
||||
|
// Default fallback
|
||||
|
return "SELECT 'information_schema query not supported' as error", nil |
||||
|
} |
||||
|
|
||||
|
// translatePgCatalog translates PostgreSQL catalog queries
|
||||
|
func (t *PostgreSQLTranslator) translatePgCatalog(query string) (string, error) { |
||||
|
queryLower := strings.ToLower(query) |
||||
|
|
||||
|
// pg_tables
|
||||
|
if strings.Contains(queryLower, "pg_tables") { |
||||
|
return "SHOW TABLES", nil |
||||
|
} |
||||
|
|
||||
|
// pg_database
|
||||
|
if strings.Contains(queryLower, "pg_database") { |
||||
|
return "SHOW DATABASES", nil |
||||
|
} |
||||
|
|
||||
|
// pg_namespace
|
||||
|
if strings.Contains(queryLower, "pg_namespace") { |
||||
|
return "SELECT 'public' as nspname, 2200 as oid", nil |
||||
|
} |
||||
|
|
||||
|
// pg_class (tables, indexes, etc.)
|
||||
|
if strings.Contains(queryLower, "pg_class") { |
||||
|
return "SHOW TABLES", nil |
||||
|
} |
||||
|
|
||||
|
// pg_type (data types)
|
||||
|
if strings.Contains(queryLower, "pg_type") { |
||||
|
return t.generatePgTypeResult(), nil |
||||
|
} |
||||
|
|
||||
|
// pg_attribute (column info)
|
||||
|
if strings.Contains(queryLower, "pg_attribute") { |
||||
|
return "SELECT 'attname' as attname, 'atttypid' as atttypid, 'attnum' as attnum WHERE 1=0", nil |
||||
|
} |
||||
|
|
||||
|
// pg_settings
|
||||
|
if strings.Contains(queryLower, "pg_settings") { |
||||
|
return t.generatePgSettingsResult(), nil |
||||
|
} |
||||
|
|
||||
|
// pg_stat_* tables
|
||||
|
if strings.Contains(queryLower, "pg_stat_") { |
||||
|
return "SELECT 0 as count", nil |
||||
|
} |
||||
|
|
||||
|
// Default: return empty result for unknown pg_ queries
|
||||
|
return "SELECT 'pg_catalog query not fully supported' as notice", nil |
||||
|
} |
||||
|
|
||||
|
// generatePgTypeResult generates a basic pg_type result
|
||||
|
func (t *PostgreSQLTranslator) generatePgTypeResult() string { |
||||
|
return ` |
||||
|
SELECT * FROM ( |
||||
|
SELECT 16 as oid, 'bool' as typname, 1 as typlen, 'b' as typtype |
||||
|
UNION ALL |
||||
|
SELECT 20 as oid, 'int8' as typname, 8 as typlen, 'b' as typtype |
||||
|
UNION ALL |
||||
|
SELECT 23 as oid, 'int4' as typname, 4 as typlen, 'b' as typtype |
||||
|
UNION ALL |
||||
|
SELECT 25 as oid, 'text' as typname, -1 as typlen, 'b' as typtype |
||||
|
UNION ALL |
||||
|
SELECT 701 as oid, 'float8' as typname, 8 as typlen, 'b' as typtype |
||||
|
UNION ALL |
||||
|
SELECT 1043 as oid, 'varchar' as typname, -1 as typlen, 'b' as typtype |
||||
|
UNION ALL |
||||
|
SELECT 1114 as oid, 'timestamp' as typname, 8 as typlen, 'b' as typtype |
||||
|
) t WHERE 1=0 |
||||
|
` |
||||
|
} |
||||
|
|
||||
|
// generatePgSettingsResult generates a basic pg_settings result
|
||||
|
func (t *PostgreSQLTranslator) generatePgSettingsResult() string { |
||||
|
return ` |
||||
|
SELECT * FROM ( |
||||
|
SELECT 'server_version' as name, '14.0' as setting, NULL as unit, 'Version and Platform Compatibility' as category, 'SeaweedFS version' as short_desc |
||||
|
UNION ALL |
||||
|
SELECT 'server_encoding' as name, 'UTF8' as setting, NULL as unit, 'Client Connection Defaults' as category, 'Server encoding' as short_desc |
||||
|
UNION ALL |
||||
|
SELECT 'client_encoding' as name, 'UTF8' as setting, NULL as unit, 'Client Connection Defaults' as category, 'Client encoding' as short_desc |
||||
|
UNION ALL |
||||
|
SELECT 'max_connections' as name, '100' as setting, NULL as unit, 'Connections and Authentication' as category, 'Maximum connections' as short_desc |
||||
|
) s WHERE 1=0 |
||||
|
` |
||||
|
} |
||||
|
|
||||
|
// GetDatabaseName returns the appropriate database name for the session
|
||||
|
func (t *PostgreSQLTranslator) GetDatabaseName(requestedDB string) string { |
||||
|
if requestedDB == "" || requestedDB == "postgres" || requestedDB == "template1" { |
||||
|
return "default" |
||||
|
} |
||||
|
return requestedDB |
||||
|
} |
||||
|
|
||||
|
// IsSystemQuery checks if a query is a system/meta query that doesn't access actual data
|
||||
|
func (t *PostgreSQLTranslator) IsSystemQuery(query string) bool { |
||||
|
queryLower := strings.ToLower(strings.TrimSpace(query)) |
||||
|
|
||||
|
// System function calls
|
||||
|
systemFunctions := []string{ |
||||
|
"version()", "current_database()", "current_user", "session_user", |
||||
|
"current_setting(", "inet_client_", "pg_backend_pid()", "txid_current()", |
||||
|
"pg_is_in_recovery()", |
||||
|
} |
||||
|
|
||||
|
for _, fn := range systemFunctions { |
||||
|
if strings.Contains(queryLower, fn) { |
||||
|
return true |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// System table queries
|
||||
|
systemTables := []string{ |
||||
|
"pg_catalog", "pg_tables", "pg_database", "pg_namespace", "pg_class", |
||||
|
"pg_type", "pg_attribute", "pg_settings", "pg_stat_", "information_schema", |
||||
|
} |
||||
|
|
||||
|
for _, table := range systemTables { |
||||
|
if strings.Contains(queryLower, table) { |
||||
|
return true |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// Meta commands
|
||||
|
if strings.HasPrefix(queryLower, "\\") { |
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
// Transaction control
|
||||
|
transactionCommands := []string{"begin", "commit", "rollback", "start transaction", "set "} |
||||
|
for _, cmd := range transactionCommands { |
||||
|
if strings.HasPrefix(queryLower, cmd) { |
||||
|
return true |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return false |
||||
|
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue