diff --git a/test/s3tables/catalog_trino/trino_blog_operations_test.go b/test/s3tables/catalog_trino/trino_blog_operations_test.go new file mode 100644 index 000000000..6603507d5 --- /dev/null +++ b/test/s3tables/catalog_trino/trino_blog_operations_test.go @@ -0,0 +1,172 @@ +package catalog_trino + +import ( + "fmt" + "strconv" + "strings" + "testing" + "time" +) + +func TestTrinoBlogOperations(t *testing.T) { + env := setupTrinoTest(t) + defer env.Cleanup(t) + + schemaName := "blog_ns_" + randomString(6) + customersTable := "customers_" + randomString(6) + trinoCustomersTable := "trino_customers_" + randomString(6) + + runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName)) + defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP SCHEMA IF EXISTS iceberg.%s", schemaName)) + defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, trinoCustomersTable)) + defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, customersTable)) + + createCustomersSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS iceberg.%s.%s ( + customer_sk INT, + customer_id VARCHAR, + salutation VARCHAR, + first_name VARCHAR, + last_name VARCHAR, + preferred_cust_flag VARCHAR, + birth_day INT, + birth_month INT, + birth_year INT, + birth_country VARCHAR, + login VARCHAR +) WITH ( + format = 'PARQUET', + sorted_by = ARRAY['customer_id'] +)`, schemaName, customersTable) + runTrinoSQL(t, env.trinoContainer, createCustomersSQL) + + insertCustomersSQL := fmt.Sprintf(`INSERT INTO iceberg.%s.%s VALUES + (1, 'AAAAA', 'Mrs', 'Amanda', 'Olson', 'Y', 8, 4, 1984, 'US', 'aolson'), + (2, 'AAAAB', 'Mr', 'Leonard', 'Eads', 'N', 22, 6, 2001, 'US', 'leads'), + (3, 'BAAAA', 'Mr', 'David', 'White', 'Y', 16, 2, 1999, 'US', 'dwhite'), + (4, 'BBAAA', 'Mr', 'Melvin', 'Lee', 'N', 30, 3, 1973, 'US', 'mlee'), + (5, 'AACAA', 'Mr', 'Donald', 'Holt', 'N', 2, 6, 1982, 'CA', 'dholt'), + (6, 'ABAAA', 'Mrs', 'Jacqueline', 'Harvey', 'N', 5, 12, 1988, 'US', 'jharvey'), + (7, 'BBAAA', 'Ms', 'Debbie', 'Ward', 'N', 6, 1, 2006, 'MX', 'dward'), + (8, 'ACAAA', 'Mr', 'Tim', 'Strong', 'N', 15, 7, 1976, 'US', 'tstrong') +`, schemaName, customersTable) + runTrinoSQL(t, env.trinoContainer, insertCustomersSQL) + + countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, customersTable)) + rowCount := mustParseCSVInt64(t, countOutput) + if rowCount != 8 { + t.Fatalf("expected 8 rows in customers table, got %d", rowCount) + } + + output := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT first_name FROM iceberg.%s.%s WHERE customer_sk = 1", schemaName, customersTable)) + if !strings.Contains(output, "Amanda") { + t.Fatalf("expected sample query to include Amanda, got: %s", output) + } + + ctasSQL := fmt.Sprintf(`CREATE TABLE iceberg.%s.%s +WITH ( + format = 'PARQUET' +) +AS SELECT * FROM iceberg.%s.%s`, schemaName, trinoCustomersTable, schemaName, customersTable) + runTrinoSQL(t, env.trinoContainer, ctasSQL) + + countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable)) + rowCount = mustParseCSVInt64(t, countOutput) + if rowCount != 8 { + t.Fatalf("expected 8 rows in CTAS table, got %d", rowCount) + } + + runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("ALTER TABLE iceberg.%s.%s ADD COLUMN updated_at TIMESTAMP", schemaName, trinoCustomersTable)) + output = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DESCRIBE iceberg.%s.%s", schemaName, trinoCustomersTable)) + if !strings.Contains(output, "updated_at") { + t.Fatalf("expected updated_at column in describe output, got: %s", output) + } + + runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("UPDATE iceberg.%s.%s SET updated_at = current_timestamp", schemaName, trinoCustomersTable)) + + // Sleep to ensure timestamps are in the past for time travel queries + time.Sleep(1 * time.Second) + + snapshotOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT snapshot_id FROM iceberg.%s."%s$snapshots" ORDER BY committed_at DESC LIMIT 1`, schemaName, trinoCustomersTable)) + snapshotID := mustParseCSVInt64(t, snapshotOutput) + if snapshotID == 0 { + t.Fatalf("expected snapshot ID from snapshots table, got 0") + } + + filesOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT file_path FROM iceberg.%s."%s$files" LIMIT 1`, schemaName, trinoCustomersTable)) + if !hasCSVDataRow(filesOutput) { + t.Fatalf("expected files metadata rows, got: %s", filesOutput) + } + + historyOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT made_current_at FROM iceberg.%s."%s$history" LIMIT 1`, schemaName, trinoCustomersTable)) + if !hasCSVDataRow(historyOutput) { + t.Fatalf("expected history metadata rows, got: %s", historyOutput) + } + + countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s FOR VERSION AS OF %d", schemaName, trinoCustomersTable, snapshotID)) + versionCount := mustParseCSVInt64(t, countOutput) + if versionCount != 8 { + t.Fatalf("expected 8 rows for version time travel, got %d", versionCount) + } + + // Use current_timestamp - interval '1 second' to ensure it's in the past (Iceberg requirement) + countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s FOR TIMESTAMP AS OF (current_timestamp - interval '1' second)", schemaName, trinoCustomersTable)) + timestampCount := mustParseCSVInt64(t, countOutput) + if timestampCount != 8 { + t.Fatalf("expected 8 rows for timestamp time travel, got %d", timestampCount) + } + + runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DELETE FROM iceberg.%s.%s WHERE customer_sk = 8", schemaName, trinoCustomersTable)) + countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable)) + rowCount = mustParseCSVInt64(t, countOutput) + if rowCount != 7 { + t.Fatalf("expected 7 rows after delete, got %d", rowCount) + } + + runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("ALTER TABLE iceberg.%s.%s EXECUTE rollback_to_snapshot(%d)", schemaName, trinoCustomersTable, snapshotID)) + countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable)) + rowCount = mustParseCSVInt64(t, countOutput) + if rowCount != 8 { + t.Fatalf("expected 8 rows after rollback, got %d", rowCount) + } +} + +func hasCSVDataRow(output string) bool { + lines := strings.Split(strings.TrimSpace(output), "\n") + if len(lines) == 0 { + return false + } + for _, line := range lines { + if strings.TrimSpace(line) != "" { + return true + } + } + return false +} + +func mustParseCSVInt64(t *testing.T, output string) int64 { + t.Helper() + value := mustFirstCSVValue(t, output) + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + t.Fatalf("failed to parse int from output %q: %v", output, err) + } + return parsed +} + +func mustFirstCSVValue(t *testing.T, output string) string { + t.Helper() + lines := strings.Split(strings.TrimSpace(output), "\n") + if len(lines) == 0 { + t.Fatalf("expected CSV output with data row, got: %q", output) + } + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, ",") + return strings.Trim(parts[0], "\"") + } + t.Fatalf("no CSV data rows found in output: %q", output) + return "" +} diff --git a/test/s3tables/catalog_trino/trino_catalog_test.go b/test/s3tables/catalog_trino/trino_catalog_test.go index 74dec1637..44beea80c 100644 --- a/test/s3tables/catalog_trino/trino_catalog_test.go +++ b/test/s3tables/catalog_trino/trino_catalog_test.go @@ -319,8 +319,9 @@ func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket strin config := fmt.Sprintf(`connector.name=iceberg iceberg.catalog.type=rest iceberg.rest-catalog.uri=http://host.docker.internal:%d -iceberg.rest-catalog.warehouse=s3://%s/ +iceberg.rest-catalog.warehouse=s3tablescatalog/%s iceberg.file-format=PARQUET +iceberg.unique-table-location=true # S3 storage config fs.native-s3.enabled=true @@ -415,7 +416,7 @@ func runTrinoSQL(t *testing.T, containerName, sql string) string { logs, _ := exec.Command("docker", "logs", containerName).CombinedOutput() t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s\nTrino logs:\n%s", err, sql, string(output), string(logs)) } - return string(output) + return sanitizeTrinoOutput(string(output)) } func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) { @@ -439,6 +440,30 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) { t.Logf("Created table bucket: %s", bucketName) } +func sanitizeTrinoOutput(output string) string { + lines := strings.Split(strings.TrimSpace(output), "\n") + filtered := make([]string, 0, len(lines)) + for _, line := range lines { + if strings.Contains(line, "org.jline.utils.Log") { + continue + } + if strings.Contains(line, "Unable to create a system terminal") { + continue + } + if strings.HasPrefix(line, "WARNING:") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + filtered = append(filtered, line) + } + if len(filtered) == 0 { + return "" + } + return strings.Join(filtered, "\n") + "\n" +} + func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) { t.Helper() diff --git a/weed/admin/dash/file_browser_data.go b/weed/admin/dash/file_browser_data.go index 4126b2ac8..f5b74f84b 100644 --- a/weed/admin/dash/file_browser_data.go +++ b/weed/admin/dash/file_browser_data.go @@ -6,7 +6,9 @@ import ( "strings" "time" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" ) // FileEntry represents a file or directory entry in the file browser @@ -218,25 +220,33 @@ func (s *AdminServer) GetFileBrowser(dir string, lastFileName string, pageSize i } } - // Check if this is a bucket path + // Check if this is a bucket path and determine if it's a table bucket isBucketPath := false bucketName := "" + isTableBucketPath := false + tableBucketName := "" if strings.HasPrefix(dir, "/buckets/") { isBucketPath = true pathParts := strings.Split(strings.Trim(dir, "/"), "/") if len(pathParts) >= 2 { bucketName = pathParts[1] - } - } - - // Check if this is a table bucket path - isTableBucketPath := false - tableBucketName := "" - if strings.HasPrefix(dir, "/table-buckets/") { - isTableBucketPath = true - pathParts := strings.Split(strings.Trim(dir, "/"), "/") - if len(pathParts) >= 2 { - tableBucketName = pathParts[1] + // Check table bucket status early to avoid second WithFilerClient call + if err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: "/buckets", + Name: bucketName, + }) + if err != nil { + return err + } + if s3tables.IsTableBucketEntry(resp.Entry) { + isTableBucketPath = true + tableBucketName = bucketName + } + return nil + }); err != nil { + glog.V(1).Infof("file browser table bucket lookup failed for %s: %v", bucketName, err) + } } } @@ -287,12 +297,8 @@ func (s *AdminServer) generateBreadcrumbs(dir string) []BreadcrumbItem { displayName := part if len(breadcrumbs) == 1 && part == "buckets" { displayName = "Object Store Buckets" - } else if len(breadcrumbs) == 1 && part == "table-buckets" { - displayName = "Table Buckets" } else if len(breadcrumbs) == 2 && strings.HasPrefix(dir, "/buckets/") { displayName = "📦 " + part // Add bucket icon to bucket name - } else if len(breadcrumbs) == 2 && strings.HasPrefix(dir, "/table-buckets/") { - displayName = "🧊 " + part } breadcrumbs = append(breadcrumbs, BreadcrumbItem{ diff --git a/weed/admin/dash/file_browser_data_test.go b/weed/admin/dash/file_browser_data_test.go index e02465034..0605735af 100644 --- a/weed/admin/dash/file_browser_data_test.go +++ b/weed/admin/dash/file_browser_data_test.go @@ -51,15 +51,6 @@ func TestGenerateBreadcrumbs(t *testing.T) { {Name: "📦 mybucket", Path: "/buckets/mybucket"}, }, }, - { - name: "table bucket path", - path: "/table-buckets/mytablebucket", - expected: []BreadcrumbItem{ - {Name: "Root", Path: "/"}, - {Name: "Table Buckets", Path: "/table-buckets"}, - {Name: "🧊 mytablebucket", Path: "/table-buckets/mytablebucket"}, - }, - }, { name: "bucket nested path", path: "/buckets/mybucket/folder", @@ -70,16 +61,6 @@ func TestGenerateBreadcrumbs(t *testing.T) { {Name: "folder", Path: "/buckets/mybucket/folder"}, }, }, - { - name: "table bucket nested path", - path: "/table-buckets/mytablebucket/folder", - expected: []BreadcrumbItem{ - {Name: "Root", Path: "/"}, - {Name: "Table Buckets", Path: "/table-buckets"}, - {Name: "🧊 mytablebucket", Path: "/table-buckets/mytablebucket"}, - {Name: "folder", Path: "/table-buckets/mytablebucket/folder"}, - }, - }, { name: "path with trailing slash", path: "/folder/", @@ -195,11 +176,6 @@ func TestParentPathCalculationLogic(t *testing.T) { currentDir: "/buckets/mybucket", expected: "/buckets", }, - { - name: "table bucket directory", - currentDir: "/table-buckets/mytablebucket", - expected: "/table-buckets", - }, } for _, tt := range tests { diff --git a/weed/admin/dash/s3tables_management.go b/weed/admin/dash/s3tables_management.go index fd9e4334b..a77a8201c 100644 --- a/weed/admin/dash/s3tables_management.go +++ b/weed/admin/dash/s3tables_management.go @@ -100,6 +100,9 @@ func (s *AdminServer) GetS3TablesBucketsData(ctx context.Context) (S3TablesBucke if strings.HasPrefix(entry.Entry.Name, ".") { continue } + if !s3tables.IsTableBucketEntry(entry.Entry) { + continue + } metaBytes, ok := entry.Entry.Extended[s3tables.ExtendedKeyMetadata] if !ok { continue diff --git a/weed/admin/view/app/file_browser.templ b/weed/admin/view/app/file_browser.templ index 58c999ebc..576a27ed9 100644 --- a/weed/admin/view/app/file_browser.templ +++ b/weed/admin/view/app/file_browser.templ @@ -14,25 +14,25 @@ script changePageSize(path string, lastFileName string) { templ FileBrowser(data dash.FileBrowserData) {