Browse Source

iceberg: persist namespace properties via s3tables metadata

pull/8276/head
Chris Lu 1 day ago
parent
commit
88e727791c
  1. 27
      weed/s3api/iceberg/iceberg.go
  2. 29
      weed/s3api/iceberg/iceberg_namespace_properties_test.go
  3. 3
      weed/s3api/s3tables/handler_namespace.go
  4. 24
      weed/s3api/s3tables/types.go
  5. 7
      weed/s3api/s3tables/utils.go
  6. 23
      weed/s3api/s3tables/utils_namespace_test.go

27
weed/s3api/iceberg/iceberg.go

@ -401,6 +401,18 @@ func parsePagination(r *http.Request) (pageToken string, pageSize int, err error
return pageToken, parsedPageSize, nil
}
func normalizeNamespaceProperties(properties map[string]string) map[string]string {
if len(properties) == 0 {
return map[string]string{}
}
normalized := make(map[string]string, len(properties))
for k, v := range properties {
normalized[k] = v
}
return normalized
}
// handleConfig returns catalog configuration.
func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -482,6 +494,7 @@ func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
createReq := &s3tables.CreateNamespaceRequest{
TableBucketARN: bucketARN,
Namespace: req.Namespace,
Properties: normalizeNamespaceProperties(req.Properties),
}
var createResp s3tables.CreateNamespaceResponse
@ -503,15 +516,9 @@ func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
return
}
// Standardize property initialization for consistency with GetNamespace
props := req.Properties
if props == nil {
props = make(map[string]string)
}
result := CreateNamespaceResponse{
Namespace: req.Namespace,
Properties: props,
Namespace: Namespace(createResp.Namespace),
Properties: normalizeNamespaceProperties(createResp.Properties),
}
writeJSON(w, http.StatusOK, result)
}
@ -554,8 +561,8 @@ func (s *Server) handleGetNamespace(w http.ResponseWriter, r *http.Request) {
}
result := GetNamespaceResponse{
Namespace: namespace,
Properties: make(map[string]string),
Namespace: Namespace(getResp.Namespace),
Properties: normalizeNamespaceProperties(getResp.Properties),
}
writeJSON(w, http.StatusOK, result)
}

29
weed/s3api/iceberg/iceberg_namespace_properties_test.go

@ -0,0 +1,29 @@
package iceberg
import "testing"
func TestNormalizeNamespacePropertiesNil(t *testing.T) {
properties := normalizeNamespaceProperties(nil)
if properties == nil {
t.Fatalf("normalizeNamespaceProperties(nil) returned nil map")
}
if len(properties) != 0 {
t.Fatalf("normalizeNamespaceProperties(nil) length = %d, want 0", len(properties))
}
}
func TestNormalizeNamespacePropertiesClonesInput(t *testing.T) {
input := map[string]string{
"owner": "analytics",
}
properties := normalizeNamespaceProperties(input)
if properties["owner"] != "analytics" {
t.Fatalf("normalized properties value = %q, want %q", properties["owner"], "analytics")
}
input["owner"] = "mutated"
if properties["owner"] != "analytics" {
t.Fatalf("normalized properties was mutated via input map")
}
}

3
weed/s3api/s3tables/handler_namespace.go

@ -147,6 +147,7 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
Namespace: req.Namespace,
CreatedAt: now,
OwnerAccountID: bucketMetadata.OwnerAccountID,
Properties: req.Properties,
}
metadataBytes, err := json.Marshal(metadata)
@ -177,6 +178,7 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
resp := &CreateNamespaceResponse{
Namespace: req.Namespace,
TableBucketARN: req.TableBucketARN,
Properties: req.Properties,
}
h.writeJSON(w, http.StatusOK, resp)
@ -265,6 +267,7 @@ func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Requ
Namespace: metadata.Namespace,
CreatedAt: metadata.CreatedAt,
OwnerAccountID: metadata.OwnerAccountID,
Properties: metadata.Properties,
}
h.writeJSON(w, http.StatusOK, resp)

24
weed/s3api/s3tables/types.go

@ -77,19 +77,22 @@ type DeleteTableBucketPolicyRequest struct {
// Namespace types
type Namespace struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
type CreateNamespaceRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Properties map[string]string `json:"properties,omitempty"`
}
type CreateNamespaceResponse struct {
Namespace []string `json:"namespace"`
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
TableBucketARN string `json:"tableBucketARN"`
Properties map[string]string `json:"properties,omitempty"`
}
type GetNamespaceRequest struct {
@ -98,9 +101,10 @@ type GetNamespaceRequest struct {
}
type GetNamespaceResponse struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
type ListNamespacesRequest struct {

7
weed/s3api/s3tables/utils.go

@ -128,9 +128,10 @@ type tableBucketMetadata struct {
// namespaceMetadata stores metadata for a namespace
type namespaceMetadata struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
// tableMetadataInternal stores metadata for a table

23
weed/s3api/s3tables/utils_namespace_test.go

@ -1,6 +1,7 @@
package s3tables
import (
"encoding/json"
"strings"
"testing"
)
@ -124,3 +125,25 @@ func TestExpandNamespace(t *testing.T) {
}
}
}
func TestNamespaceMetadataPropertiesRoundTrip(t *testing.T) {
metadata := namespaceMetadata{
Namespace: []string{"analytics"},
Properties: map[string]string{"owner": "finance"},
OwnerAccountID: "123456789012",
}
data, err := json.Marshal(metadata)
if err != nil {
t.Fatalf("json.Marshal(metadata) returned error: %v", err)
}
var decoded namespaceMetadata
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("json.Unmarshal(data) returned error: %v", err)
}
if decoded.Properties["owner"] != "finance" {
t.Fatalf("decoded.Properties[owner] = %q, want %q", decoded.Properties["owner"], "finance")
}
}
Loading…
Cancel
Save