Browse Source

address comments

pull/7178/head
chrislu 4 weeks ago
parent
commit
b768d88087
  1. 6
      test/foundationdb/Dockerfile.test
  2. 7
      test/foundationdb/docker-compose.arm64.yml
  3. 1
      test/foundationdb/docker-compose.yml
  4. 4
      test/foundationdb/filer.toml
  5. 2
      test/foundationdb/foundationdb_concurrent_test.go
  6. 2
      test/foundationdb/foundationdb_integration_test.go
  7. 30
      test/foundationdb/mock_integration_test.go
  8. 14
      weed/filer/foundationdb/CONFIGURATION.md
  9. 16
      weed/filer/foundationdb/INSTALL.md
  10. 4
      weed/filer/foundationdb/README.md
  11. 4
      weed/filer/foundationdb/foundationdb_store_test.go

6
test/foundationdb/Dockerfile.test

@ -9,9 +9,9 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
# Download and install FoundationDB client libraries
RUN wget -q https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb \
&& dpkg -i foundationdb-clients_7.1.61-1_amd64.deb || apt-get install -f -y \
&& rm foundationdb-clients_7.1.61-1_amd64.deb
RUN wget -q https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-clients_7.4.5-1_amd64.deb \
&& dpkg -i foundationdb-clients_7.4.5-1_amd64.deb || apt-get install -f -y \
&& rm foundationdb-clients_7.4.5-1_amd64.deb
# Set up Go environment for CGO
ENV CGO_ENABLED=1

7
test/foundationdb/docker-compose.arm64.yml

@ -25,7 +25,7 @@ services:
bash -c "
# Initialize cluster configuration
if [ ! -f /var/fdb/config/fdb.cluster ]; then
echo 'testing:testing@fdb1:4500,fdb2:4500,fdb3:4500' > /var/fdb/config/fdb.cluster
echo 'testing:testing@fdb1:4500,fdb2:4502,fdb3:4504' > /var/fdb/config/fdb.cluster
fi
# Start FDB processes
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
@ -113,6 +113,7 @@ services:
- fdb3
command: |
bash -c "
set -euo pipefail
# Wait for cluster file
while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
@ -121,13 +122,13 @@ services:
# Configure database
echo 'Initializing FoundationDB database...'
fdbcli --exec 'configure new single ssd'
fdbcli -C /var/fdb/config/fdb.cluster --exec 'configure new single ssd'
# Wait for configuration to complete
sleep 5
# Verify cluster status
fdbcli --exec 'status'
fdbcli -C /var/fdb/config/fdb.cluster --exec 'status'
echo 'FoundationDB cluster initialization complete!'

1
test/foundationdb/docker-compose.yml

@ -48,7 +48,6 @@ services:
# Initialize and configure the database
fdb-init:
image: $FOUNDATIONDB_IMAGE
platform: linux/amd64
configs:
- target: /var/fdb/config/fdb.cluster
source: fdb.cluster

4
test/foundationdb/filer.toml

@ -3,7 +3,7 @@
[foundationdb]
enabled = true
cluster_file = "/var/fdb/config/fdb.cluster"
api_version = 720
api_version = 740
timeout = "5s"
max_retry_delay = "1s"
directory_prefix = "seaweedfs"
@ -12,7 +12,7 @@ directory_prefix = "seaweedfs"
[foundationdb.test]
enabled = false
cluster_file = "/var/fdb/config/fdb.cluster"
api_version = 720
api_version = 740
timeout = "10s"
max_retry_delay = "2s"
directory_prefix = "seaweedfs_test"

2
test/foundationdb/foundationdb_concurrent_test.go

@ -430,7 +430,7 @@ func createTestStore(t *testing.T) *foundationdb.FoundationDBStore {
config := util.GetViper()
config.Set("foundationdb.cluster_file", clusterFile)
config.Set("foundationdb.api_version", 720)
config.Set("foundationdb.api_version", 740)
config.Set("foundationdb.timeout", "10s")
config.Set("foundationdb.max_retry_delay", "2s")
config.Set("foundationdb.directory_prefix", fmt.Sprintf("seaweedfs_concurrent_test_%d", time.Now().UnixNano()))

2
test/foundationdb/foundationdb_integration_test.go

@ -354,7 +354,7 @@ func createTestStore(t *testing.T) *foundationdb.FoundationDBStore {
config := util.GetViper()
config.Set("foundationdb.cluster_file", clusterFile)
config.Set("foundationdb.api_version", 630)
config.Set("foundationdb.api_version", 740)
config.Set("foundationdb.timeout", "10s")
config.Set("foundationdb.max_retry_delay", "2s")
config.Set("foundationdb.directory_prefix", "seaweedfs_test")

30
test/foundationdb/mock_integration_test.go

@ -2,6 +2,7 @@ package foundationdb
import (
"context"
"sort"
"strings"
"testing"
"time"
@ -121,12 +122,32 @@ func (store *MockFoundationDBStore) ListDirectoryPrefixedEntries(ctx context.Con
}
}
// Simple sorting (not comprehensive)
for i, entryPath := range entries {
if int64(i) >= limit {
break
// Sort entries for consistent ordering
sort.Strings(entries)
// Apply startFileName filter
startIndex := 0
if startFileName != "" {
for i, entryPath := range entries {
fileName := strings.TrimPrefix(entryPath, dirPrefix)
if fileName == startFileName {
if includeStartFile {
startIndex = i
} else {
startIndex = i + 1
}
break
} else if fileName > startFileName {
startIndex = i
break
}
}
}
// Iterate through sorted entries with limit
count := int64(0)
for i := startIndex; i < len(entries) && count < limit; i++ {
entryPath := entries[i]
data := store.data[entryPath]
entry := &filer.Entry{
FullPath: util.FullPath(entryPath),
@ -140,6 +161,7 @@ func (store *MockFoundationDBStore) ListDirectoryPrefixedEntries(ctx context.Con
break
}
lastFileName = entry.Name()
count++
}
return lastFileName, nil

14
weed/filer/foundationdb/CONFIGURATION.md

@ -10,7 +10,7 @@ This document provides comprehensive configuration options for the FoundationDB
[foundationdb]
enabled = true
cluster_file = "/etc/foundationdb/fdb.cluster"
api_version = 720
api_version = 740
timeout = "5s"
max_retry_delay = "1s"
directory_prefix = "seaweedfs"
@ -41,7 +41,7 @@ While not directly supported, configuration can be specified via config files pa
|--------|------|---------|-------------|
| `enabled` | boolean | `false` | Enable the FoundationDB filer store |
| `cluster_file` | string | `/etc/foundationdb/fdb.cluster` | Path to FoundationDB cluster file |
| `api_version` | integer | `720` | FoundationDB API version to use |
| `api_version` | integer | `740` | FoundationDB API version to use |
### Connection Options
@ -64,7 +64,7 @@ While not directly supported, configuration can be specified via config files pa
[foundationdb]
enabled = true
cluster_file = "/var/fdb/config/fdb.cluster"
api_version = 720
api_version = 740
timeout = "10s"
max_retry_delay = "2s"
directory_prefix = "seaweedfs_dev"
@ -76,7 +76,7 @@ directory_prefix = "seaweedfs_dev"
[foundationdb]
enabled = true
cluster_file = "/etc/foundationdb/fdb.cluster"
api_version = 720
api_version = 740
timeout = "30s"
max_retry_delay = "5s"
directory_prefix = "seaweedfs_prod"
@ -88,7 +88,7 @@ directory_prefix = "seaweedfs_prod"
[foundationdb]
enabled = true
cluster_file = "/etc/foundationdb/fdb.cluster"
api_version = 720
api_version = 740
timeout = "60s"
max_retry_delay = "10s"
directory_prefix = "sw" # Shorter prefix for efficiency
@ -135,7 +135,7 @@ The following settings are required for FoundationDB to function:
### Validation Rules
- `api_version` must be between 600 and 720
- `api_version` must be between 600 and 740
- `timeout` must be a valid duration string (e.g., "5s", "30s", "2m")
- `max_retry_delay` must be a valid duration string
- `cluster_file` must exist and be readable
@ -289,7 +289,7 @@ data:
[foundationdb]
enabled = true
cluster_file = "/var/fdb/config/cluster_file"
api_version = 720
api_version = 740
timeout = "30s"
max_retry_delay = "5s"
directory_prefix = "k8s_seaweedfs"

16
weed/filer/foundationdb/INSTALL.md

@ -11,8 +11,8 @@ This guide covers the installation and setup of the FoundationDB filer store for
**Ubuntu/Debian:**
```bash
# Add FoundationDB repository
curl -L https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb -o foundationdb-clients.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-server_7.1.61-1_amd64.deb -o foundationdb-server.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-clients_7.4.5-1_amd64.deb -o foundationdb-clients.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-server_7.4.5-1_amd64.deb -o foundationdb-server.deb
sudo dpkg -i foundationdb-clients.deb foundationdb-server.deb
```
@ -20,10 +20,10 @@ This guide covers the installation and setup of the FoundationDB filer store for
**CentOS/RHEL:**
```bash
# Install RPM packages
wget https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients-7.1.61-1.el7.x86_64.rpm
wget https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-server-7.1.61-1.el7.x86_64.rpm
wget https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-clients-7.4.5-1.el7.x86_64.rpm
wget https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-server-7.4.5-1.el7.x86_64.rpm
sudo rpm -Uvh foundationdb-clients-7.1.61-1.el7.x86_64.rpm foundationdb-server-7.1.61-1.el7.x86_64.rpm
sudo rpm -Uvh foundationdb-clients-7.4.5-1.el7.x86_64.rpm foundationdb-server-7.4.5-1.el7.x86_64.rpm
```
**macOS:**
@ -122,7 +122,7 @@ Create or edit `filer.toml`:
[foundationdb]
enabled = true
cluster_file = "/etc/foundationdb/fdb.cluster"
api_version = 720
api_version = 740
timeout = "5s"
max_retry_delay = "1s"
directory_prefix = "seaweedfs"
@ -149,7 +149,7 @@ For production deployments:
[foundationdb]
enabled = true
cluster_file = "/etc/foundationdb/fdb.cluster"
api_version = 720
api_version = 740
timeout = "30s"
max_retry_delay = "5s"
directory_prefix = "seaweedfs_prod"
@ -208,7 +208,7 @@ timeout = "60s"
version: '3.9'
services:
foundationdb:
image: foundationdb/foundationdb:7.1.61
image: foundationdb/foundationdb:7.4.5
ports:
- "4500:4500"
volumes:

4
weed/filer/foundationdb/README.md

@ -35,7 +35,7 @@ Add the following to your `filer.toml`:
[foundationdb]
enabled = true
cluster_file = "/etc/foundationdb/fdb.cluster"
api_version = 720
api_version = 740
timeout = "5s"
max_retry_delay = "1s"
directory_prefix = "seaweedfs"
@ -47,7 +47,7 @@ directory_prefix = "seaweedfs"
|--------|-------------|---------|----------|
| `enabled` | Enable FoundationDB filer store | `false` | Yes |
| `cluster_file` | Path to FDB cluster file | `/etc/foundationdb/fdb.cluster` | Yes |
| `api_version` | FoundationDB API version | `720` | No |
| `api_version` | FoundationDB API version | `740` | No |
| `timeout` | Operation timeout duration | `5s` | No |
| `max_retry_delay` | Maximum retry delay | `1s` | No |
| `directory_prefix` | Directory prefix for organization | `seaweedfs` | No |

4
weed/filer/foundationdb/foundationdb_store_test.go

@ -77,7 +77,7 @@ func TestFoundationDBStore_InitializeInvalidConfig(t *testing.T) {
name: "invalid timeout",
config: map[string]interface{}{
"foundationdb.cluster_file": getTestClusterFile(),
"foundationdb.api_version": 720,
"foundationdb.api_version": 740,
"foundationdb.timeout": "invalid",
"foundationdb.directory_prefix": "test",
},
@ -87,7 +87,7 @@ func TestFoundationDBStore_InitializeInvalidConfig(t *testing.T) {
name: "invalid max_retry_delay",
config: map[string]interface{}{
"foundationdb.cluster_file": getTestClusterFile(),
"foundationdb.api_version": 720,
"foundationdb.api_version": 740,
"foundationdb.timeout": "5s",
"foundationdb.max_retry_delay": "invalid",
"foundationdb.directory_prefix": "test",

Loading…
Cancel
Save