Browse Source
chore(deps): bump org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0 in /other/java/hdfs3 (#7512)
chore(deps): bump org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0 in /other/java/hdfs3 (#7512)
* chore(deps): bump org.apache.hadoop:hadoop-common in /other/java/hdfs3 Bumps org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0. --- updated-dependencies: - dependency-name: org.apache.hadoop:hadoop-common dependency-version: 3.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> * add java client unit tests * Update dependency-reduced-pom.xml * add java integration tests * fix * fix buffer --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: chrislu <chris.lu@gmail.com>pull/7514/head
dependabot[bot]
6 days ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 2585 additions and 10 deletions
-
170.github/workflows/java_integration_tests.yml
-
64.github/workflows/java_unit_tests.yml
-
323other/java/client/src/test/java/seaweedfs/client/FilerClientIntegrationTest.java
-
417other/java/client/src/test/java/seaweedfs/client/SeaweedStreamIntegrationTest.java
-
190other/java/hdfs2/README.md
-
19other/java/hdfs2/pom.xml
-
90other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java
-
379other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java
-
190other/java/hdfs3/README.md
-
263other/java/hdfs3/dependency-reduced-pom.xml
-
21other/java/hdfs3/pom.xml
-
90other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java
-
379other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java
@ -0,0 +1,170 @@ |
|||
name: Java Client Integration Tests |
|||
|
|||
on: |
|||
push: |
|||
branches: [ master ] |
|||
paths: |
|||
- 'other/java/**' |
|||
- 'weed/**' |
|||
- '.github/workflows/java_integration_tests.yml' |
|||
pull_request: |
|||
branches: [ master ] |
|||
paths: |
|||
- 'other/java/**' |
|||
- 'weed/**' |
|||
- '.github/workflows/java_integration_tests.yml' |
|||
|
|||
jobs: |
|||
test: |
|||
name: Java Integration Tests |
|||
runs-on: ubuntu-latest |
|||
|
|||
strategy: |
|||
matrix: |
|||
java: ['11', '17'] |
|||
|
|||
steps: |
|||
- name: Checkout code |
|||
uses: actions/checkout@v4 |
|||
|
|||
- name: Set up Go |
|||
uses: actions/setup-go@v6 |
|||
with: |
|||
go-version-file: 'go.mod' |
|||
id: go |
|||
|
|||
- name: Set up Java |
|||
uses: actions/setup-java@v4 |
|||
with: |
|||
java-version: ${{ matrix.java }} |
|||
distribution: 'temurin' |
|||
cache: 'maven' |
|||
|
|||
- name: Build SeaweedFS |
|||
run: | |
|||
cd weed |
|||
go install -buildvcs=false |
|||
weed version |
|||
|
|||
- name: Start SeaweedFS Server |
|||
run: | |
|||
# Create clean data directory |
|||
export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)" |
|||
mkdir -p "$WEED_DATA_DIR" |
|||
|
|||
# Start SeaweedFS with optimized settings for CI |
|||
weed server -dir="$WEED_DATA_DIR" \ |
|||
-master.raftHashicorp \ |
|||
-master.electionTimeout=1s \ |
|||
-master.volumeSizeLimitMB=100 \ |
|||
-volume.max=100 \ |
|||
-volume.preStopSeconds=1 \ |
|||
-master.peers=none \ |
|||
-filer -filer.maxMB=64 \ |
|||
-master.port=9333 \ |
|||
-volume.port=8080 \ |
|||
-filer.port=8888 \ |
|||
-metricsPort=9324 > seaweedfs.log 2>&1 & |
|||
|
|||
SERVER_PID=$! |
|||
echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV |
|||
echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV |
|||
echo "SeaweedFS server started with PID: $SERVER_PID" |
|||
|
|||
- name: Wait for SeaweedFS Components |
|||
run: | |
|||
echo "Waiting for SeaweedFS components to start..." |
|||
|
|||
# Wait for master |
|||
for i in {1..30}; do |
|||
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then |
|||
echo "✓ Master server is ready" |
|||
break |
|||
fi |
|||
echo "Waiting for master server... ($i/30)" |
|||
sleep 2 |
|||
done |
|||
|
|||
# Wait for volume |
|||
for i in {1..30}; do |
|||
if curl -s http://localhost:8080/status > /dev/null 2>&1; then |
|||
echo "✓ Volume server is ready" |
|||
break |
|||
fi |
|||
echo "Waiting for volume server... ($i/30)" |
|||
sleep 2 |
|||
done |
|||
|
|||
# Wait for filer |
|||
for i in {1..30}; do |
|||
if curl -s http://localhost:8888/ > /dev/null 2>&1; then |
|||
echo "✓ Filer is ready" |
|||
break |
|||
fi |
|||
echo "Waiting for filer... ($i/30)" |
|||
sleep 2 |
|||
done |
|||
|
|||
echo "✓ All SeaweedFS components are ready!" |
|||
|
|||
# Display cluster status |
|||
echo "Cluster status:" |
|||
curl -s http://localhost:9333/cluster/status | head -20 |
|||
|
|||
- name: Build and Install SeaweedFS Client |
|||
working-directory: other/java/client |
|||
run: | |
|||
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true |
|||
|
|||
- name: Run Client Unit Tests |
|||
working-directory: other/java/client |
|||
run: | |
|||
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest |
|||
|
|||
- name: Run Client Integration Tests |
|||
working-directory: other/java/client |
|||
env: |
|||
SEAWEEDFS_TEST_ENABLED: true |
|||
run: | |
|||
mvn test -Dtest=*IntegrationTest |
|||
|
|||
- name: Run HDFS2 Configuration Tests |
|||
working-directory: other/java/hdfs2 |
|||
run: | |
|||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true |
|||
|
|||
- name: Run HDFS3 Configuration Tests |
|||
working-directory: other/java/hdfs3 |
|||
run: | |
|||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true |
|||
|
|||
- name: Display logs on failure |
|||
if: failure() |
|||
run: | |
|||
echo "=== SeaweedFS Server Log ===" |
|||
tail -100 seaweedfs.log || echo "No server log" |
|||
echo "" |
|||
echo "=== Cluster Status ===" |
|||
curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster" |
|||
echo "" |
|||
echo "=== Process Status ===" |
|||
ps aux | grep weed || echo "No weed processes" |
|||
|
|||
- name: Cleanup |
|||
if: always() |
|||
run: | |
|||
# Stop server using stored PID |
|||
if [ -n "$SERVER_PID" ]; then |
|||
echo "Stopping SeaweedFS server (PID: $SERVER_PID)" |
|||
kill -9 $SERVER_PID 2>/dev/null || true |
|||
fi |
|||
|
|||
# Fallback: kill any remaining weed processes |
|||
pkill -f "weed server" || true |
|||
|
|||
# Clean up data directory |
|||
if [ -n "$WEED_DATA_DIR" ]; then |
|||
echo "Cleaning up data directory: $WEED_DATA_DIR" |
|||
rm -rf "$WEED_DATA_DIR" || true |
|||
fi |
|||
|
|||
@ -0,0 +1,64 @@ |
|||
name: Java Client Unit Tests |
|||
|
|||
on: |
|||
push: |
|||
branches: [ master ] |
|||
paths: |
|||
- 'other/java/**' |
|||
- '.github/workflows/java_unit_tests.yml' |
|||
pull_request: |
|||
branches: [ master ] |
|||
paths: |
|||
- 'other/java/**' |
|||
- '.github/workflows/java_unit_tests.yml' |
|||
|
|||
jobs: |
|||
test: |
|||
name: Java Unit Tests |
|||
runs-on: ubuntu-latest |
|||
|
|||
strategy: |
|||
matrix: |
|||
java: ['8', '11', '17', '21'] |
|||
|
|||
steps: |
|||
- name: Checkout code |
|||
uses: actions/checkout@v4 |
|||
|
|||
- name: Set up Java |
|||
uses: actions/setup-java@v4 |
|||
with: |
|||
java-version: ${{ matrix.java }} |
|||
distribution: 'temurin' |
|||
cache: 'maven' |
|||
|
|||
- name: Build and Install SeaweedFS Client |
|||
working-directory: other/java/client |
|||
run: | |
|||
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true |
|||
|
|||
- name: Run Client Unit Tests |
|||
working-directory: other/java/client |
|||
run: | |
|||
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest |
|||
|
|||
- name: Run HDFS2 Configuration Tests |
|||
working-directory: other/java/hdfs2 |
|||
run: | |
|||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true |
|||
|
|||
- name: Run HDFS3 Configuration Tests |
|||
working-directory: other/java/hdfs3 |
|||
run: | |
|||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true |
|||
|
|||
- name: Upload Test Reports |
|||
if: always() |
|||
uses: actions/upload-artifact@v5 |
|||
with: |
|||
name: test-reports-java-${{ matrix.java }} |
|||
path: | |
|||
other/java/client/target/surefire-reports/ |
|||
other/java/hdfs2/target/surefire-reports/ |
|||
other/java/hdfs3/target/surefire-reports/ |
|||
|
|||
@ -0,0 +1,323 @@ |
|||
package seaweedfs.client; |
|||
|
|||
import org.junit.After; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
|
|||
import java.io.IOException; |
|||
import java.nio.charset.StandardCharsets; |
|||
import java.util.List; |
|||
|
|||
import static org.junit.Assert.*; |
|||
|
|||
/** |
|||
* Integration tests for FilerClient. |
|||
* |
|||
* These tests verify FilerClient operations against a running SeaweedFS filer |
|||
* instance. |
|||
* |
|||
* Prerequisites: |
|||
* - SeaweedFS master, volume server, and filer must be running |
|||
* - Default ports: filer HTTP 8888, filer gRPC 18888 |
|||
* |
|||
* To run tests: |
|||
* export SEAWEEDFS_TEST_ENABLED=true |
|||
* mvn test -Dtest=FilerClientIntegrationTest |
|||
*/ |
|||
public class FilerClientIntegrationTest { |
|||
|
|||
private FilerClient filerClient; |
|||
private static final String TEST_ROOT = "/test-client-integration"; |
|||
private static final boolean TESTS_ENABLED = "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); |
|||
|
|||
@Before |
|||
public void setUp() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
return; |
|||
} |
|||
|
|||
filerClient = new FilerClient("localhost", 18888); |
|||
|
|||
// Clean up any existing test directory |
|||
if (filerClient.exists(TEST_ROOT)) { |
|||
filerClient.rm(TEST_ROOT, true, true); |
|||
} |
|||
|
|||
// Create test root directory |
|||
filerClient.mkdirs(TEST_ROOT, 0755); |
|||
} |
|||
|
|||
@After |
|||
public void tearDown() throws Exception { |
|||
if (!TESTS_ENABLED || filerClient == null) { |
|||
return; |
|||
} |
|||
|
|||
try { |
|||
// Clean up test directory |
|||
if (filerClient.exists(TEST_ROOT)) { |
|||
filerClient.rm(TEST_ROOT, true, true); |
|||
} |
|||
} finally { |
|||
filerClient.shutdown(); |
|||
} |
|||
} |
|||
|
|||
@Test |
|||
public void testMkdirs() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testDir = TEST_ROOT + "/testdir"; |
|||
boolean success = filerClient.mkdirs(testDir, 0755); |
|||
|
|||
assertTrue("Directory creation should succeed", success); |
|||
assertTrue("Directory should exist", filerClient.exists(testDir)); |
|||
} |
|||
|
|||
@Test |
|||
public void testTouch() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testFile = TEST_ROOT + "/testfile.txt"; |
|||
boolean success = filerClient.touch(testFile, 0644); |
|||
|
|||
assertTrue("Touch should succeed", success); |
|||
assertTrue("File should exist", filerClient.exists(testFile)); |
|||
} |
|||
|
|||
@Test |
|||
public void testExists() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
assertTrue("Root should exist", filerClient.exists("/")); |
|||
assertTrue("Test root should exist", filerClient.exists(TEST_ROOT)); |
|||
assertFalse("Non-existent path should not exist", |
|||
filerClient.exists(TEST_ROOT + "/nonexistent")); |
|||
} |
|||
|
|||
@Test |
|||
public void testListEntries() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
// Create some test files and directories |
|||
filerClient.touch(TEST_ROOT + "/file1.txt", 0644); |
|||
filerClient.touch(TEST_ROOT + "/file2.txt", 0644); |
|||
filerClient.mkdirs(TEST_ROOT + "/subdir", 0755); |
|||
|
|||
List<FilerProto.Entry> entries = filerClient.listEntries(TEST_ROOT); |
|||
|
|||
assertNotNull("Entries should not be null", entries); |
|||
assertEquals("Should have 3 entries", 3, entries.size()); |
|||
} |
|||
|
|||
@Test |
|||
public void testListEntriesWithPrefix() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
// Create test files |
|||
filerClient.touch(TEST_ROOT + "/test1.txt", 0644); |
|||
filerClient.touch(TEST_ROOT + "/test2.txt", 0644); |
|||
filerClient.touch(TEST_ROOT + "/other.txt", 0644); |
|||
|
|||
List<FilerProto.Entry> entries = filerClient.listEntries(TEST_ROOT, "test", "", 100, false); |
|||
|
|||
assertNotNull("Entries should not be null", entries); |
|||
assertEquals("Should have 2 entries starting with 'test'", 2, entries.size()); |
|||
} |
|||
|
|||
@Test |
|||
public void testDeleteFile() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testFile = TEST_ROOT + "/deleteme.txt"; |
|||
filerClient.touch(testFile, 0644); |
|||
|
|||
assertTrue("File should exist before delete", filerClient.exists(testFile)); |
|||
|
|||
boolean success = filerClient.rm(testFile, false, true); |
|||
|
|||
assertTrue("Delete should succeed", success); |
|||
assertFalse("File should not exist after delete", filerClient.exists(testFile)); |
|||
} |
|||
|
|||
@Test |
|||
public void testDeleteDirectoryRecursive() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testDir = TEST_ROOT + "/deletedir"; |
|||
filerClient.mkdirs(testDir, 0755); |
|||
filerClient.touch(testDir + "/file.txt", 0644); |
|||
|
|||
assertTrue("Directory should exist", filerClient.exists(testDir)); |
|||
assertTrue("File should exist", filerClient.exists(testDir + "/file.txt")); |
|||
|
|||
boolean success = filerClient.rm(testDir, true, true); |
|||
|
|||
assertTrue("Delete should succeed", success); |
|||
assertFalse("Directory should not exist after delete", filerClient.exists(testDir)); |
|||
} |
|||
|
|||
@Test |
|||
public void testRename() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String srcFile = TEST_ROOT + "/source.txt"; |
|||
String dstFile = TEST_ROOT + "/destination.txt"; |
|||
|
|||
filerClient.touch(srcFile, 0644); |
|||
assertTrue("Source file should exist", filerClient.exists(srcFile)); |
|||
|
|||
boolean success = filerClient.mv(srcFile, dstFile); |
|||
|
|||
assertTrue("Rename should succeed", success); |
|||
assertFalse("Source file should not exist after rename", filerClient.exists(srcFile)); |
|||
assertTrue("Destination file should exist after rename", filerClient.exists(dstFile)); |
|||
} |
|||
|
|||
@Test |
|||
public void testGetEntry() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testFile = TEST_ROOT + "/getentry.txt"; |
|||
filerClient.touch(testFile, 0644); |
|||
|
|||
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "getentry.txt"); |
|||
|
|||
assertNotNull("Entry should not be null", entry); |
|||
assertEquals("Entry name should match", "getentry.txt", entry.getName()); |
|||
assertFalse("Entry should not be a directory", entry.getIsDirectory()); |
|||
} |
|||
|
|||
@Test |
|||
public void testGetEntryForDirectory() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testDir = TEST_ROOT + "/testsubdir"; |
|||
filerClient.mkdirs(testDir, 0755); |
|||
|
|||
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "testsubdir"); |
|||
|
|||
assertNotNull("Entry should not be null", entry); |
|||
assertEquals("Entry name should match", "testsubdir", entry.getName()); |
|||
assertTrue("Entry should be a directory", entry.getIsDirectory()); |
|||
} |
|||
|
|||
@Test |
|||
public void testCreateAndListNestedDirectories() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String nestedPath = TEST_ROOT + "/level1/level2/level3"; |
|||
boolean success = filerClient.mkdirs(nestedPath, 0755); |
|||
|
|||
assertTrue("Nested directory creation should succeed", success); |
|||
assertTrue("Nested directory should exist", filerClient.exists(nestedPath)); |
|||
|
|||
// Verify each level exists |
|||
assertTrue("Level 1 should exist", filerClient.exists(TEST_ROOT + "/level1")); |
|||
assertTrue("Level 2 should exist", filerClient.exists(TEST_ROOT + "/level1/level2")); |
|||
assertTrue("Level 3 should exist", filerClient.exists(nestedPath)); |
|||
} |
|||
|
|||
@Test |
|||
public void testMultipleFilesInDirectory() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testDir = TEST_ROOT + "/multifiles"; |
|||
filerClient.mkdirs(testDir, 0755); |
|||
|
|||
// Create 10 files |
|||
for (int i = 0; i < 10; i++) { |
|||
filerClient.touch(testDir + "/file" + i + ".txt", 0644); |
|||
} |
|||
|
|||
List<FilerProto.Entry> entries = filerClient.listEntries(testDir); |
|||
|
|||
assertNotNull("Entries should not be null", entries); |
|||
assertEquals("Should have 10 files", 10, entries.size()); |
|||
} |
|||
|
|||
@Test |
|||
public void testRenameDirectory() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String srcDir = TEST_ROOT + "/sourcedir"; |
|||
String dstDir = TEST_ROOT + "/destdir"; |
|||
|
|||
filerClient.mkdirs(srcDir, 0755); |
|||
filerClient.touch(srcDir + "/file.txt", 0644); |
|||
|
|||
boolean success = filerClient.mv(srcDir, dstDir); |
|||
|
|||
assertTrue("Directory rename should succeed", success); |
|||
assertFalse("Source directory should not exist", filerClient.exists(srcDir)); |
|||
assertTrue("Destination directory should exist", filerClient.exists(dstDir)); |
|||
assertTrue("File should exist in destination", filerClient.exists(dstDir + "/file.txt")); |
|||
} |
|||
|
|||
@Test |
|||
public void testLookupNonExistentEntry() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "nonexistent.txt"); |
|||
|
|||
assertNull("Entry for non-existent file should be null", entry); |
|||
} |
|||
|
|||
@Test |
|||
public void testEmptyDirectory() { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String emptyDir = TEST_ROOT + "/emptydir"; |
|||
filerClient.mkdirs(emptyDir, 0755); |
|||
|
|||
List<FilerProto.Entry> entries = filerClient.listEntries(emptyDir); |
|||
|
|||
assertNotNull("Entries should not be null", entries); |
|||
assertTrue("Empty directory should have no entries", entries.isEmpty()); |
|||
} |
|||
} |
|||
@ -0,0 +1,417 @@ |
|||
package seaweedfs.client; |
|||
|
|||
import org.junit.After; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
|
|||
import java.io.IOException; |
|||
import java.nio.charset.StandardCharsets; |
|||
import java.util.Arrays; |
|||
import java.util.Random; |
|||
|
|||
import static org.junit.Assert.*; |
|||
|
|||
/** |
|||
* Integration tests for SeaweedInputStream and SeaweedOutputStream. |
|||
* |
|||
* These tests verify stream operations against a running SeaweedFS instance. |
|||
* |
|||
* Prerequisites: |
|||
* - SeaweedFS master, volume server, and filer must be running |
|||
* - Default ports: filer HTTP 8888, filer gRPC 18888 |
|||
* |
|||
* To run tests: |
|||
* export SEAWEEDFS_TEST_ENABLED=true |
|||
* mvn test -Dtest=SeaweedStreamIntegrationTest |
|||
*/ |
|||
public class SeaweedStreamIntegrationTest { |
|||
|
|||
private FilerClient filerClient; |
|||
private static final String TEST_ROOT = "/test-stream-integration"; |
|||
private static final boolean TESTS_ENABLED = |
|||
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); |
|||
|
|||
@Before |
|||
public void setUp() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
return; |
|||
} |
|||
|
|||
filerClient = new FilerClient("localhost", 18888); |
|||
|
|||
// Clean up any existing test directory |
|||
if (filerClient.exists(TEST_ROOT)) { |
|||
filerClient.rm(TEST_ROOT, true, true); |
|||
} |
|||
|
|||
// Create test root directory |
|||
filerClient.mkdirs(TEST_ROOT, 0755); |
|||
} |
|||
|
|||
@After |
|||
public void tearDown() throws Exception { |
|||
if (!TESTS_ENABLED || filerClient == null) { |
|||
return; |
|||
} |
|||
|
|||
try { |
|||
// Clean up test directory |
|||
if (filerClient.exists(TEST_ROOT)) { |
|||
filerClient.rm(TEST_ROOT, true, true); |
|||
} |
|||
} finally { |
|||
filerClient.shutdown(); |
|||
} |
|||
} |
|||
|
|||
@Test |
|||
public void testWriteAndReadSmallFile() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/small.txt"; |
|||
String testContent = "Hello, SeaweedFS!"; |
|||
|
|||
// Write file |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
|||
outputStream.close(); |
|||
|
|||
// Verify file exists |
|||
assertTrue("File should exist", filerClient.exists(testPath)); |
|||
|
|||
// Read file |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
assertNotNull("Entry should not be null", entry); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
byte[] buffer = new byte[testContent.length()]; |
|||
int bytesRead = inputStream.read(buffer); |
|||
inputStream.close(); |
|||
|
|||
assertEquals("Should read all bytes", testContent.length(), bytesRead); |
|||
assertEquals("Content should match", testContent, new String(buffer, StandardCharsets.UTF_8)); |
|||
} |
|||
|
|||
@Test |
|||
public void testWriteAndReadLargeFile() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/large.bin"; |
|||
int fileSize = 10 * 1024 * 1024; // 10 MB |
|||
|
|||
// Generate random data |
|||
byte[] originalData = new byte[fileSize]; |
|||
new Random(42).nextBytes(originalData); // Use seed for reproducibility |
|||
|
|||
// Write file |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(originalData); |
|||
outputStream.close(); |
|||
|
|||
// Verify file exists |
|||
assertTrue("File should exist", filerClient.exists(testPath)); |
|||
|
|||
// Read file |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
assertNotNull("Entry should not be null", entry); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
|
|||
// Read file in chunks to handle large files properly |
|||
byte[] readData = new byte[fileSize]; |
|||
int totalRead = 0; |
|||
int bytesRead; |
|||
byte[] buffer = new byte[8192]; // Read in 8KB chunks |
|||
|
|||
while ((bytesRead = inputStream.read(buffer)) > 0) { |
|||
System.arraycopy(buffer, 0, readData, totalRead, bytesRead); |
|||
totalRead += bytesRead; |
|||
} |
|||
inputStream.close(); |
|||
|
|||
assertEquals("Should read all bytes", fileSize, totalRead); |
|||
assertArrayEquals("Content should match", originalData, readData); |
|||
} |
|||
|
|||
@Test |
|||
public void testWriteInChunks() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/chunked.txt"; |
|||
String[] chunks = {"First chunk. ", "Second chunk. ", "Third chunk."}; |
|||
|
|||
// Write file in chunks |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
for (String chunk : chunks) { |
|||
outputStream.write(chunk.getBytes(StandardCharsets.UTF_8)); |
|||
} |
|||
outputStream.close(); |
|||
|
|||
// Read and verify |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
byte[] buffer = new byte[1024]; |
|||
int bytesRead = inputStream.read(buffer); |
|||
inputStream.close(); |
|||
|
|||
String expected = String.join("", chunks); |
|||
String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); |
|||
|
|||
assertEquals("Content should match", expected, actual); |
|||
} |
|||
|
|||
@Test |
|||
public void testReadWithOffset() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/offset.txt"; |
|||
String testContent = "0123456789ABCDEFGHIJ"; |
|||
|
|||
// Write file |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
|||
outputStream.close(); |
|||
|
|||
// Read with offset |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
inputStream.seek(10); // Skip first 10 bytes |
|||
|
|||
byte[] buffer = new byte[10]; |
|||
int bytesRead = inputStream.read(buffer); |
|||
inputStream.close(); |
|||
|
|||
assertEquals("Should read 10 bytes", 10, bytesRead); |
|||
assertEquals("Should read from offset", "ABCDEFGHIJ", |
|||
new String(buffer, StandardCharsets.UTF_8)); |
|||
} |
|||
|
|||
@Test |
|||
public void testReadPartial() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/partial.txt"; |
|||
String testContent = "The quick brown fox jumps over the lazy dog"; |
|||
|
|||
// Write file |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
|||
outputStream.close(); |
|||
|
|||
// Read partial |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
|
|||
// Read only "quick brown" |
|||
inputStream.seek(4); |
|||
byte[] buffer = new byte[11]; |
|||
int bytesRead = inputStream.read(buffer); |
|||
inputStream.close(); |
|||
|
|||
assertEquals("Should read 11 bytes", 11, bytesRead); |
|||
assertEquals("Should read partial content", "quick brown", |
|||
new String(buffer, StandardCharsets.UTF_8)); |
|||
} |
|||
|
|||
@Test |
|||
public void testEmptyFile() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/empty.txt"; |
|||
|
|||
// Write empty file |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.close(); |
|||
|
|||
// Verify file exists |
|||
assertTrue("File should exist", filerClient.exists(testPath)); |
|||
|
|||
// Read empty file |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
assertNotNull("Entry should not be null", entry); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
byte[] buffer = new byte[100]; |
|||
int bytesRead = inputStream.read(buffer); |
|||
inputStream.close(); |
|||
|
|||
assertEquals("Should read 0 bytes from empty file", -1, bytesRead); |
|||
} |
|||
|
|||
@Test |
|||
public void testOverwriteFile() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/overwrite.txt"; |
|||
String originalContent = "Original content"; |
|||
String newContent = "New content that overwrites the original"; |
|||
|
|||
// Write original file |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(originalContent.getBytes(StandardCharsets.UTF_8)); |
|||
outputStream.close(); |
|||
|
|||
// Overwrite file |
|||
outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(newContent.getBytes(StandardCharsets.UTF_8)); |
|||
outputStream.close(); |
|||
|
|||
// Read and verify |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
byte[] buffer = new byte[1024]; |
|||
int bytesRead = inputStream.read(buffer); |
|||
inputStream.close(); |
|||
|
|||
String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); |
|||
assertEquals("Should have new content", newContent, actual); |
|||
} |
|||
|
|||
@Test |
|||
public void testMultipleReads() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/multireads.txt"; |
|||
String testContent = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; |
|||
|
|||
// Write file |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
|||
outputStream.close(); |
|||
|
|||
// Read in multiple small chunks |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
|
|||
StringBuilder result = new StringBuilder(); |
|||
byte[] buffer = new byte[5]; |
|||
int bytesRead; |
|||
while ((bytesRead = inputStream.read(buffer)) > 0) { |
|||
result.append(new String(buffer, 0, bytesRead, StandardCharsets.UTF_8)); |
|||
} |
|||
inputStream.close(); |
|||
|
|||
assertEquals("Should read entire content", testContent, result.toString()); |
|||
} |
|||
|
|||
@Test |
|||
public void testBinaryData() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/binary.bin"; |
|||
byte[] binaryData = new byte[256]; |
|||
for (int i = 0; i < 256; i++) { |
|||
binaryData[i] = (byte) i; |
|||
} |
|||
|
|||
// Write binary file |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(binaryData); |
|||
outputStream.close(); |
|||
|
|||
// Read and verify |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
byte[] readData = new byte[256]; |
|||
int bytesRead = inputStream.read(readData); |
|||
inputStream.close(); |
|||
|
|||
assertEquals("Should read all bytes", 256, bytesRead); |
|||
assertArrayEquals("Binary data should match", binaryData, readData); |
|||
} |
|||
|
|||
@Test |
|||
public void testFlush() throws IOException { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
String testPath = TEST_ROOT + "/flush.txt"; |
|||
String testContent = "Content to flush"; |
|||
|
|||
// Write file with flush |
|||
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
|||
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
|||
outputStream.flush(); // Explicitly flush |
|||
outputStream.close(); |
|||
|
|||
// Verify file was written |
|||
assertTrue("File should exist after flush", filerClient.exists(testPath)); |
|||
|
|||
// Read and verify |
|||
FilerProto.Entry entry = filerClient.lookupEntry( |
|||
SeaweedOutputStream.getParentDirectory(testPath), |
|||
SeaweedOutputStream.getFileName(testPath) |
|||
); |
|||
|
|||
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
|||
byte[] buffer = new byte[testContent.length()]; |
|||
int bytesRead = inputStream.read(buffer); |
|||
inputStream.close(); |
|||
|
|||
assertEquals("Content should match", testContent, |
|||
new String(buffer, 0, bytesRead, StandardCharsets.UTF_8)); |
|||
} |
|||
} |
|||
|
|||
@ -0,0 +1,190 @@ |
|||
# SeaweedFS Hadoop2 Client |
|||
|
|||
Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 2.x/3.x. |
|||
|
|||
## Building |
|||
|
|||
```bash |
|||
mvn clean install |
|||
``` |
|||
|
|||
## Testing |
|||
|
|||
This project includes two types of tests: |
|||
|
|||
### 1. Configuration Tests (No SeaweedFS Required) |
|||
|
|||
These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance: |
|||
|
|||
```bash |
|||
mvn test -Dtest=SeaweedFileSystemConfigTest |
|||
``` |
|||
|
|||
### 2. Integration Tests (Requires SeaweedFS) |
|||
|
|||
These tests verify actual FileSystem operations against a running SeaweedFS instance. |
|||
|
|||
#### Prerequisites |
|||
|
|||
1. Start SeaweedFS with default ports: |
|||
```bash |
|||
# Terminal 1: Start master |
|||
weed master |
|||
|
|||
# Terminal 2: Start volume server |
|||
weed volume -mserver=localhost:9333 |
|||
|
|||
# Terminal 3: Start filer |
|||
weed filer -master=localhost:9333 |
|||
``` |
|||
|
|||
2. Verify services are running: |
|||
- Master: http://localhost:9333 |
|||
- Filer HTTP: http://localhost:8888 |
|||
- Filer gRPC: localhost:18888 |
|||
|
|||
#### Running Integration Tests |
|||
|
|||
```bash |
|||
# Enable integration tests |
|||
export SEAWEEDFS_TEST_ENABLED=true |
|||
|
|||
# Run all tests |
|||
mvn test |
|||
|
|||
# Run specific test |
|||
mvn test -Dtest=SeaweedFileSystemTest |
|||
``` |
|||
|
|||
### Test Configuration |
|||
|
|||
Integration tests can be configured via environment variables or system properties: |
|||
|
|||
- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false) |
|||
- Tests use these default connection settings: |
|||
- Filer Host: localhost |
|||
- Filer HTTP Port: 8888 |
|||
- Filer gRPC Port: 18888 |
|||
|
|||
### Running Tests with Custom Configuration |
|||
|
|||
To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration: |
|||
|
|||
```java |
|||
conf.set("fs.seaweed.filer.host", "your-host"); |
|||
conf.setInt("fs.seaweed.filer.port", 8888); |
|||
conf.setInt("fs.seaweed.filer.port.grpc", 18888); |
|||
``` |
|||
|
|||
## Test Coverage |
|||
|
|||
The test suite covers: |
|||
|
|||
- **Configuration & Initialization** |
|||
- URI parsing and configuration |
|||
- Default values |
|||
- Configuration overrides |
|||
- Working directory management |
|||
|
|||
- **File Operations** |
|||
- Create files |
|||
- Read files |
|||
- Write files |
|||
- Append to files |
|||
- Delete files |
|||
|
|||
- **Directory Operations** |
|||
- Create directories |
|||
- List directory contents |
|||
- Delete directories (recursive and non-recursive) |
|||
|
|||
- **Metadata Operations** |
|||
- Get file status |
|||
- Set permissions |
|||
- Set owner/group |
|||
- Rename files and directories |
|||
|
|||
## Usage in Hadoop |
|||
|
|||
1. Copy the built JAR to your Hadoop classpath: |
|||
```bash |
|||
cp target/seaweedfs-hadoop2-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/ |
|||
``` |
|||
|
|||
2. Configure `core-site.xml`: |
|||
```xml |
|||
<configuration> |
|||
<property> |
|||
<name>fs.seaweedfs.impl</name> |
|||
<value>seaweed.hdfs.SeaweedFileSystem</value> |
|||
</property> |
|||
<property> |
|||
<name>fs.seaweed.filer.host</name> |
|||
<value>localhost</value> |
|||
</property> |
|||
<property> |
|||
<name>fs.seaweed.filer.port</name> |
|||
<value>8888</value> |
|||
</property> |
|||
<property> |
|||
<name>fs.seaweed.filer.port.grpc</name> |
|||
<value>18888</value> |
|||
</property> |
|||
</configuration> |
|||
``` |
|||
|
|||
3. Use SeaweedFS with Hadoop commands: |
|||
```bash |
|||
hadoop fs -ls seaweedfs://localhost:8888/ |
|||
hadoop fs -mkdir seaweedfs://localhost:8888/test |
|||
hadoop fs -put local.txt seaweedfs://localhost:8888/test/ |
|||
``` |
|||
|
|||
## Continuous Integration |
|||
|
|||
For CI environments, tests can be run in two modes: |
|||
|
|||
1. **Configuration Tests Only** (default, no SeaweedFS required): |
|||
```bash |
|||
mvn test -Dtest=SeaweedFileSystemConfigTest |
|||
``` |
|||
|
|||
2. **Full Integration Tests** (requires SeaweedFS): |
|||
```bash |
|||
# Start SeaweedFS in CI environment |
|||
# Then run: |
|||
export SEAWEEDFS_TEST_ENABLED=true |
|||
mvn test |
|||
``` |
|||
|
|||
## Troubleshooting |
|||
|
|||
### Tests are skipped |
|||
|
|||
If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set": |
|||
```bash |
|||
export SEAWEEDFS_TEST_ENABLED=true |
|||
``` |
|||
|
|||
### Connection refused errors |
|||
|
|||
Ensure SeaweedFS is running and accessible: |
|||
```bash |
|||
curl http://localhost:8888/ |
|||
``` |
|||
|
|||
### gRPC errors |
|||
|
|||
Verify the gRPC port is accessible: |
|||
```bash |
|||
# Should show the port is listening |
|||
netstat -an | grep 18888 |
|||
``` |
|||
|
|||
## Contributing |
|||
|
|||
When adding new features, please include: |
|||
1. Configuration tests (no SeaweedFS required) |
|||
2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard) |
|||
3. Documentation updates |
|||
|
|||
@ -0,0 +1,90 @@ |
|||
package seaweed.hdfs; |
|||
|
|||
import org.apache.hadoop.conf.Configuration; |
|||
import org.apache.hadoop.fs.Path; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
|
|||
import static org.junit.Assert.*; |
|||
|
|||
/** |
|||
* Unit tests for SeaweedFileSystem configuration that don't require a running SeaweedFS instance. |
|||
* |
|||
* These tests verify basic properties and constants. |
|||
*/ |
|||
public class SeaweedFileSystemConfigTest { |
|||
|
|||
private SeaweedFileSystem fs; |
|||
private Configuration conf; |
|||
|
|||
@Before |
|||
public void setUp() { |
|||
fs = new SeaweedFileSystem(); |
|||
conf = new Configuration(); |
|||
} |
|||
|
|||
@Test |
|||
public void testScheme() { |
|||
assertEquals("seaweedfs", fs.getScheme()); |
|||
} |
|||
|
|||
@Test |
|||
public void testConstants() { |
|||
// Test that constants are defined correctly |
|||
assertEquals("fs.seaweed.filer.host", SeaweedFileSystem.FS_SEAWEED_FILER_HOST); |
|||
assertEquals("fs.seaweed.filer.port", SeaweedFileSystem.FS_SEAWEED_FILER_PORT); |
|||
assertEquals("fs.seaweed.filer.port.grpc", SeaweedFileSystem.FS_SEAWEED_FILER_PORT_GRPC); |
|||
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); |
|||
assertEquals("fs.seaweed.buffer.size", SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE); |
|||
assertEquals(4 * 1024 * 1024, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); |
|||
assertEquals("fs.seaweed.replication", SeaweedFileSystem.FS_SEAWEED_REPLICATION); |
|||
assertEquals("fs.seaweed.volume.server.access", SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS); |
|||
assertEquals("fs.seaweed.filer.cn", SeaweedFileSystem.FS_SEAWEED_FILER_CN); |
|||
} |
|||
|
|||
@Test |
|||
public void testWorkingDirectoryPathOperations() { |
|||
// Test path operations that don't require initialization |
|||
Path testPath = new Path("/test/path"); |
|||
assertTrue("Path should be absolute", testPath.isAbsolute()); |
|||
assertEquals("/test/path", testPath.toUri().getPath()); |
|||
|
|||
Path childPath = new Path(testPath, "child"); |
|||
assertEquals("/test/path/child", childPath.toUri().getPath()); |
|||
} |
|||
|
|||
@Test |
|||
public void testConfigurationProperties() { |
|||
// Test that configuration can be set and read |
|||
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_HOST, "testhost"); |
|||
assertEquals("testhost", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_HOST)); |
|||
|
|||
conf.setInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 9999); |
|||
assertEquals(9999, conf.getInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 0)); |
|||
|
|||
conf.setInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 8 * 1024 * 1024); |
|||
assertEquals(8 * 1024 * 1024, conf.getInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 0)); |
|||
|
|||
conf.set(SeaweedFileSystem.FS_SEAWEED_REPLICATION, "001"); |
|||
assertEquals("001", conf.get(SeaweedFileSystem.FS_SEAWEED_REPLICATION)); |
|||
|
|||
conf.set(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS, "publicUrl"); |
|||
assertEquals("publicUrl", conf.get(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS)); |
|||
|
|||
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_CN, "test-cn"); |
|||
assertEquals("test-cn", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_CN)); |
|||
} |
|||
|
|||
@Test |
|||
public void testDefaultBufferSize() { |
|||
// Test default buffer size constant |
|||
int expected = 4 * 1024 * 1024; // 4MB |
|||
assertEquals(expected, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); |
|||
} |
|||
|
|||
@Test |
|||
public void testDefaultPort() { |
|||
// Test default port constant |
|||
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); |
|||
} |
|||
} |
|||
@ -0,0 +1,379 @@ |
|||
package seaweed.hdfs; |
|||
|
|||
import org.apache.hadoop.conf.Configuration; |
|||
import org.apache.hadoop.fs.FSDataInputStream; |
|||
import org.apache.hadoop.fs.FSDataOutputStream; |
|||
import org.apache.hadoop.fs.FileStatus; |
|||
import org.apache.hadoop.fs.Path; |
|||
import org.apache.hadoop.fs.permission.FsPermission; |
|||
import org.junit.After; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
|
|||
import java.io.IOException; |
|||
import java.net.URI; |
|||
|
|||
import static org.junit.Assert.*; |
|||
|
|||
/** |
|||
* Unit tests for SeaweedFileSystem. |
|||
* |
|||
* These tests verify basic FileSystem operations against a SeaweedFS backend. |
|||
* Note: These tests require a running SeaweedFS filer instance. |
|||
* |
|||
* To run tests, ensure SeaweedFS is running with default ports: |
|||
* - Filer HTTP: 8888 |
|||
* - Filer gRPC: 18888 |
|||
* |
|||
* Set environment variable SEAWEEDFS_TEST_ENABLED=true to enable these tests. |
|||
*/ |
|||
public class SeaweedFileSystemTest { |
|||
|
|||
private SeaweedFileSystem fs; |
|||
private Configuration conf; |
|||
private static final String TEST_ROOT = "/test-hdfs2"; |
|||
private static final boolean TESTS_ENABLED = |
|||
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); |
|||
|
|||
@Before |
|||
public void setUp() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
return; |
|||
} |
|||
|
|||
conf = new Configuration(); |
|||
conf.set("fs.seaweed.filer.host", "localhost"); |
|||
conf.setInt("fs.seaweed.filer.port", 8888); |
|||
conf.setInt("fs.seaweed.filer.port.grpc", 18888); |
|||
|
|||
fs = new SeaweedFileSystem(); |
|||
URI uri = new URI("seaweedfs://localhost:8888/"); |
|||
fs.initialize(uri, conf); |
|||
|
|||
// Clean up any existing test directory |
|||
Path testPath = new Path(TEST_ROOT); |
|||
if (fs.exists(testPath)) { |
|||
fs.delete(testPath, true); |
|||
} |
|||
} |
|||
|
|||
@After |
|||
public void tearDown() throws Exception { |
|||
if (!TESTS_ENABLED || fs == null) { |
|||
return; |
|||
} |
|||
|
|||
// Clean up test directory |
|||
Path testPath = new Path(TEST_ROOT); |
|||
if (fs.exists(testPath)) { |
|||
fs.delete(testPath, true); |
|||
} |
|||
|
|||
fs.close(); |
|||
} |
|||
|
|||
@Test |
|||
public void testInitialization() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
assertNotNull(fs); |
|||
assertEquals("seaweedfs", fs.getScheme()); |
|||
assertNotNull(fs.getUri()); |
|||
assertEquals("/", fs.getWorkingDirectory().toUri().getPath()); |
|||
} |
|||
|
|||
@Test |
|||
public void testMkdirs() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testDir = new Path(TEST_ROOT + "/testdir"); |
|||
assertTrue("Failed to create directory", fs.mkdirs(testDir)); |
|||
assertTrue("Directory should exist", fs.exists(testDir)); |
|||
|
|||
FileStatus status = fs.getFileStatus(testDir); |
|||
assertTrue("Path should be a directory", status.isDirectory()); |
|||
} |
|||
|
|||
@Test |
|||
public void testCreateAndReadFile() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/testfile.txt"); |
|||
String testContent = "Hello, SeaweedFS!"; |
|||
|
|||
// Create and write to file |
|||
FSDataOutputStream out = fs.create(testFile, FsPermission.getDefault(), |
|||
false, 4096, (short) 1, 4 * 1024 * 1024, null); |
|||
assertNotNull("Output stream should not be null", out); |
|||
out.write(testContent.getBytes()); |
|||
out.close(); |
|||
|
|||
// Verify file exists |
|||
assertTrue("File should exist", fs.exists(testFile)); |
|||
|
|||
// Read and verify content |
|||
FSDataInputStream in = fs.open(testFile, 4096); |
|||
assertNotNull("Input stream should not be null", in); |
|||
byte[] buffer = new byte[testContent.length()]; |
|||
int bytesRead = in.read(buffer); |
|||
in.close(); |
|||
|
|||
assertEquals("Should read all bytes", testContent.length(), bytesRead); |
|||
assertEquals("Content should match", testContent, new String(buffer)); |
|||
} |
|||
|
|||
@Test |
|||
public void testFileStatus() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/statustest.txt"); |
|||
String content = "test content"; |
|||
|
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write(content.getBytes()); |
|||
out.close(); |
|||
|
|||
FileStatus status = fs.getFileStatus(testFile); |
|||
assertNotNull("FileStatus should not be null", status); |
|||
assertFalse("Should not be a directory", status.isDirectory()); |
|||
assertTrue("Should be a file", status.isFile()); |
|||
assertEquals("File length should match", content.length(), status.getLen()); |
|||
assertNotNull("Path should not be null", status.getPath()); |
|||
} |
|||
|
|||
@Test |
|||
public void testListStatus() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testDir = new Path(TEST_ROOT + "/listtest"); |
|||
fs.mkdirs(testDir); |
|||
|
|||
// Create multiple files |
|||
for (int i = 0; i < 3; i++) { |
|||
Path file = new Path(testDir, "file" + i + ".txt"); |
|||
FSDataOutputStream out = fs.create(file); |
|||
out.write(("content" + i).getBytes()); |
|||
out.close(); |
|||
} |
|||
|
|||
FileStatus[] statuses = fs.listStatus(testDir); |
|||
assertNotNull("List should not be null", statuses); |
|||
assertEquals("Should have 3 files", 3, statuses.length); |
|||
} |
|||
|
|||
@Test |
|||
public void testRename() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path srcFile = new Path(TEST_ROOT + "/source.txt"); |
|||
Path dstFile = new Path(TEST_ROOT + "/destination.txt"); |
|||
String content = "rename test"; |
|||
|
|||
// Create source file |
|||
FSDataOutputStream out = fs.create(srcFile); |
|||
out.write(content.getBytes()); |
|||
out.close(); |
|||
|
|||
assertTrue("Source file should exist", fs.exists(srcFile)); |
|||
|
|||
// Rename |
|||
assertTrue("Rename should succeed", fs.rename(srcFile, dstFile)); |
|||
|
|||
// Verify |
|||
assertFalse("Source file should not exist", fs.exists(srcFile)); |
|||
assertTrue("Destination file should exist", fs.exists(dstFile)); |
|||
|
|||
// Verify content preserved |
|||
FSDataInputStream in = fs.open(dstFile); |
|||
byte[] buffer = new byte[content.length()]; |
|||
in.read(buffer); |
|||
in.close(); |
|||
assertEquals("Content should be preserved", content, new String(buffer)); |
|||
} |
|||
|
|||
@Test |
|||
public void testDelete() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/deletetest.txt"); |
|||
|
|||
// Create file |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write("delete me".getBytes()); |
|||
out.close(); |
|||
|
|||
assertTrue("File should exist before delete", fs.exists(testFile)); |
|||
|
|||
// Delete |
|||
assertTrue("Delete should succeed", fs.delete(testFile, false)); |
|||
assertFalse("File should not exist after delete", fs.exists(testFile)); |
|||
} |
|||
|
|||
@Test |
|||
public void testDeleteDirectory() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testDir = new Path(TEST_ROOT + "/deletedir"); |
|||
Path testFile = new Path(testDir, "file.txt"); |
|||
|
|||
// Create directory with file |
|||
fs.mkdirs(testDir); |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write("content".getBytes()); |
|||
out.close(); |
|||
|
|||
assertTrue("Directory should exist", fs.exists(testDir)); |
|||
assertTrue("File should exist", fs.exists(testFile)); |
|||
|
|||
// Recursive delete |
|||
assertTrue("Recursive delete should succeed", fs.delete(testDir, true)); |
|||
assertFalse("Directory should not exist after delete", fs.exists(testDir)); |
|||
assertFalse("File should not exist after delete", fs.exists(testFile)); |
|||
} |
|||
|
|||
@Test |
|||
public void testAppend() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/appendtest.txt"); |
|||
String initialContent = "initial"; |
|||
String appendContent = " appended"; |
|||
|
|||
// Create initial file |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write(initialContent.getBytes()); |
|||
out.close(); |
|||
|
|||
// Append |
|||
FSDataOutputStream appendOut = fs.append(testFile, 4096, null); |
|||
assertNotNull("Append stream should not be null", appendOut); |
|||
appendOut.write(appendContent.getBytes()); |
|||
appendOut.close(); |
|||
|
|||
// Verify combined content |
|||
FSDataInputStream in = fs.open(testFile); |
|||
byte[] buffer = new byte[initialContent.length() + appendContent.length()]; |
|||
int bytesRead = in.read(buffer); |
|||
in.close(); |
|||
|
|||
String expected = initialContent + appendContent; |
|||
assertEquals("Should read all bytes", expected.length(), bytesRead); |
|||
assertEquals("Content should match", expected, new String(buffer)); |
|||
} |
|||
|
|||
@Test |
|||
public void testSetWorkingDirectory() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path originalWd = fs.getWorkingDirectory(); |
|||
assertEquals("Original working directory should be /", "/", originalWd.toUri().getPath()); |
|||
|
|||
Path newWd = new Path(TEST_ROOT); |
|||
fs.mkdirs(newWd); |
|||
fs.setWorkingDirectory(newWd); |
|||
|
|||
Path currentWd = fs.getWorkingDirectory(); |
|||
assertTrue("Working directory should be updated", |
|||
currentWd.toUri().getPath().contains(TEST_ROOT)); |
|||
} |
|||
|
|||
@Test |
|||
public void testSetPermission() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/permtest.txt"); |
|||
|
|||
// Create file |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write("permission test".getBytes()); |
|||
out.close(); |
|||
|
|||
// Set permission |
|||
FsPermission newPerm = new FsPermission((short) 0644); |
|||
fs.setPermission(testFile, newPerm); |
|||
|
|||
FileStatus status = fs.getFileStatus(testFile); |
|||
assertNotNull("Permission should not be null", status.getPermission()); |
|||
} |
|||
|
|||
@Test |
|||
public void testSetOwner() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/ownertest.txt"); |
|||
|
|||
// Create file |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write("owner test".getBytes()); |
|||
out.close(); |
|||
|
|||
// Set owner - this may not fail even if not fully implemented |
|||
fs.setOwner(testFile, "testuser", "testgroup"); |
|||
|
|||
// Just verify the call doesn't throw an exception |
|||
FileStatus status = fs.getFileStatus(testFile); |
|||
assertNotNull("FileStatus should not be null", status); |
|||
} |
|||
|
|||
@Test |
|||
public void testRenameToExistingDirectory() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path srcFile = new Path(TEST_ROOT + "/movefile.txt"); |
|||
Path dstDir = new Path(TEST_ROOT + "/movedir"); |
|||
|
|||
// Create source file and destination directory |
|||
FSDataOutputStream out = fs.create(srcFile); |
|||
out.write("move test".getBytes()); |
|||
out.close(); |
|||
fs.mkdirs(dstDir); |
|||
|
|||
// Rename file to existing directory (should move file into directory) |
|||
assertTrue("Rename to directory should succeed", fs.rename(srcFile, dstDir)); |
|||
|
|||
// File should be moved into the directory |
|||
Path expectedLocation = new Path(dstDir, srcFile.getName()); |
|||
assertTrue("File should exist in destination directory", fs.exists(expectedLocation)); |
|||
assertFalse("Source file should not exist", fs.exists(srcFile)); |
|||
} |
|||
} |
|||
|
|||
@ -0,0 +1,190 @@ |
|||
# SeaweedFS Hadoop3 Client |
|||
|
|||
Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 3.x. |
|||
|
|||
## Building |
|||
|
|||
```bash |
|||
mvn clean install |
|||
``` |
|||
|
|||
## Testing |
|||
|
|||
This project includes two types of tests: |
|||
|
|||
### 1. Configuration Tests (No SeaweedFS Required) |
|||
|
|||
These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance: |
|||
|
|||
```bash |
|||
mvn test -Dtest=SeaweedFileSystemConfigTest |
|||
``` |
|||
|
|||
### 2. Integration Tests (Requires SeaweedFS) |
|||
|
|||
These tests verify actual FileSystem operations against a running SeaweedFS instance. |
|||
|
|||
#### Prerequisites |
|||
|
|||
1. Start SeaweedFS with default ports: |
|||
```bash |
|||
# Terminal 1: Start master |
|||
weed master |
|||
|
|||
# Terminal 2: Start volume server |
|||
weed volume -mserver=localhost:9333 |
|||
|
|||
# Terminal 3: Start filer |
|||
weed filer -master=localhost:9333 |
|||
``` |
|||
|
|||
2. Verify services are running: |
|||
- Master: http://localhost:9333 |
|||
- Filer HTTP: http://localhost:8888 |
|||
- Filer gRPC: localhost:18888 |
|||
|
|||
#### Running Integration Tests |
|||
|
|||
```bash |
|||
# Enable integration tests |
|||
export SEAWEEDFS_TEST_ENABLED=true |
|||
|
|||
# Run all tests |
|||
mvn test |
|||
|
|||
# Run specific test |
|||
mvn test -Dtest=SeaweedFileSystemTest |
|||
``` |
|||
|
|||
### Test Configuration |
|||
|
|||
Integration tests can be configured via environment variables or system properties: |
|||
|
|||
- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false) |
|||
- Tests use these default connection settings: |
|||
- Filer Host: localhost |
|||
- Filer HTTP Port: 8888 |
|||
- Filer gRPC Port: 18888 |
|||
|
|||
### Running Tests with Custom Configuration |
|||
|
|||
To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration: |
|||
|
|||
```java |
|||
conf.set("fs.seaweed.filer.host", "your-host"); |
|||
conf.setInt("fs.seaweed.filer.port", 8888); |
|||
conf.setInt("fs.seaweed.filer.port.grpc", 18888); |
|||
``` |
|||
|
|||
## Test Coverage |
|||
|
|||
The test suite covers: |
|||
|
|||
- **Configuration & Initialization** |
|||
- URI parsing and configuration |
|||
- Default values |
|||
- Configuration overrides |
|||
- Working directory management |
|||
|
|||
- **File Operations** |
|||
- Create files |
|||
- Read files |
|||
- Write files |
|||
- Append to files |
|||
- Delete files |
|||
|
|||
- **Directory Operations** |
|||
- Create directories |
|||
- List directory contents |
|||
- Delete directories (recursive and non-recursive) |
|||
|
|||
- **Metadata Operations** |
|||
- Get file status |
|||
- Set permissions |
|||
- Set owner/group |
|||
- Rename files and directories |
|||
|
|||
## Usage in Hadoop |
|||
|
|||
1. Copy the built JAR to your Hadoop classpath: |
|||
```bash |
|||
cp target/seaweedfs-hadoop3-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/ |
|||
``` |
|||
|
|||
2. Configure `core-site.xml`: |
|||
```xml |
|||
<configuration> |
|||
<property> |
|||
<name>fs.seaweedfs.impl</name> |
|||
<value>seaweed.hdfs.SeaweedFileSystem</value> |
|||
</property> |
|||
<property> |
|||
<name>fs.seaweed.filer.host</name> |
|||
<value>localhost</value> |
|||
</property> |
|||
<property> |
|||
<name>fs.seaweed.filer.port</name> |
|||
<value>8888</value> |
|||
</property> |
|||
<property> |
|||
<name>fs.seaweed.filer.port.grpc</name> |
|||
<value>18888</value> |
|||
</property> |
|||
</configuration> |
|||
``` |
|||
|
|||
3. Use SeaweedFS with Hadoop commands: |
|||
```bash |
|||
hadoop fs -ls seaweedfs://localhost:8888/ |
|||
hadoop fs -mkdir seaweedfs://localhost:8888/test |
|||
hadoop fs -put local.txt seaweedfs://localhost:8888/test/ |
|||
``` |
|||
|
|||
## Continuous Integration |
|||
|
|||
For CI environments, tests can be run in two modes: |
|||
|
|||
1. **Configuration Tests Only** (default, no SeaweedFS required): |
|||
```bash |
|||
mvn test -Dtest=SeaweedFileSystemConfigTest |
|||
``` |
|||
|
|||
2. **Full Integration Tests** (requires SeaweedFS): |
|||
```bash |
|||
# Start SeaweedFS in CI environment |
|||
# Then run: |
|||
export SEAWEEDFS_TEST_ENABLED=true |
|||
mvn test |
|||
``` |
|||
|
|||
## Troubleshooting |
|||
|
|||
### Tests are skipped |
|||
|
|||
If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set": |
|||
```bash |
|||
export SEAWEEDFS_TEST_ENABLED=true |
|||
``` |
|||
|
|||
### Connection refused errors |
|||
|
|||
Ensure SeaweedFS is running and accessible: |
|||
```bash |
|||
curl http://localhost:8888/ |
|||
``` |
|||
|
|||
### gRPC errors |
|||
|
|||
Verify the gRPC port is accessible: |
|||
```bash |
|||
# Should show the port is listening |
|||
netstat -an | grep 18888 |
|||
``` |
|||
|
|||
## Contributing |
|||
|
|||
When adding new features, please include: |
|||
1. Configuration tests (no SeaweedFS required) |
|||
2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard) |
|||
3. Documentation updates |
|||
|
|||
@ -0,0 +1,90 @@ |
|||
package seaweed.hdfs; |
|||
|
|||
import org.apache.hadoop.conf.Configuration; |
|||
import org.apache.hadoop.fs.Path; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
|
|||
import static org.junit.Assert.*; |
|||
|
|||
/** |
|||
* Unit tests for SeaweedFileSystem configuration that don't require a running SeaweedFS instance. |
|||
* |
|||
* These tests verify basic properties and constants. |
|||
*/ |
|||
public class SeaweedFileSystemConfigTest { |
|||
|
|||
private SeaweedFileSystem fs; |
|||
private Configuration conf; |
|||
|
|||
@Before |
|||
public void setUp() { |
|||
fs = new SeaweedFileSystem(); |
|||
conf = new Configuration(); |
|||
} |
|||
|
|||
@Test |
|||
public void testScheme() { |
|||
assertEquals("seaweedfs", fs.getScheme()); |
|||
} |
|||
|
|||
@Test |
|||
public void testConstants() { |
|||
// Test that constants are defined correctly |
|||
assertEquals("fs.seaweed.filer.host", SeaweedFileSystem.FS_SEAWEED_FILER_HOST); |
|||
assertEquals("fs.seaweed.filer.port", SeaweedFileSystem.FS_SEAWEED_FILER_PORT); |
|||
assertEquals("fs.seaweed.filer.port.grpc", SeaweedFileSystem.FS_SEAWEED_FILER_PORT_GRPC); |
|||
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); |
|||
assertEquals("fs.seaweed.buffer.size", SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE); |
|||
assertEquals(4 * 1024 * 1024, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); |
|||
assertEquals("fs.seaweed.replication", SeaweedFileSystem.FS_SEAWEED_REPLICATION); |
|||
assertEquals("fs.seaweed.volume.server.access", SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS); |
|||
assertEquals("fs.seaweed.filer.cn", SeaweedFileSystem.FS_SEAWEED_FILER_CN); |
|||
} |
|||
|
|||
@Test |
|||
public void testWorkingDirectoryPathOperations() { |
|||
// Test path operations that don't require initialization |
|||
Path testPath = new Path("/test/path"); |
|||
assertTrue("Path should be absolute", testPath.isAbsolute()); |
|||
assertEquals("/test/path", testPath.toUri().getPath()); |
|||
|
|||
Path childPath = new Path(testPath, "child"); |
|||
assertEquals("/test/path/child", childPath.toUri().getPath()); |
|||
} |
|||
|
|||
@Test |
|||
public void testConfigurationProperties() { |
|||
// Test that configuration can be set and read |
|||
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_HOST, "testhost"); |
|||
assertEquals("testhost", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_HOST)); |
|||
|
|||
conf.setInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 9999); |
|||
assertEquals(9999, conf.getInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 0)); |
|||
|
|||
conf.setInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 8 * 1024 * 1024); |
|||
assertEquals(8 * 1024 * 1024, conf.getInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 0)); |
|||
|
|||
conf.set(SeaweedFileSystem.FS_SEAWEED_REPLICATION, "001"); |
|||
assertEquals("001", conf.get(SeaweedFileSystem.FS_SEAWEED_REPLICATION)); |
|||
|
|||
conf.set(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS, "publicUrl"); |
|||
assertEquals("publicUrl", conf.get(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS)); |
|||
|
|||
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_CN, "test-cn"); |
|||
assertEquals("test-cn", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_CN)); |
|||
} |
|||
|
|||
@Test |
|||
public void testDefaultBufferSize() { |
|||
// Test default buffer size constant |
|||
int expected = 4 * 1024 * 1024; // 4MB |
|||
assertEquals(expected, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); |
|||
} |
|||
|
|||
@Test |
|||
public void testDefaultPort() { |
|||
// Test default port constant |
|||
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); |
|||
} |
|||
} |
|||
@ -0,0 +1,379 @@ |
|||
package seaweed.hdfs; |
|||
|
|||
import org.apache.hadoop.conf.Configuration; |
|||
import org.apache.hadoop.fs.FSDataInputStream; |
|||
import org.apache.hadoop.fs.FSDataOutputStream; |
|||
import org.apache.hadoop.fs.FileStatus; |
|||
import org.apache.hadoop.fs.Path; |
|||
import org.apache.hadoop.fs.permission.FsPermission; |
|||
import org.junit.After; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
|
|||
import java.io.IOException; |
|||
import java.net.URI; |
|||
|
|||
import static org.junit.Assert.*; |
|||
|
|||
/** |
|||
* Unit tests for SeaweedFileSystem. |
|||
* |
|||
* These tests verify basic FileSystem operations against a SeaweedFS backend. |
|||
* Note: These tests require a running SeaweedFS filer instance. |
|||
* |
|||
* To run tests, ensure SeaweedFS is running with default ports: |
|||
* - Filer HTTP: 8888 |
|||
* - Filer gRPC: 18888 |
|||
* |
|||
* Set environment variable SEAWEEDFS_TEST_ENABLED=true to enable these tests. |
|||
*/ |
|||
public class SeaweedFileSystemTest { |
|||
|
|||
private SeaweedFileSystem fs; |
|||
private Configuration conf; |
|||
private static final String TEST_ROOT = "/test-hdfs3"; |
|||
private static final boolean TESTS_ENABLED = |
|||
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); |
|||
|
|||
@Before |
|||
public void setUp() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
return; |
|||
} |
|||
|
|||
conf = new Configuration(); |
|||
conf.set("fs.seaweed.filer.host", "localhost"); |
|||
conf.setInt("fs.seaweed.filer.port", 8888); |
|||
conf.setInt("fs.seaweed.filer.port.grpc", 18888); |
|||
|
|||
fs = new SeaweedFileSystem(); |
|||
URI uri = new URI("seaweedfs://localhost:8888/"); |
|||
fs.initialize(uri, conf); |
|||
|
|||
// Clean up any existing test directory |
|||
Path testPath = new Path(TEST_ROOT); |
|||
if (fs.exists(testPath)) { |
|||
fs.delete(testPath, true); |
|||
} |
|||
} |
|||
|
|||
@After |
|||
public void tearDown() throws Exception { |
|||
if (!TESTS_ENABLED || fs == null) { |
|||
return; |
|||
} |
|||
|
|||
// Clean up test directory |
|||
Path testPath = new Path(TEST_ROOT); |
|||
if (fs.exists(testPath)) { |
|||
fs.delete(testPath, true); |
|||
} |
|||
|
|||
fs.close(); |
|||
} |
|||
|
|||
@Test |
|||
public void testInitialization() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
assertNotNull(fs); |
|||
assertEquals("seaweedfs", fs.getScheme()); |
|||
assertNotNull(fs.getUri()); |
|||
assertEquals("/", fs.getWorkingDirectory().toUri().getPath()); |
|||
} |
|||
|
|||
@Test |
|||
public void testMkdirs() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testDir = new Path(TEST_ROOT + "/testdir"); |
|||
assertTrue("Failed to create directory", fs.mkdirs(testDir)); |
|||
assertTrue("Directory should exist", fs.exists(testDir)); |
|||
|
|||
FileStatus status = fs.getFileStatus(testDir); |
|||
assertTrue("Path should be a directory", status.isDirectory()); |
|||
} |
|||
|
|||
@Test |
|||
public void testCreateAndReadFile() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/testfile.txt"); |
|||
String testContent = "Hello, SeaweedFS!"; |
|||
|
|||
// Create and write to file |
|||
FSDataOutputStream out = fs.create(testFile, FsPermission.getDefault(), |
|||
false, 4096, (short) 1, 4 * 1024 * 1024, null); |
|||
assertNotNull("Output stream should not be null", out); |
|||
out.write(testContent.getBytes()); |
|||
out.close(); |
|||
|
|||
// Verify file exists |
|||
assertTrue("File should exist", fs.exists(testFile)); |
|||
|
|||
// Read and verify content |
|||
FSDataInputStream in = fs.open(testFile, 4096); |
|||
assertNotNull("Input stream should not be null", in); |
|||
byte[] buffer = new byte[testContent.length()]; |
|||
int bytesRead = in.read(buffer); |
|||
in.close(); |
|||
|
|||
assertEquals("Should read all bytes", testContent.length(), bytesRead); |
|||
assertEquals("Content should match", testContent, new String(buffer)); |
|||
} |
|||
|
|||
@Test |
|||
public void testFileStatus() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/statustest.txt"); |
|||
String content = "test content"; |
|||
|
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write(content.getBytes()); |
|||
out.close(); |
|||
|
|||
FileStatus status = fs.getFileStatus(testFile); |
|||
assertNotNull("FileStatus should not be null", status); |
|||
assertFalse("Should not be a directory", status.isDirectory()); |
|||
assertTrue("Should be a file", status.isFile()); |
|||
assertEquals("File length should match", content.length(), status.getLen()); |
|||
assertNotNull("Path should not be null", status.getPath()); |
|||
} |
|||
|
|||
@Test |
|||
public void testListStatus() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testDir = new Path(TEST_ROOT + "/listtest"); |
|||
fs.mkdirs(testDir); |
|||
|
|||
// Create multiple files |
|||
for (int i = 0; i < 3; i++) { |
|||
Path file = new Path(testDir, "file" + i + ".txt"); |
|||
FSDataOutputStream out = fs.create(file); |
|||
out.write(("content" + i).getBytes()); |
|||
out.close(); |
|||
} |
|||
|
|||
FileStatus[] statuses = fs.listStatus(testDir); |
|||
assertNotNull("List should not be null", statuses); |
|||
assertEquals("Should have 3 files", 3, statuses.length); |
|||
} |
|||
|
|||
@Test |
|||
public void testRename() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path srcFile = new Path(TEST_ROOT + "/source.txt"); |
|||
Path dstFile = new Path(TEST_ROOT + "/destination.txt"); |
|||
String content = "rename test"; |
|||
|
|||
// Create source file |
|||
FSDataOutputStream out = fs.create(srcFile); |
|||
out.write(content.getBytes()); |
|||
out.close(); |
|||
|
|||
assertTrue("Source file should exist", fs.exists(srcFile)); |
|||
|
|||
// Rename |
|||
assertTrue("Rename should succeed", fs.rename(srcFile, dstFile)); |
|||
|
|||
// Verify |
|||
assertFalse("Source file should not exist", fs.exists(srcFile)); |
|||
assertTrue("Destination file should exist", fs.exists(dstFile)); |
|||
|
|||
// Verify content preserved |
|||
FSDataInputStream in = fs.open(dstFile); |
|||
byte[] buffer = new byte[content.length()]; |
|||
in.read(buffer); |
|||
in.close(); |
|||
assertEquals("Content should be preserved", content, new String(buffer)); |
|||
} |
|||
|
|||
@Test |
|||
public void testDelete() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/deletetest.txt"); |
|||
|
|||
// Create file |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write("delete me".getBytes()); |
|||
out.close(); |
|||
|
|||
assertTrue("File should exist before delete", fs.exists(testFile)); |
|||
|
|||
// Delete |
|||
assertTrue("Delete should succeed", fs.delete(testFile, false)); |
|||
assertFalse("File should not exist after delete", fs.exists(testFile)); |
|||
} |
|||
|
|||
@Test |
|||
public void testDeleteDirectory() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testDir = new Path(TEST_ROOT + "/deletedir"); |
|||
Path testFile = new Path(testDir, "file.txt"); |
|||
|
|||
// Create directory with file |
|||
fs.mkdirs(testDir); |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write("content".getBytes()); |
|||
out.close(); |
|||
|
|||
assertTrue("Directory should exist", fs.exists(testDir)); |
|||
assertTrue("File should exist", fs.exists(testFile)); |
|||
|
|||
// Recursive delete |
|||
assertTrue("Recursive delete should succeed", fs.delete(testDir, true)); |
|||
assertFalse("Directory should not exist after delete", fs.exists(testDir)); |
|||
assertFalse("File should not exist after delete", fs.exists(testFile)); |
|||
} |
|||
|
|||
@Test |
|||
public void testAppend() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/appendtest.txt"); |
|||
String initialContent = "initial"; |
|||
String appendContent = " appended"; |
|||
|
|||
// Create initial file |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write(initialContent.getBytes()); |
|||
out.close(); |
|||
|
|||
// Append |
|||
FSDataOutputStream appendOut = fs.append(testFile, 4096, null); |
|||
assertNotNull("Append stream should not be null", appendOut); |
|||
appendOut.write(appendContent.getBytes()); |
|||
appendOut.close(); |
|||
|
|||
// Verify combined content |
|||
FSDataInputStream in = fs.open(testFile); |
|||
byte[] buffer = new byte[initialContent.length() + appendContent.length()]; |
|||
int bytesRead = in.read(buffer); |
|||
in.close(); |
|||
|
|||
String expected = initialContent + appendContent; |
|||
assertEquals("Should read all bytes", expected.length(), bytesRead); |
|||
assertEquals("Content should match", expected, new String(buffer)); |
|||
} |
|||
|
|||
@Test |
|||
public void testSetWorkingDirectory() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path originalWd = fs.getWorkingDirectory(); |
|||
assertEquals("Original working directory should be /", "/", originalWd.toUri().getPath()); |
|||
|
|||
Path newWd = new Path(TEST_ROOT); |
|||
fs.mkdirs(newWd); |
|||
fs.setWorkingDirectory(newWd); |
|||
|
|||
Path currentWd = fs.getWorkingDirectory(); |
|||
assertTrue("Working directory should be updated", |
|||
currentWd.toUri().getPath().contains(TEST_ROOT)); |
|||
} |
|||
|
|||
@Test |
|||
public void testSetPermission() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/permtest.txt"); |
|||
|
|||
// Create file |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write("permission test".getBytes()); |
|||
out.close(); |
|||
|
|||
// Set permission |
|||
FsPermission newPerm = new FsPermission((short) 0644); |
|||
fs.setPermission(testFile, newPerm); |
|||
|
|||
FileStatus status = fs.getFileStatus(testFile); |
|||
assertNotNull("Permission should not be null", status.getPermission()); |
|||
} |
|||
|
|||
@Test |
|||
public void testSetOwner() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path testFile = new Path(TEST_ROOT + "/ownertest.txt"); |
|||
|
|||
// Create file |
|||
FSDataOutputStream out = fs.create(testFile); |
|||
out.write("owner test".getBytes()); |
|||
out.close(); |
|||
|
|||
// Set owner - this may not fail even if not fully implemented |
|||
fs.setOwner(testFile, "testuser", "testgroup"); |
|||
|
|||
// Just verify the call doesn't throw an exception |
|||
FileStatus status = fs.getFileStatus(testFile); |
|||
assertNotNull("FileStatus should not be null", status); |
|||
} |
|||
|
|||
@Test |
|||
public void testRenameToExistingDirectory() throws Exception { |
|||
if (!TESTS_ENABLED) { |
|||
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
|||
return; |
|||
} |
|||
|
|||
Path srcFile = new Path(TEST_ROOT + "/movefile.txt"); |
|||
Path dstDir = new Path(TEST_ROOT + "/movedir"); |
|||
|
|||
// Create source file and destination directory |
|||
FSDataOutputStream out = fs.create(srcFile); |
|||
out.write("move test".getBytes()); |
|||
out.close(); |
|||
fs.mkdirs(dstDir); |
|||
|
|||
// Rename file to existing directory (should move file into directory) |
|||
assertTrue("Rename to directory should succeed", fs.rename(srcFile, dstDir)); |
|||
|
|||
// File should be moved into the directory |
|||
Path expectedLocation = new Path(dstDir, srcFile.getName()); |
|||
assertTrue("File should exist in destination directory", fs.exists(expectedLocation)); |
|||
assertFalse("Source file should not exist", fs.exists(srcFile)); |
|||
} |
|||
} |
|||
|
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue