Browse Source
chore(deps): bump org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0 in /other/java/hdfs3 (#7512)
chore(deps): bump org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0 in /other/java/hdfs3 (#7512)
* chore(deps): bump org.apache.hadoop:hadoop-common in /other/java/hdfs3 Bumps org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0. --- updated-dependencies: - dependency-name: org.apache.hadoop:hadoop-common dependency-version: 3.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> * add java client unit tests * Update dependency-reduced-pom.xml * add java integration tests * fix * fix buffer --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: chrislu <chris.lu@gmail.com>pull/7514/head
dependabot[bot]
6 days ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 2585 additions and 10 deletions
-
170.github/workflows/java_integration_tests.yml
-
64.github/workflows/java_unit_tests.yml
-
323other/java/client/src/test/java/seaweedfs/client/FilerClientIntegrationTest.java
-
417other/java/client/src/test/java/seaweedfs/client/SeaweedStreamIntegrationTest.java
-
190other/java/hdfs2/README.md
-
19other/java/hdfs2/pom.xml
-
90other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java
-
379other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java
-
190other/java/hdfs3/README.md
-
263other/java/hdfs3/dependency-reduced-pom.xml
-
21other/java/hdfs3/pom.xml
-
90other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java
-
379other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java
@ -0,0 +1,170 @@ |
|||||
|
name: Java Client Integration Tests |
||||
|
|
||||
|
on: |
||||
|
push: |
||||
|
branches: [ master ] |
||||
|
paths: |
||||
|
- 'other/java/**' |
||||
|
- 'weed/**' |
||||
|
- '.github/workflows/java_integration_tests.yml' |
||||
|
pull_request: |
||||
|
branches: [ master ] |
||||
|
paths: |
||||
|
- 'other/java/**' |
||||
|
- 'weed/**' |
||||
|
- '.github/workflows/java_integration_tests.yml' |
||||
|
|
||||
|
jobs: |
||||
|
test: |
||||
|
name: Java Integration Tests |
||||
|
runs-on: ubuntu-latest |
||||
|
|
||||
|
strategy: |
||||
|
matrix: |
||||
|
java: ['11', '17'] |
||||
|
|
||||
|
steps: |
||||
|
- name: Checkout code |
||||
|
uses: actions/checkout@v4 |
||||
|
|
||||
|
- name: Set up Go |
||||
|
uses: actions/setup-go@v6 |
||||
|
with: |
||||
|
go-version-file: 'go.mod' |
||||
|
id: go |
||||
|
|
||||
|
- name: Set up Java |
||||
|
uses: actions/setup-java@v4 |
||||
|
with: |
||||
|
java-version: ${{ matrix.java }} |
||||
|
distribution: 'temurin' |
||||
|
cache: 'maven' |
||||
|
|
||||
|
- name: Build SeaweedFS |
||||
|
run: | |
||||
|
cd weed |
||||
|
go install -buildvcs=false |
||||
|
weed version |
||||
|
|
||||
|
- name: Start SeaweedFS Server |
||||
|
run: | |
||||
|
# Create clean data directory |
||||
|
export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)" |
||||
|
mkdir -p "$WEED_DATA_DIR" |
||||
|
|
||||
|
# Start SeaweedFS with optimized settings for CI |
||||
|
weed server -dir="$WEED_DATA_DIR" \ |
||||
|
-master.raftHashicorp \ |
||||
|
-master.electionTimeout=1s \ |
||||
|
-master.volumeSizeLimitMB=100 \ |
||||
|
-volume.max=100 \ |
||||
|
-volume.preStopSeconds=1 \ |
||||
|
-master.peers=none \ |
||||
|
-filer -filer.maxMB=64 \ |
||||
|
-master.port=9333 \ |
||||
|
-volume.port=8080 \ |
||||
|
-filer.port=8888 \ |
||||
|
-metricsPort=9324 > seaweedfs.log 2>&1 & |
||||
|
|
||||
|
SERVER_PID=$! |
||||
|
echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV |
||||
|
echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV |
||||
|
echo "SeaweedFS server started with PID: $SERVER_PID" |
||||
|
|
||||
|
- name: Wait for SeaweedFS Components |
||||
|
run: | |
||||
|
echo "Waiting for SeaweedFS components to start..." |
||||
|
|
||||
|
# Wait for master |
||||
|
for i in {1..30}; do |
||||
|
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then |
||||
|
echo "✓ Master server is ready" |
||||
|
break |
||||
|
fi |
||||
|
echo "Waiting for master server... ($i/30)" |
||||
|
sleep 2 |
||||
|
done |
||||
|
|
||||
|
# Wait for volume |
||||
|
for i in {1..30}; do |
||||
|
if curl -s http://localhost:8080/status > /dev/null 2>&1; then |
||||
|
echo "✓ Volume server is ready" |
||||
|
break |
||||
|
fi |
||||
|
echo "Waiting for volume server... ($i/30)" |
||||
|
sleep 2 |
||||
|
done |
||||
|
|
||||
|
# Wait for filer |
||||
|
for i in {1..30}; do |
||||
|
if curl -s http://localhost:8888/ > /dev/null 2>&1; then |
||||
|
echo "✓ Filer is ready" |
||||
|
break |
||||
|
fi |
||||
|
echo "Waiting for filer... ($i/30)" |
||||
|
sleep 2 |
||||
|
done |
||||
|
|
||||
|
echo "✓ All SeaweedFS components are ready!" |
||||
|
|
||||
|
# Display cluster status |
||||
|
echo "Cluster status:" |
||||
|
curl -s http://localhost:9333/cluster/status | head -20 |
||||
|
|
||||
|
- name: Build and Install SeaweedFS Client |
||||
|
working-directory: other/java/client |
||||
|
run: | |
||||
|
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true |
||||
|
|
||||
|
- name: Run Client Unit Tests |
||||
|
working-directory: other/java/client |
||||
|
run: | |
||||
|
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest |
||||
|
|
||||
|
- name: Run Client Integration Tests |
||||
|
working-directory: other/java/client |
||||
|
env: |
||||
|
SEAWEEDFS_TEST_ENABLED: true |
||||
|
run: | |
||||
|
mvn test -Dtest=*IntegrationTest |
||||
|
|
||||
|
- name: Run HDFS2 Configuration Tests |
||||
|
working-directory: other/java/hdfs2 |
||||
|
run: | |
||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true |
||||
|
|
||||
|
- name: Run HDFS3 Configuration Tests |
||||
|
working-directory: other/java/hdfs3 |
||||
|
run: | |
||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true |
||||
|
|
||||
|
- name: Display logs on failure |
||||
|
if: failure() |
||||
|
run: | |
||||
|
echo "=== SeaweedFS Server Log ===" |
||||
|
tail -100 seaweedfs.log || echo "No server log" |
||||
|
echo "" |
||||
|
echo "=== Cluster Status ===" |
||||
|
curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster" |
||||
|
echo "" |
||||
|
echo "=== Process Status ===" |
||||
|
ps aux | grep weed || echo "No weed processes" |
||||
|
|
||||
|
- name: Cleanup |
||||
|
if: always() |
||||
|
run: | |
||||
|
# Stop server using stored PID |
||||
|
if [ -n "$SERVER_PID" ]; then |
||||
|
echo "Stopping SeaweedFS server (PID: $SERVER_PID)" |
||||
|
kill -9 $SERVER_PID 2>/dev/null || true |
||||
|
fi |
||||
|
|
||||
|
# Fallback: kill any remaining weed processes |
||||
|
pkill -f "weed server" || true |
||||
|
|
||||
|
# Clean up data directory |
||||
|
if [ -n "$WEED_DATA_DIR" ]; then |
||||
|
echo "Cleaning up data directory: $WEED_DATA_DIR" |
||||
|
rm -rf "$WEED_DATA_DIR" || true |
||||
|
fi |
||||
|
|
||||
@ -0,0 +1,64 @@ |
|||||
|
name: Java Client Unit Tests |
||||
|
|
||||
|
on: |
||||
|
push: |
||||
|
branches: [ master ] |
||||
|
paths: |
||||
|
- 'other/java/**' |
||||
|
- '.github/workflows/java_unit_tests.yml' |
||||
|
pull_request: |
||||
|
branches: [ master ] |
||||
|
paths: |
||||
|
- 'other/java/**' |
||||
|
- '.github/workflows/java_unit_tests.yml' |
||||
|
|
||||
|
jobs: |
||||
|
test: |
||||
|
name: Java Unit Tests |
||||
|
runs-on: ubuntu-latest |
||||
|
|
||||
|
strategy: |
||||
|
matrix: |
||||
|
java: ['8', '11', '17', '21'] |
||||
|
|
||||
|
steps: |
||||
|
- name: Checkout code |
||||
|
uses: actions/checkout@v4 |
||||
|
|
||||
|
- name: Set up Java |
||||
|
uses: actions/setup-java@v4 |
||||
|
with: |
||||
|
java-version: ${{ matrix.java }} |
||||
|
distribution: 'temurin' |
||||
|
cache: 'maven' |
||||
|
|
||||
|
- name: Build and Install SeaweedFS Client |
||||
|
working-directory: other/java/client |
||||
|
run: | |
||||
|
mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true |
||||
|
|
||||
|
- name: Run Client Unit Tests |
||||
|
working-directory: other/java/client |
||||
|
run: | |
||||
|
mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest |
||||
|
|
||||
|
- name: Run HDFS2 Configuration Tests |
||||
|
working-directory: other/java/hdfs2 |
||||
|
run: | |
||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true |
||||
|
|
||||
|
- name: Run HDFS3 Configuration Tests |
||||
|
working-directory: other/java/hdfs3 |
||||
|
run: | |
||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true |
||||
|
|
||||
|
- name: Upload Test Reports |
||||
|
if: always() |
||||
|
uses: actions/upload-artifact@v5 |
||||
|
with: |
||||
|
name: test-reports-java-${{ matrix.java }} |
||||
|
path: | |
||||
|
other/java/client/target/surefire-reports/ |
||||
|
other/java/hdfs2/target/surefire-reports/ |
||||
|
other/java/hdfs3/target/surefire-reports/ |
||||
|
|
||||
@ -0,0 +1,323 @@ |
|||||
|
package seaweedfs.client; |
||||
|
|
||||
|
import org.junit.After; |
||||
|
import org.junit.Before; |
||||
|
import org.junit.Test; |
||||
|
|
||||
|
import java.io.IOException; |
||||
|
import java.nio.charset.StandardCharsets; |
||||
|
import java.util.List; |
||||
|
|
||||
|
import static org.junit.Assert.*; |
||||
|
|
||||
|
/** |
||||
|
* Integration tests for FilerClient. |
||||
|
* |
||||
|
* These tests verify FilerClient operations against a running SeaweedFS filer |
||||
|
* instance. |
||||
|
* |
||||
|
* Prerequisites: |
||||
|
* - SeaweedFS master, volume server, and filer must be running |
||||
|
* - Default ports: filer HTTP 8888, filer gRPC 18888 |
||||
|
* |
||||
|
* To run tests: |
||||
|
* export SEAWEEDFS_TEST_ENABLED=true |
||||
|
* mvn test -Dtest=FilerClientIntegrationTest |
||||
|
*/ |
||||
|
public class FilerClientIntegrationTest { |
||||
|
|
||||
|
private FilerClient filerClient; |
||||
|
private static final String TEST_ROOT = "/test-client-integration"; |
||||
|
private static final boolean TESTS_ENABLED = "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); |
||||
|
|
||||
|
@Before |
||||
|
public void setUp() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
filerClient = new FilerClient("localhost", 18888); |
||||
|
|
||||
|
// Clean up any existing test directory |
||||
|
if (filerClient.exists(TEST_ROOT)) { |
||||
|
filerClient.rm(TEST_ROOT, true, true); |
||||
|
} |
||||
|
|
||||
|
// Create test root directory |
||||
|
filerClient.mkdirs(TEST_ROOT, 0755); |
||||
|
} |
||||
|
|
||||
|
@After |
||||
|
public void tearDown() throws Exception { |
||||
|
if (!TESTS_ENABLED || filerClient == null) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
try { |
||||
|
// Clean up test directory |
||||
|
if (filerClient.exists(TEST_ROOT)) { |
||||
|
filerClient.rm(TEST_ROOT, true, true); |
||||
|
} |
||||
|
} finally { |
||||
|
filerClient.shutdown(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testMkdirs() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testDir = TEST_ROOT + "/testdir"; |
||||
|
boolean success = filerClient.mkdirs(testDir, 0755); |
||||
|
|
||||
|
assertTrue("Directory creation should succeed", success); |
||||
|
assertTrue("Directory should exist", filerClient.exists(testDir)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testTouch() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testFile = TEST_ROOT + "/testfile.txt"; |
||||
|
boolean success = filerClient.touch(testFile, 0644); |
||||
|
|
||||
|
assertTrue("Touch should succeed", success); |
||||
|
assertTrue("File should exist", filerClient.exists(testFile)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testExists() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
assertTrue("Root should exist", filerClient.exists("/")); |
||||
|
assertTrue("Test root should exist", filerClient.exists(TEST_ROOT)); |
||||
|
assertFalse("Non-existent path should not exist", |
||||
|
filerClient.exists(TEST_ROOT + "/nonexistent")); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testListEntries() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// Create some test files and directories |
||||
|
filerClient.touch(TEST_ROOT + "/file1.txt", 0644); |
||||
|
filerClient.touch(TEST_ROOT + "/file2.txt", 0644); |
||||
|
filerClient.mkdirs(TEST_ROOT + "/subdir", 0755); |
||||
|
|
||||
|
List<FilerProto.Entry> entries = filerClient.listEntries(TEST_ROOT); |
||||
|
|
||||
|
assertNotNull("Entries should not be null", entries); |
||||
|
assertEquals("Should have 3 entries", 3, entries.size()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testListEntriesWithPrefix() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// Create test files |
||||
|
filerClient.touch(TEST_ROOT + "/test1.txt", 0644); |
||||
|
filerClient.touch(TEST_ROOT + "/test2.txt", 0644); |
||||
|
filerClient.touch(TEST_ROOT + "/other.txt", 0644); |
||||
|
|
||||
|
List<FilerProto.Entry> entries = filerClient.listEntries(TEST_ROOT, "test", "", 100, false); |
||||
|
|
||||
|
assertNotNull("Entries should not be null", entries); |
||||
|
assertEquals("Should have 2 entries starting with 'test'", 2, entries.size()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDeleteFile() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testFile = TEST_ROOT + "/deleteme.txt"; |
||||
|
filerClient.touch(testFile, 0644); |
||||
|
|
||||
|
assertTrue("File should exist before delete", filerClient.exists(testFile)); |
||||
|
|
||||
|
boolean success = filerClient.rm(testFile, false, true); |
||||
|
|
||||
|
assertTrue("Delete should succeed", success); |
||||
|
assertFalse("File should not exist after delete", filerClient.exists(testFile)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDeleteDirectoryRecursive() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testDir = TEST_ROOT + "/deletedir"; |
||||
|
filerClient.mkdirs(testDir, 0755); |
||||
|
filerClient.touch(testDir + "/file.txt", 0644); |
||||
|
|
||||
|
assertTrue("Directory should exist", filerClient.exists(testDir)); |
||||
|
assertTrue("File should exist", filerClient.exists(testDir + "/file.txt")); |
||||
|
|
||||
|
boolean success = filerClient.rm(testDir, true, true); |
||||
|
|
||||
|
assertTrue("Delete should succeed", success); |
||||
|
assertFalse("Directory should not exist after delete", filerClient.exists(testDir)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testRename() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String srcFile = TEST_ROOT + "/source.txt"; |
||||
|
String dstFile = TEST_ROOT + "/destination.txt"; |
||||
|
|
||||
|
filerClient.touch(srcFile, 0644); |
||||
|
assertTrue("Source file should exist", filerClient.exists(srcFile)); |
||||
|
|
||||
|
boolean success = filerClient.mv(srcFile, dstFile); |
||||
|
|
||||
|
assertTrue("Rename should succeed", success); |
||||
|
assertFalse("Source file should not exist after rename", filerClient.exists(srcFile)); |
||||
|
assertTrue("Destination file should exist after rename", filerClient.exists(dstFile)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testGetEntry() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testFile = TEST_ROOT + "/getentry.txt"; |
||||
|
filerClient.touch(testFile, 0644); |
||||
|
|
||||
|
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "getentry.txt"); |
||||
|
|
||||
|
assertNotNull("Entry should not be null", entry); |
||||
|
assertEquals("Entry name should match", "getentry.txt", entry.getName()); |
||||
|
assertFalse("Entry should not be a directory", entry.getIsDirectory()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testGetEntryForDirectory() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testDir = TEST_ROOT + "/testsubdir"; |
||||
|
filerClient.mkdirs(testDir, 0755); |
||||
|
|
||||
|
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "testsubdir"); |
||||
|
|
||||
|
assertNotNull("Entry should not be null", entry); |
||||
|
assertEquals("Entry name should match", "testsubdir", entry.getName()); |
||||
|
assertTrue("Entry should be a directory", entry.getIsDirectory()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testCreateAndListNestedDirectories() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String nestedPath = TEST_ROOT + "/level1/level2/level3"; |
||||
|
boolean success = filerClient.mkdirs(nestedPath, 0755); |
||||
|
|
||||
|
assertTrue("Nested directory creation should succeed", success); |
||||
|
assertTrue("Nested directory should exist", filerClient.exists(nestedPath)); |
||||
|
|
||||
|
// Verify each level exists |
||||
|
assertTrue("Level 1 should exist", filerClient.exists(TEST_ROOT + "/level1")); |
||||
|
assertTrue("Level 2 should exist", filerClient.exists(TEST_ROOT + "/level1/level2")); |
||||
|
assertTrue("Level 3 should exist", filerClient.exists(nestedPath)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testMultipleFilesInDirectory() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testDir = TEST_ROOT + "/multifiles"; |
||||
|
filerClient.mkdirs(testDir, 0755); |
||||
|
|
||||
|
// Create 10 files |
||||
|
for (int i = 0; i < 10; i++) { |
||||
|
filerClient.touch(testDir + "/file" + i + ".txt", 0644); |
||||
|
} |
||||
|
|
||||
|
List<FilerProto.Entry> entries = filerClient.listEntries(testDir); |
||||
|
|
||||
|
assertNotNull("Entries should not be null", entries); |
||||
|
assertEquals("Should have 10 files", 10, entries.size()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testRenameDirectory() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String srcDir = TEST_ROOT + "/sourcedir"; |
||||
|
String dstDir = TEST_ROOT + "/destdir"; |
||||
|
|
||||
|
filerClient.mkdirs(srcDir, 0755); |
||||
|
filerClient.touch(srcDir + "/file.txt", 0644); |
||||
|
|
||||
|
boolean success = filerClient.mv(srcDir, dstDir); |
||||
|
|
||||
|
assertTrue("Directory rename should succeed", success); |
||||
|
assertFalse("Source directory should not exist", filerClient.exists(srcDir)); |
||||
|
assertTrue("Destination directory should exist", filerClient.exists(dstDir)); |
||||
|
assertTrue("File should exist in destination", filerClient.exists(dstDir + "/file.txt")); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testLookupNonExistentEntry() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "nonexistent.txt"); |
||||
|
|
||||
|
assertNull("Entry for non-existent file should be null", entry); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testEmptyDirectory() { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String emptyDir = TEST_ROOT + "/emptydir"; |
||||
|
filerClient.mkdirs(emptyDir, 0755); |
||||
|
|
||||
|
List<FilerProto.Entry> entries = filerClient.listEntries(emptyDir); |
||||
|
|
||||
|
assertNotNull("Entries should not be null", entries); |
||||
|
assertTrue("Empty directory should have no entries", entries.isEmpty()); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,417 @@ |
|||||
|
package seaweedfs.client; |
||||
|
|
||||
|
import org.junit.After; |
||||
|
import org.junit.Before; |
||||
|
import org.junit.Test; |
||||
|
|
||||
|
import java.io.IOException; |
||||
|
import java.nio.charset.StandardCharsets; |
||||
|
import java.util.Arrays; |
||||
|
import java.util.Random; |
||||
|
|
||||
|
import static org.junit.Assert.*; |
||||
|
|
||||
|
/** |
||||
|
* Integration tests for SeaweedInputStream and SeaweedOutputStream. |
||||
|
* |
||||
|
* These tests verify stream operations against a running SeaweedFS instance. |
||||
|
* |
||||
|
* Prerequisites: |
||||
|
* - SeaweedFS master, volume server, and filer must be running |
||||
|
* - Default ports: filer HTTP 8888, filer gRPC 18888 |
||||
|
* |
||||
|
* To run tests: |
||||
|
* export SEAWEEDFS_TEST_ENABLED=true |
||||
|
* mvn test -Dtest=SeaweedStreamIntegrationTest |
||||
|
*/ |
||||
|
public class SeaweedStreamIntegrationTest { |
||||
|
|
||||
|
private FilerClient filerClient; |
||||
|
private static final String TEST_ROOT = "/test-stream-integration"; |
||||
|
private static final boolean TESTS_ENABLED = |
||||
|
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); |
||||
|
|
||||
|
@Before |
||||
|
public void setUp() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
filerClient = new FilerClient("localhost", 18888); |
||||
|
|
||||
|
// Clean up any existing test directory |
||||
|
if (filerClient.exists(TEST_ROOT)) { |
||||
|
filerClient.rm(TEST_ROOT, true, true); |
||||
|
} |
||||
|
|
||||
|
// Create test root directory |
||||
|
filerClient.mkdirs(TEST_ROOT, 0755); |
||||
|
} |
||||
|
|
||||
|
@After |
||||
|
public void tearDown() throws Exception { |
||||
|
if (!TESTS_ENABLED || filerClient == null) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
try { |
||||
|
// Clean up test directory |
||||
|
if (filerClient.exists(TEST_ROOT)) { |
||||
|
filerClient.rm(TEST_ROOT, true, true); |
||||
|
} |
||||
|
} finally { |
||||
|
filerClient.shutdown(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testWriteAndReadSmallFile() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/small.txt"; |
||||
|
String testContent = "Hello, SeaweedFS!"; |
||||
|
|
||||
|
// Write file |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Verify file exists |
||||
|
assertTrue("File should exist", filerClient.exists(testPath)); |
||||
|
|
||||
|
// Read file |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
assertNotNull("Entry should not be null", entry); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
byte[] buffer = new byte[testContent.length()]; |
||||
|
int bytesRead = inputStream.read(buffer); |
||||
|
inputStream.close(); |
||||
|
|
||||
|
assertEquals("Should read all bytes", testContent.length(), bytesRead); |
||||
|
assertEquals("Content should match", testContent, new String(buffer, StandardCharsets.UTF_8)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testWriteAndReadLargeFile() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/large.bin"; |
||||
|
int fileSize = 10 * 1024 * 1024; // 10 MB |
||||
|
|
||||
|
// Generate random data |
||||
|
byte[] originalData = new byte[fileSize]; |
||||
|
new Random(42).nextBytes(originalData); // Use seed for reproducibility |
||||
|
|
||||
|
// Write file |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(originalData); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Verify file exists |
||||
|
assertTrue("File should exist", filerClient.exists(testPath)); |
||||
|
|
||||
|
// Read file |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
assertNotNull("Entry should not be null", entry); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
|
||||
|
// Read file in chunks to handle large files properly |
||||
|
byte[] readData = new byte[fileSize]; |
||||
|
int totalRead = 0; |
||||
|
int bytesRead; |
||||
|
byte[] buffer = new byte[8192]; // Read in 8KB chunks |
||||
|
|
||||
|
while ((bytesRead = inputStream.read(buffer)) > 0) { |
||||
|
System.arraycopy(buffer, 0, readData, totalRead, bytesRead); |
||||
|
totalRead += bytesRead; |
||||
|
} |
||||
|
inputStream.close(); |
||||
|
|
||||
|
assertEquals("Should read all bytes", fileSize, totalRead); |
||||
|
assertArrayEquals("Content should match", originalData, readData); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testWriteInChunks() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/chunked.txt"; |
||||
|
String[] chunks = {"First chunk. ", "Second chunk. ", "Third chunk."}; |
||||
|
|
||||
|
// Write file in chunks |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
for (String chunk : chunks) { |
||||
|
outputStream.write(chunk.getBytes(StandardCharsets.UTF_8)); |
||||
|
} |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Read and verify |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
byte[] buffer = new byte[1024]; |
||||
|
int bytesRead = inputStream.read(buffer); |
||||
|
inputStream.close(); |
||||
|
|
||||
|
String expected = String.join("", chunks); |
||||
|
String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); |
||||
|
|
||||
|
assertEquals("Content should match", expected, actual); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testReadWithOffset() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/offset.txt"; |
||||
|
String testContent = "0123456789ABCDEFGHIJ"; |
||||
|
|
||||
|
// Write file |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Read with offset |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
inputStream.seek(10); // Skip first 10 bytes |
||||
|
|
||||
|
byte[] buffer = new byte[10]; |
||||
|
int bytesRead = inputStream.read(buffer); |
||||
|
inputStream.close(); |
||||
|
|
||||
|
assertEquals("Should read 10 bytes", 10, bytesRead); |
||||
|
assertEquals("Should read from offset", "ABCDEFGHIJ", |
||||
|
new String(buffer, StandardCharsets.UTF_8)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testReadPartial() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/partial.txt"; |
||||
|
String testContent = "The quick brown fox jumps over the lazy dog"; |
||||
|
|
||||
|
// Write file |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Read partial |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
|
||||
|
// Read only "quick brown" |
||||
|
inputStream.seek(4); |
||||
|
byte[] buffer = new byte[11]; |
||||
|
int bytesRead = inputStream.read(buffer); |
||||
|
inputStream.close(); |
||||
|
|
||||
|
assertEquals("Should read 11 bytes", 11, bytesRead); |
||||
|
assertEquals("Should read partial content", "quick brown", |
||||
|
new String(buffer, StandardCharsets.UTF_8)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testEmptyFile() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/empty.txt"; |
||||
|
|
||||
|
// Write empty file |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Verify file exists |
||||
|
assertTrue("File should exist", filerClient.exists(testPath)); |
||||
|
|
||||
|
// Read empty file |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
assertNotNull("Entry should not be null", entry); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
byte[] buffer = new byte[100]; |
||||
|
int bytesRead = inputStream.read(buffer); |
||||
|
inputStream.close(); |
||||
|
|
||||
|
assertEquals("Should read 0 bytes from empty file", -1, bytesRead); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testOverwriteFile() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/overwrite.txt"; |
||||
|
String originalContent = "Original content"; |
||||
|
String newContent = "New content that overwrites the original"; |
||||
|
|
||||
|
// Write original file |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(originalContent.getBytes(StandardCharsets.UTF_8)); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Overwrite file |
||||
|
outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(newContent.getBytes(StandardCharsets.UTF_8)); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Read and verify |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
byte[] buffer = new byte[1024]; |
||||
|
int bytesRead = inputStream.read(buffer); |
||||
|
inputStream.close(); |
||||
|
|
||||
|
String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); |
||||
|
assertEquals("Should have new content", newContent, actual); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testMultipleReads() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/multireads.txt"; |
||||
|
String testContent = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; |
||||
|
|
||||
|
// Write file |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Read in multiple small chunks |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
|
||||
|
StringBuilder result = new StringBuilder(); |
||||
|
byte[] buffer = new byte[5]; |
||||
|
int bytesRead; |
||||
|
while ((bytesRead = inputStream.read(buffer)) > 0) { |
||||
|
result.append(new String(buffer, 0, bytesRead, StandardCharsets.UTF_8)); |
||||
|
} |
||||
|
inputStream.close(); |
||||
|
|
||||
|
assertEquals("Should read entire content", testContent, result.toString()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testBinaryData() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/binary.bin"; |
||||
|
byte[] binaryData = new byte[256]; |
||||
|
for (int i = 0; i < 256; i++) { |
||||
|
binaryData[i] = (byte) i; |
||||
|
} |
||||
|
|
||||
|
// Write binary file |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(binaryData); |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Read and verify |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
byte[] readData = new byte[256]; |
||||
|
int bytesRead = inputStream.read(readData); |
||||
|
inputStream.close(); |
||||
|
|
||||
|
assertEquals("Should read all bytes", 256, bytesRead); |
||||
|
assertArrayEquals("Binary data should match", binaryData, readData); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testFlush() throws IOException { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String testPath = TEST_ROOT + "/flush.txt"; |
||||
|
String testContent = "Content to flush"; |
||||
|
|
||||
|
// Write file with flush |
||||
|
SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); |
||||
|
outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); |
||||
|
outputStream.flush(); // Explicitly flush |
||||
|
outputStream.close(); |
||||
|
|
||||
|
// Verify file was written |
||||
|
assertTrue("File should exist after flush", filerClient.exists(testPath)); |
||||
|
|
||||
|
// Read and verify |
||||
|
FilerProto.Entry entry = filerClient.lookupEntry( |
||||
|
SeaweedOutputStream.getParentDirectory(testPath), |
||||
|
SeaweedOutputStream.getFileName(testPath) |
||||
|
); |
||||
|
|
||||
|
SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); |
||||
|
byte[] buffer = new byte[testContent.length()]; |
||||
|
int bytesRead = inputStream.read(buffer); |
||||
|
inputStream.close(); |
||||
|
|
||||
|
assertEquals("Content should match", testContent, |
||||
|
new String(buffer, 0, bytesRead, StandardCharsets.UTF_8)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
@ -0,0 +1,190 @@ |
|||||
|
# SeaweedFS Hadoop2 Client |
||||
|
|
||||
|
Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 2.x/3.x. |
||||
|
|
||||
|
## Building |
||||
|
|
||||
|
```bash |
||||
|
mvn clean install |
||||
|
``` |
||||
|
|
||||
|
## Testing |
||||
|
|
||||
|
This project includes two types of tests: |
||||
|
|
||||
|
### 1. Configuration Tests (No SeaweedFS Required) |
||||
|
|
||||
|
These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance: |
||||
|
|
||||
|
```bash |
||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest |
||||
|
``` |
||||
|
|
||||
|
### 2. Integration Tests (Requires SeaweedFS) |
||||
|
|
||||
|
These tests verify actual FileSystem operations against a running SeaweedFS instance. |
||||
|
|
||||
|
#### Prerequisites |
||||
|
|
||||
|
1. Start SeaweedFS with default ports: |
||||
|
```bash |
||||
|
# Terminal 1: Start master |
||||
|
weed master |
||||
|
|
||||
|
# Terminal 2: Start volume server |
||||
|
weed volume -mserver=localhost:9333 |
||||
|
|
||||
|
# Terminal 3: Start filer |
||||
|
weed filer -master=localhost:9333 |
||||
|
``` |
||||
|
|
||||
|
2. Verify services are running: |
||||
|
- Master: http://localhost:9333 |
||||
|
- Filer HTTP: http://localhost:8888 |
||||
|
- Filer gRPC: localhost:18888 |
||||
|
|
||||
|
#### Running Integration Tests |
||||
|
|
||||
|
```bash |
||||
|
# Enable integration tests |
||||
|
export SEAWEEDFS_TEST_ENABLED=true |
||||
|
|
||||
|
# Run all tests |
||||
|
mvn test |
||||
|
|
||||
|
# Run specific test |
||||
|
mvn test -Dtest=SeaweedFileSystemTest |
||||
|
``` |
||||
|
|
||||
|
### Test Configuration |
||||
|
|
||||
|
Integration tests can be configured via environment variables or system properties: |
||||
|
|
||||
|
- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false) |
||||
|
- Tests use these default connection settings: |
||||
|
- Filer Host: localhost |
||||
|
- Filer HTTP Port: 8888 |
||||
|
- Filer gRPC Port: 18888 |
||||
|
|
||||
|
### Running Tests with Custom Configuration |
||||
|
|
||||
|
To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration: |
||||
|
|
||||
|
```java |
||||
|
conf.set("fs.seaweed.filer.host", "your-host"); |
||||
|
conf.setInt("fs.seaweed.filer.port", 8888); |
||||
|
conf.setInt("fs.seaweed.filer.port.grpc", 18888); |
||||
|
``` |
||||
|
|
||||
|
## Test Coverage |
||||
|
|
||||
|
The test suite covers: |
||||
|
|
||||
|
- **Configuration & Initialization** |
||||
|
- URI parsing and configuration |
||||
|
- Default values |
||||
|
- Configuration overrides |
||||
|
- Working directory management |
||||
|
|
||||
|
- **File Operations** |
||||
|
- Create files |
||||
|
- Read files |
||||
|
- Write files |
||||
|
- Append to files |
||||
|
- Delete files |
||||
|
|
||||
|
- **Directory Operations** |
||||
|
- Create directories |
||||
|
- List directory contents |
||||
|
- Delete directories (recursive and non-recursive) |
||||
|
|
||||
|
- **Metadata Operations** |
||||
|
- Get file status |
||||
|
- Set permissions |
||||
|
- Set owner/group |
||||
|
- Rename files and directories |
||||
|
|
||||
|
## Usage in Hadoop |
||||
|
|
||||
|
1. Copy the built JAR to your Hadoop classpath: |
||||
|
```bash |
||||
|
cp target/seaweedfs-hadoop2-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/ |
||||
|
``` |
||||
|
|
||||
|
2. Configure `core-site.xml`: |
||||
|
```xml |
||||
|
<configuration> |
||||
|
<property> |
||||
|
<name>fs.seaweedfs.impl</name> |
||||
|
<value>seaweed.hdfs.SeaweedFileSystem</value> |
||||
|
</property> |
||||
|
<property> |
||||
|
<name>fs.seaweed.filer.host</name> |
||||
|
<value>localhost</value> |
||||
|
</property> |
||||
|
<property> |
||||
|
<name>fs.seaweed.filer.port</name> |
||||
|
<value>8888</value> |
||||
|
</property> |
||||
|
<property> |
||||
|
<name>fs.seaweed.filer.port.grpc</name> |
||||
|
<value>18888</value> |
||||
|
</property> |
||||
|
</configuration> |
||||
|
``` |
||||
|
|
||||
|
3. Use SeaweedFS with Hadoop commands: |
||||
|
```bash |
||||
|
hadoop fs -ls seaweedfs://localhost:8888/ |
||||
|
hadoop fs -mkdir seaweedfs://localhost:8888/test |
||||
|
hadoop fs -put local.txt seaweedfs://localhost:8888/test/ |
||||
|
``` |
||||
|
|
||||
|
## Continuous Integration |
||||
|
|
||||
|
For CI environments, tests can be run in two modes: |
||||
|
|
||||
|
1. **Configuration Tests Only** (default, no SeaweedFS required): |
||||
|
```bash |
||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest |
||||
|
``` |
||||
|
|
||||
|
2. **Full Integration Tests** (requires SeaweedFS): |
||||
|
```bash |
||||
|
# Start SeaweedFS in CI environment |
||||
|
# Then run: |
||||
|
export SEAWEEDFS_TEST_ENABLED=true |
||||
|
mvn test |
||||
|
``` |
||||
|
|
||||
|
## Troubleshooting |
||||
|
|
||||
|
### Tests are skipped |
||||
|
|
||||
|
If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set": |
||||
|
```bash |
||||
|
export SEAWEEDFS_TEST_ENABLED=true |
||||
|
``` |
||||
|
|
||||
|
### Connection refused errors |
||||
|
|
||||
|
Ensure SeaweedFS is running and accessible: |
||||
|
```bash |
||||
|
curl http://localhost:8888/ |
||||
|
``` |
||||
|
|
||||
|
### gRPC errors |
||||
|
|
||||
|
Verify the gRPC port is accessible: |
||||
|
```bash |
||||
|
# Should show the port is listening |
||||
|
netstat -an | grep 18888 |
||||
|
``` |
||||
|
|
||||
|
## Contributing |
||||
|
|
||||
|
When adding new features, please include: |
||||
|
1. Configuration tests (no SeaweedFS required) |
||||
|
2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard) |
||||
|
3. Documentation updates |
||||
|
|
||||
@ -0,0 +1,90 @@ |
|||||
|
package seaweed.hdfs; |
||||
|
|
||||
|
import org.apache.hadoop.conf.Configuration; |
||||
|
import org.apache.hadoop.fs.Path; |
||||
|
import org.junit.Before; |
||||
|
import org.junit.Test; |
||||
|
|
||||
|
import static org.junit.Assert.*; |
||||
|
|
||||
|
/** |
||||
|
* Unit tests for SeaweedFileSystem configuration that don't require a running SeaweedFS instance. |
||||
|
* |
||||
|
* These tests verify basic properties and constants. |
||||
|
*/ |
||||
|
public class SeaweedFileSystemConfigTest { |
||||
|
|
||||
|
private SeaweedFileSystem fs; |
||||
|
private Configuration conf; |
||||
|
|
||||
|
@Before |
||||
|
public void setUp() { |
||||
|
fs = new SeaweedFileSystem(); |
||||
|
conf = new Configuration(); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testScheme() { |
||||
|
assertEquals("seaweedfs", fs.getScheme()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testConstants() { |
||||
|
// Test that constants are defined correctly |
||||
|
assertEquals("fs.seaweed.filer.host", SeaweedFileSystem.FS_SEAWEED_FILER_HOST); |
||||
|
assertEquals("fs.seaweed.filer.port", SeaweedFileSystem.FS_SEAWEED_FILER_PORT); |
||||
|
assertEquals("fs.seaweed.filer.port.grpc", SeaweedFileSystem.FS_SEAWEED_FILER_PORT_GRPC); |
||||
|
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); |
||||
|
assertEquals("fs.seaweed.buffer.size", SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE); |
||||
|
assertEquals(4 * 1024 * 1024, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); |
||||
|
assertEquals("fs.seaweed.replication", SeaweedFileSystem.FS_SEAWEED_REPLICATION); |
||||
|
assertEquals("fs.seaweed.volume.server.access", SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS); |
||||
|
assertEquals("fs.seaweed.filer.cn", SeaweedFileSystem.FS_SEAWEED_FILER_CN); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testWorkingDirectoryPathOperations() { |
||||
|
// Test path operations that don't require initialization |
||||
|
Path testPath = new Path("/test/path"); |
||||
|
assertTrue("Path should be absolute", testPath.isAbsolute()); |
||||
|
assertEquals("/test/path", testPath.toUri().getPath()); |
||||
|
|
||||
|
Path childPath = new Path(testPath, "child"); |
||||
|
assertEquals("/test/path/child", childPath.toUri().getPath()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testConfigurationProperties() { |
||||
|
// Test that configuration can be set and read |
||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_HOST, "testhost"); |
||||
|
assertEquals("testhost", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_HOST)); |
||||
|
|
||||
|
conf.setInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 9999); |
||||
|
assertEquals(9999, conf.getInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 0)); |
||||
|
|
||||
|
conf.setInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 8 * 1024 * 1024); |
||||
|
assertEquals(8 * 1024 * 1024, conf.getInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 0)); |
||||
|
|
||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_REPLICATION, "001"); |
||||
|
assertEquals("001", conf.get(SeaweedFileSystem.FS_SEAWEED_REPLICATION)); |
||||
|
|
||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS, "publicUrl"); |
||||
|
assertEquals("publicUrl", conf.get(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS)); |
||||
|
|
||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_CN, "test-cn"); |
||||
|
assertEquals("test-cn", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_CN)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDefaultBufferSize() { |
||||
|
// Test default buffer size constant |
||||
|
int expected = 4 * 1024 * 1024; // 4MB |
||||
|
assertEquals(expected, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDefaultPort() { |
||||
|
// Test default port constant |
||||
|
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,379 @@ |
|||||
|
package seaweed.hdfs; |
||||
|
|
||||
|
import org.apache.hadoop.conf.Configuration; |
||||
|
import org.apache.hadoop.fs.FSDataInputStream; |
||||
|
import org.apache.hadoop.fs.FSDataOutputStream; |
||||
|
import org.apache.hadoop.fs.FileStatus; |
||||
|
import org.apache.hadoop.fs.Path; |
||||
|
import org.apache.hadoop.fs.permission.FsPermission; |
||||
|
import org.junit.After; |
||||
|
import org.junit.Before; |
||||
|
import org.junit.Test; |
||||
|
|
||||
|
import java.io.IOException; |
||||
|
import java.net.URI; |
||||
|
|
||||
|
import static org.junit.Assert.*; |
||||
|
|
||||
|
/** |
||||
|
* Unit tests for SeaweedFileSystem. |
||||
|
* |
||||
|
* These tests verify basic FileSystem operations against a SeaweedFS backend. |
||||
|
* Note: These tests require a running SeaweedFS filer instance. |
||||
|
* |
||||
|
* To run tests, ensure SeaweedFS is running with default ports: |
||||
|
* - Filer HTTP: 8888 |
||||
|
* - Filer gRPC: 18888 |
||||
|
* |
||||
|
* Set environment variable SEAWEEDFS_TEST_ENABLED=true to enable these tests. |
||||
|
*/ |
||||
|
public class SeaweedFileSystemTest { |
||||
|
|
||||
|
private SeaweedFileSystem fs; |
||||
|
private Configuration conf; |
||||
|
private static final String TEST_ROOT = "/test-hdfs2"; |
||||
|
private static final boolean TESTS_ENABLED = |
||||
|
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); |
||||
|
|
||||
|
@Before |
||||
|
public void setUp() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
conf = new Configuration(); |
||||
|
conf.set("fs.seaweed.filer.host", "localhost"); |
||||
|
conf.setInt("fs.seaweed.filer.port", 8888); |
||||
|
conf.setInt("fs.seaweed.filer.port.grpc", 18888); |
||||
|
|
||||
|
fs = new SeaweedFileSystem(); |
||||
|
URI uri = new URI("seaweedfs://localhost:8888/"); |
||||
|
fs.initialize(uri, conf); |
||||
|
|
||||
|
// Clean up any existing test directory |
||||
|
Path testPath = new Path(TEST_ROOT); |
||||
|
if (fs.exists(testPath)) { |
||||
|
fs.delete(testPath, true); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@After |
||||
|
public void tearDown() throws Exception { |
||||
|
if (!TESTS_ENABLED || fs == null) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// Clean up test directory |
||||
|
Path testPath = new Path(TEST_ROOT); |
||||
|
if (fs.exists(testPath)) { |
||||
|
fs.delete(testPath, true); |
||||
|
} |
||||
|
|
||||
|
fs.close(); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testInitialization() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
assertNotNull(fs); |
||||
|
assertEquals("seaweedfs", fs.getScheme()); |
||||
|
assertNotNull(fs.getUri()); |
||||
|
assertEquals("/", fs.getWorkingDirectory().toUri().getPath()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testMkdirs() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testDir = new Path(TEST_ROOT + "/testdir"); |
||||
|
assertTrue("Failed to create directory", fs.mkdirs(testDir)); |
||||
|
assertTrue("Directory should exist", fs.exists(testDir)); |
||||
|
|
||||
|
FileStatus status = fs.getFileStatus(testDir); |
||||
|
assertTrue("Path should be a directory", status.isDirectory()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testCreateAndReadFile() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/testfile.txt"); |
||||
|
String testContent = "Hello, SeaweedFS!"; |
||||
|
|
||||
|
// Create and write to file |
||||
|
FSDataOutputStream out = fs.create(testFile, FsPermission.getDefault(), |
||||
|
false, 4096, (short) 1, 4 * 1024 * 1024, null); |
||||
|
assertNotNull("Output stream should not be null", out); |
||||
|
out.write(testContent.getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
// Verify file exists |
||||
|
assertTrue("File should exist", fs.exists(testFile)); |
||||
|
|
||||
|
// Read and verify content |
||||
|
FSDataInputStream in = fs.open(testFile, 4096); |
||||
|
assertNotNull("Input stream should not be null", in); |
||||
|
byte[] buffer = new byte[testContent.length()]; |
||||
|
int bytesRead = in.read(buffer); |
||||
|
in.close(); |
||||
|
|
||||
|
assertEquals("Should read all bytes", testContent.length(), bytesRead); |
||||
|
assertEquals("Content should match", testContent, new String(buffer)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testFileStatus() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/statustest.txt"); |
||||
|
String content = "test content"; |
||||
|
|
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write(content.getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
FileStatus status = fs.getFileStatus(testFile); |
||||
|
assertNotNull("FileStatus should not be null", status); |
||||
|
assertFalse("Should not be a directory", status.isDirectory()); |
||||
|
assertTrue("Should be a file", status.isFile()); |
||||
|
assertEquals("File length should match", content.length(), status.getLen()); |
||||
|
assertNotNull("Path should not be null", status.getPath()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testListStatus() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testDir = new Path(TEST_ROOT + "/listtest"); |
||||
|
fs.mkdirs(testDir); |
||||
|
|
||||
|
// Create multiple files |
||||
|
for (int i = 0; i < 3; i++) { |
||||
|
Path file = new Path(testDir, "file" + i + ".txt"); |
||||
|
FSDataOutputStream out = fs.create(file); |
||||
|
out.write(("content" + i).getBytes()); |
||||
|
out.close(); |
||||
|
} |
||||
|
|
||||
|
FileStatus[] statuses = fs.listStatus(testDir); |
||||
|
assertNotNull("List should not be null", statuses); |
||||
|
assertEquals("Should have 3 files", 3, statuses.length); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testRename() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path srcFile = new Path(TEST_ROOT + "/source.txt"); |
||||
|
Path dstFile = new Path(TEST_ROOT + "/destination.txt"); |
||||
|
String content = "rename test"; |
||||
|
|
||||
|
// Create source file |
||||
|
FSDataOutputStream out = fs.create(srcFile); |
||||
|
out.write(content.getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
assertTrue("Source file should exist", fs.exists(srcFile)); |
||||
|
|
||||
|
// Rename |
||||
|
assertTrue("Rename should succeed", fs.rename(srcFile, dstFile)); |
||||
|
|
||||
|
// Verify |
||||
|
assertFalse("Source file should not exist", fs.exists(srcFile)); |
||||
|
assertTrue("Destination file should exist", fs.exists(dstFile)); |
||||
|
|
||||
|
// Verify content preserved |
||||
|
FSDataInputStream in = fs.open(dstFile); |
||||
|
byte[] buffer = new byte[content.length()]; |
||||
|
in.read(buffer); |
||||
|
in.close(); |
||||
|
assertEquals("Content should be preserved", content, new String(buffer)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDelete() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/deletetest.txt"); |
||||
|
|
||||
|
// Create file |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write("delete me".getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
assertTrue("File should exist before delete", fs.exists(testFile)); |
||||
|
|
||||
|
// Delete |
||||
|
assertTrue("Delete should succeed", fs.delete(testFile, false)); |
||||
|
assertFalse("File should not exist after delete", fs.exists(testFile)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDeleteDirectory() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testDir = new Path(TEST_ROOT + "/deletedir"); |
||||
|
Path testFile = new Path(testDir, "file.txt"); |
||||
|
|
||||
|
// Create directory with file |
||||
|
fs.mkdirs(testDir); |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write("content".getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
assertTrue("Directory should exist", fs.exists(testDir)); |
||||
|
assertTrue("File should exist", fs.exists(testFile)); |
||||
|
|
||||
|
// Recursive delete |
||||
|
assertTrue("Recursive delete should succeed", fs.delete(testDir, true)); |
||||
|
assertFalse("Directory should not exist after delete", fs.exists(testDir)); |
||||
|
assertFalse("File should not exist after delete", fs.exists(testFile)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testAppend() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/appendtest.txt"); |
||||
|
String initialContent = "initial"; |
||||
|
String appendContent = " appended"; |
||||
|
|
||||
|
// Create initial file |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write(initialContent.getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
// Append |
||||
|
FSDataOutputStream appendOut = fs.append(testFile, 4096, null); |
||||
|
assertNotNull("Append stream should not be null", appendOut); |
||||
|
appendOut.write(appendContent.getBytes()); |
||||
|
appendOut.close(); |
||||
|
|
||||
|
// Verify combined content |
||||
|
FSDataInputStream in = fs.open(testFile); |
||||
|
byte[] buffer = new byte[initialContent.length() + appendContent.length()]; |
||||
|
int bytesRead = in.read(buffer); |
||||
|
in.close(); |
||||
|
|
||||
|
String expected = initialContent + appendContent; |
||||
|
assertEquals("Should read all bytes", expected.length(), bytesRead); |
||||
|
assertEquals("Content should match", expected, new String(buffer)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testSetWorkingDirectory() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path originalWd = fs.getWorkingDirectory(); |
||||
|
assertEquals("Original working directory should be /", "/", originalWd.toUri().getPath()); |
||||
|
|
||||
|
Path newWd = new Path(TEST_ROOT); |
||||
|
fs.mkdirs(newWd); |
||||
|
fs.setWorkingDirectory(newWd); |
||||
|
|
||||
|
Path currentWd = fs.getWorkingDirectory(); |
||||
|
assertTrue("Working directory should be updated", |
||||
|
currentWd.toUri().getPath().contains(TEST_ROOT)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testSetPermission() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/permtest.txt"); |
||||
|
|
||||
|
// Create file |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write("permission test".getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
// Set permission |
||||
|
FsPermission newPerm = new FsPermission((short) 0644); |
||||
|
fs.setPermission(testFile, newPerm); |
||||
|
|
||||
|
FileStatus status = fs.getFileStatus(testFile); |
||||
|
assertNotNull("Permission should not be null", status.getPermission()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testSetOwner() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/ownertest.txt"); |
||||
|
|
||||
|
// Create file |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write("owner test".getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
// Set owner - this may not fail even if not fully implemented |
||||
|
fs.setOwner(testFile, "testuser", "testgroup"); |
||||
|
|
||||
|
// Just verify the call doesn't throw an exception |
||||
|
FileStatus status = fs.getFileStatus(testFile); |
||||
|
assertNotNull("FileStatus should not be null", status); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testRenameToExistingDirectory() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path srcFile = new Path(TEST_ROOT + "/movefile.txt"); |
||||
|
Path dstDir = new Path(TEST_ROOT + "/movedir"); |
||||
|
|
||||
|
// Create source file and destination directory |
||||
|
FSDataOutputStream out = fs.create(srcFile); |
||||
|
out.write("move test".getBytes()); |
||||
|
out.close(); |
||||
|
fs.mkdirs(dstDir); |
||||
|
|
||||
|
// Rename file to existing directory (should move file into directory) |
||||
|
assertTrue("Rename to directory should succeed", fs.rename(srcFile, dstDir)); |
||||
|
|
||||
|
// File should be moved into the directory |
||||
|
Path expectedLocation = new Path(dstDir, srcFile.getName()); |
||||
|
assertTrue("File should exist in destination directory", fs.exists(expectedLocation)); |
||||
|
assertFalse("Source file should not exist", fs.exists(srcFile)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
@ -0,0 +1,190 @@ |
|||||
|
# SeaweedFS Hadoop3 Client |
||||
|
|
||||
|
Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 3.x. |
||||
|
|
||||
|
## Building |
||||
|
|
||||
|
```bash |
||||
|
mvn clean install |
||||
|
``` |
||||
|
|
||||
|
## Testing |
||||
|
|
||||
|
This project includes two types of tests: |
||||
|
|
||||
|
### 1. Configuration Tests (No SeaweedFS Required) |
||||
|
|
||||
|
These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance: |
||||
|
|
||||
|
```bash |
||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest |
||||
|
``` |
||||
|
|
||||
|
### 2. Integration Tests (Requires SeaweedFS) |
||||
|
|
||||
|
These tests verify actual FileSystem operations against a running SeaweedFS instance. |
||||
|
|
||||
|
#### Prerequisites |
||||
|
|
||||
|
1. Start SeaweedFS with default ports: |
||||
|
```bash |
||||
|
# Terminal 1: Start master |
||||
|
weed master |
||||
|
|
||||
|
# Terminal 2: Start volume server |
||||
|
weed volume -mserver=localhost:9333 |
||||
|
|
||||
|
# Terminal 3: Start filer |
||||
|
weed filer -master=localhost:9333 |
||||
|
``` |
||||
|
|
||||
|
2. Verify services are running: |
||||
|
- Master: http://localhost:9333 |
||||
|
- Filer HTTP: http://localhost:8888 |
||||
|
- Filer gRPC: localhost:18888 |
||||
|
|
||||
|
#### Running Integration Tests |
||||
|
|
||||
|
```bash |
||||
|
# Enable integration tests |
||||
|
export SEAWEEDFS_TEST_ENABLED=true |
||||
|
|
||||
|
# Run all tests |
||||
|
mvn test |
||||
|
|
||||
|
# Run specific test |
||||
|
mvn test -Dtest=SeaweedFileSystemTest |
||||
|
``` |
||||
|
|
||||
|
### Test Configuration |
||||
|
|
||||
|
Integration tests can be configured via environment variables or system properties: |
||||
|
|
||||
|
- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false) |
||||
|
- Tests use these default connection settings: |
||||
|
- Filer Host: localhost |
||||
|
- Filer HTTP Port: 8888 |
||||
|
- Filer gRPC Port: 18888 |
||||
|
|
||||
|
### Running Tests with Custom Configuration |
||||
|
|
||||
|
To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration: |
||||
|
|
||||
|
```java |
||||
|
conf.set("fs.seaweed.filer.host", "your-host"); |
||||
|
conf.setInt("fs.seaweed.filer.port", 8888); |
||||
|
conf.setInt("fs.seaweed.filer.port.grpc", 18888); |
||||
|
``` |
||||
|
|
||||
|
## Test Coverage |
||||
|
|
||||
|
The test suite covers: |
||||
|
|
||||
|
- **Configuration & Initialization** |
||||
|
- URI parsing and configuration |
||||
|
- Default values |
||||
|
- Configuration overrides |
||||
|
- Working directory management |
||||
|
|
||||
|
- **File Operations** |
||||
|
- Create files |
||||
|
- Read files |
||||
|
- Write files |
||||
|
- Append to files |
||||
|
- Delete files |
||||
|
|
||||
|
- **Directory Operations** |
||||
|
- Create directories |
||||
|
- List directory contents |
||||
|
- Delete directories (recursive and non-recursive) |
||||
|
|
||||
|
- **Metadata Operations** |
||||
|
- Get file status |
||||
|
- Set permissions |
||||
|
- Set owner/group |
||||
|
- Rename files and directories |
||||
|
|
||||
|
## Usage in Hadoop |
||||
|
|
||||
|
1. Copy the built JAR to your Hadoop classpath: |
||||
|
```bash |
||||
|
cp target/seaweedfs-hadoop3-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/ |
||||
|
``` |
||||
|
|
||||
|
2. Configure `core-site.xml`: |
||||
|
```xml |
||||
|
<configuration> |
||||
|
<property> |
||||
|
<name>fs.seaweedfs.impl</name> |
||||
|
<value>seaweed.hdfs.SeaweedFileSystem</value> |
||||
|
</property> |
||||
|
<property> |
||||
|
<name>fs.seaweed.filer.host</name> |
||||
|
<value>localhost</value> |
||||
|
</property> |
||||
|
<property> |
||||
|
<name>fs.seaweed.filer.port</name> |
||||
|
<value>8888</value> |
||||
|
</property> |
||||
|
<property> |
||||
|
<name>fs.seaweed.filer.port.grpc</name> |
||||
|
<value>18888</value> |
||||
|
</property> |
||||
|
</configuration> |
||||
|
``` |
||||
|
|
||||
|
3. Use SeaweedFS with Hadoop commands: |
||||
|
```bash |
||||
|
hadoop fs -ls seaweedfs://localhost:8888/ |
||||
|
hadoop fs -mkdir seaweedfs://localhost:8888/test |
||||
|
hadoop fs -put local.txt seaweedfs://localhost:8888/test/ |
||||
|
``` |
||||
|
|
||||
|
## Continuous Integration |
||||
|
|
||||
|
For CI environments, tests can be run in two modes: |
||||
|
|
||||
|
1. **Configuration Tests Only** (default, no SeaweedFS required): |
||||
|
```bash |
||||
|
mvn test -Dtest=SeaweedFileSystemConfigTest |
||||
|
``` |
||||
|
|
||||
|
2. **Full Integration Tests** (requires SeaweedFS): |
||||
|
```bash |
||||
|
# Start SeaweedFS in CI environment |
||||
|
# Then run: |
||||
|
export SEAWEEDFS_TEST_ENABLED=true |
||||
|
mvn test |
||||
|
``` |
||||
|
|
||||
|
## Troubleshooting |
||||
|
|
||||
|
### Tests are skipped |
||||
|
|
||||
|
If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set": |
||||
|
```bash |
||||
|
export SEAWEEDFS_TEST_ENABLED=true |
||||
|
``` |
||||
|
|
||||
|
### Connection refused errors |
||||
|
|
||||
|
Ensure SeaweedFS is running and accessible: |
||||
|
```bash |
||||
|
curl http://localhost:8888/ |
||||
|
``` |
||||
|
|
||||
|
### gRPC errors |
||||
|
|
||||
|
Verify the gRPC port is accessible: |
||||
|
```bash |
||||
|
# Should show the port is listening |
||||
|
netstat -an | grep 18888 |
||||
|
``` |
||||
|
|
||||
|
## Contributing |
||||
|
|
||||
|
When adding new features, please include: |
||||
|
1. Configuration tests (no SeaweedFS required) |
||||
|
2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard) |
||||
|
3. Documentation updates |
||||
|
|
||||
@ -0,0 +1,90 @@ |
|||||
|
package seaweed.hdfs; |
||||
|
|
||||
|
import org.apache.hadoop.conf.Configuration; |
||||
|
import org.apache.hadoop.fs.Path; |
||||
|
import org.junit.Before; |
||||
|
import org.junit.Test; |
||||
|
|
||||
|
import static org.junit.Assert.*; |
||||
|
|
||||
|
/** |
||||
|
* Unit tests for SeaweedFileSystem configuration that don't require a running SeaweedFS instance. |
||||
|
* |
||||
|
* These tests verify basic properties and constants. |
||||
|
*/ |
||||
|
public class SeaweedFileSystemConfigTest { |
||||
|
|
||||
|
private SeaweedFileSystem fs; |
||||
|
private Configuration conf; |
||||
|
|
||||
|
@Before |
||||
|
public void setUp() { |
||||
|
fs = new SeaweedFileSystem(); |
||||
|
conf = new Configuration(); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testScheme() { |
||||
|
assertEquals("seaweedfs", fs.getScheme()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testConstants() { |
||||
|
// Test that constants are defined correctly |
||||
|
assertEquals("fs.seaweed.filer.host", SeaweedFileSystem.FS_SEAWEED_FILER_HOST); |
||||
|
assertEquals("fs.seaweed.filer.port", SeaweedFileSystem.FS_SEAWEED_FILER_PORT); |
||||
|
assertEquals("fs.seaweed.filer.port.grpc", SeaweedFileSystem.FS_SEAWEED_FILER_PORT_GRPC); |
||||
|
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); |
||||
|
assertEquals("fs.seaweed.buffer.size", SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE); |
||||
|
assertEquals(4 * 1024 * 1024, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); |
||||
|
assertEquals("fs.seaweed.replication", SeaweedFileSystem.FS_SEAWEED_REPLICATION); |
||||
|
assertEquals("fs.seaweed.volume.server.access", SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS); |
||||
|
assertEquals("fs.seaweed.filer.cn", SeaweedFileSystem.FS_SEAWEED_FILER_CN); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testWorkingDirectoryPathOperations() { |
||||
|
// Test path operations that don't require initialization |
||||
|
Path testPath = new Path("/test/path"); |
||||
|
assertTrue("Path should be absolute", testPath.isAbsolute()); |
||||
|
assertEquals("/test/path", testPath.toUri().getPath()); |
||||
|
|
||||
|
Path childPath = new Path(testPath, "child"); |
||||
|
assertEquals("/test/path/child", childPath.toUri().getPath()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testConfigurationProperties() { |
||||
|
// Test that configuration can be set and read |
||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_HOST, "testhost"); |
||||
|
assertEquals("testhost", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_HOST)); |
||||
|
|
||||
|
conf.setInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 9999); |
||||
|
assertEquals(9999, conf.getInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 0)); |
||||
|
|
||||
|
conf.setInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 8 * 1024 * 1024); |
||||
|
assertEquals(8 * 1024 * 1024, conf.getInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 0)); |
||||
|
|
||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_REPLICATION, "001"); |
||||
|
assertEquals("001", conf.get(SeaweedFileSystem.FS_SEAWEED_REPLICATION)); |
||||
|
|
||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS, "publicUrl"); |
||||
|
assertEquals("publicUrl", conf.get(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS)); |
||||
|
|
||||
|
conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_CN, "test-cn"); |
||||
|
assertEquals("test-cn", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_CN)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDefaultBufferSize() { |
||||
|
// Test default buffer size constant |
||||
|
int expected = 4 * 1024 * 1024; // 4MB |
||||
|
assertEquals(expected, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDefaultPort() { |
||||
|
// Test default port constant |
||||
|
assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,379 @@ |
|||||
|
package seaweed.hdfs; |
||||
|
|
||||
|
import org.apache.hadoop.conf.Configuration; |
||||
|
import org.apache.hadoop.fs.FSDataInputStream; |
||||
|
import org.apache.hadoop.fs.FSDataOutputStream; |
||||
|
import org.apache.hadoop.fs.FileStatus; |
||||
|
import org.apache.hadoop.fs.Path; |
||||
|
import org.apache.hadoop.fs.permission.FsPermission; |
||||
|
import org.junit.After; |
||||
|
import org.junit.Before; |
||||
|
import org.junit.Test; |
||||
|
|
||||
|
import java.io.IOException; |
||||
|
import java.net.URI; |
||||
|
|
||||
|
import static org.junit.Assert.*; |
||||
|
|
||||
|
/** |
||||
|
* Unit tests for SeaweedFileSystem. |
||||
|
* |
||||
|
* These tests verify basic FileSystem operations against a SeaweedFS backend. |
||||
|
* Note: These tests require a running SeaweedFS filer instance. |
||||
|
* |
||||
|
* To run tests, ensure SeaweedFS is running with default ports: |
||||
|
* - Filer HTTP: 8888 |
||||
|
* - Filer gRPC: 18888 |
||||
|
* |
||||
|
* Set environment variable SEAWEEDFS_TEST_ENABLED=true to enable these tests. |
||||
|
*/ |
||||
|
public class SeaweedFileSystemTest { |
||||
|
|
||||
|
private SeaweedFileSystem fs; |
||||
|
private Configuration conf; |
||||
|
private static final String TEST_ROOT = "/test-hdfs3"; |
||||
|
private static final boolean TESTS_ENABLED = |
||||
|
"true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); |
||||
|
|
||||
|
@Before |
||||
|
public void setUp() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
conf = new Configuration(); |
||||
|
conf.set("fs.seaweed.filer.host", "localhost"); |
||||
|
conf.setInt("fs.seaweed.filer.port", 8888); |
||||
|
conf.setInt("fs.seaweed.filer.port.grpc", 18888); |
||||
|
|
||||
|
fs = new SeaweedFileSystem(); |
||||
|
URI uri = new URI("seaweedfs://localhost:8888/"); |
||||
|
fs.initialize(uri, conf); |
||||
|
|
||||
|
// Clean up any existing test directory |
||||
|
Path testPath = new Path(TEST_ROOT); |
||||
|
if (fs.exists(testPath)) { |
||||
|
fs.delete(testPath, true); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@After |
||||
|
public void tearDown() throws Exception { |
||||
|
if (!TESTS_ENABLED || fs == null) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// Clean up test directory |
||||
|
Path testPath = new Path(TEST_ROOT); |
||||
|
if (fs.exists(testPath)) { |
||||
|
fs.delete(testPath, true); |
||||
|
} |
||||
|
|
||||
|
fs.close(); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testInitialization() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
assertNotNull(fs); |
||||
|
assertEquals("seaweedfs", fs.getScheme()); |
||||
|
assertNotNull(fs.getUri()); |
||||
|
assertEquals("/", fs.getWorkingDirectory().toUri().getPath()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testMkdirs() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testDir = new Path(TEST_ROOT + "/testdir"); |
||||
|
assertTrue("Failed to create directory", fs.mkdirs(testDir)); |
||||
|
assertTrue("Directory should exist", fs.exists(testDir)); |
||||
|
|
||||
|
FileStatus status = fs.getFileStatus(testDir); |
||||
|
assertTrue("Path should be a directory", status.isDirectory()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testCreateAndReadFile() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/testfile.txt"); |
||||
|
String testContent = "Hello, SeaweedFS!"; |
||||
|
|
||||
|
// Create and write to file |
||||
|
FSDataOutputStream out = fs.create(testFile, FsPermission.getDefault(), |
||||
|
false, 4096, (short) 1, 4 * 1024 * 1024, null); |
||||
|
assertNotNull("Output stream should not be null", out); |
||||
|
out.write(testContent.getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
// Verify file exists |
||||
|
assertTrue("File should exist", fs.exists(testFile)); |
||||
|
|
||||
|
// Read and verify content |
||||
|
FSDataInputStream in = fs.open(testFile, 4096); |
||||
|
assertNotNull("Input stream should not be null", in); |
||||
|
byte[] buffer = new byte[testContent.length()]; |
||||
|
int bytesRead = in.read(buffer); |
||||
|
in.close(); |
||||
|
|
||||
|
assertEquals("Should read all bytes", testContent.length(), bytesRead); |
||||
|
assertEquals("Content should match", testContent, new String(buffer)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testFileStatus() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/statustest.txt"); |
||||
|
String content = "test content"; |
||||
|
|
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write(content.getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
FileStatus status = fs.getFileStatus(testFile); |
||||
|
assertNotNull("FileStatus should not be null", status); |
||||
|
assertFalse("Should not be a directory", status.isDirectory()); |
||||
|
assertTrue("Should be a file", status.isFile()); |
||||
|
assertEquals("File length should match", content.length(), status.getLen()); |
||||
|
assertNotNull("Path should not be null", status.getPath()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testListStatus() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testDir = new Path(TEST_ROOT + "/listtest"); |
||||
|
fs.mkdirs(testDir); |
||||
|
|
||||
|
// Create multiple files |
||||
|
for (int i = 0; i < 3; i++) { |
||||
|
Path file = new Path(testDir, "file" + i + ".txt"); |
||||
|
FSDataOutputStream out = fs.create(file); |
||||
|
out.write(("content" + i).getBytes()); |
||||
|
out.close(); |
||||
|
} |
||||
|
|
||||
|
FileStatus[] statuses = fs.listStatus(testDir); |
||||
|
assertNotNull("List should not be null", statuses); |
||||
|
assertEquals("Should have 3 files", 3, statuses.length); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testRename() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path srcFile = new Path(TEST_ROOT + "/source.txt"); |
||||
|
Path dstFile = new Path(TEST_ROOT + "/destination.txt"); |
||||
|
String content = "rename test"; |
||||
|
|
||||
|
// Create source file |
||||
|
FSDataOutputStream out = fs.create(srcFile); |
||||
|
out.write(content.getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
assertTrue("Source file should exist", fs.exists(srcFile)); |
||||
|
|
||||
|
// Rename |
||||
|
assertTrue("Rename should succeed", fs.rename(srcFile, dstFile)); |
||||
|
|
||||
|
// Verify |
||||
|
assertFalse("Source file should not exist", fs.exists(srcFile)); |
||||
|
assertTrue("Destination file should exist", fs.exists(dstFile)); |
||||
|
|
||||
|
// Verify content preserved |
||||
|
FSDataInputStream in = fs.open(dstFile); |
||||
|
byte[] buffer = new byte[content.length()]; |
||||
|
in.read(buffer); |
||||
|
in.close(); |
||||
|
assertEquals("Content should be preserved", content, new String(buffer)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDelete() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/deletetest.txt"); |
||||
|
|
||||
|
// Create file |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write("delete me".getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
assertTrue("File should exist before delete", fs.exists(testFile)); |
||||
|
|
||||
|
// Delete |
||||
|
assertTrue("Delete should succeed", fs.delete(testFile, false)); |
||||
|
assertFalse("File should not exist after delete", fs.exists(testFile)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testDeleteDirectory() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testDir = new Path(TEST_ROOT + "/deletedir"); |
||||
|
Path testFile = new Path(testDir, "file.txt"); |
||||
|
|
||||
|
// Create directory with file |
||||
|
fs.mkdirs(testDir); |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write("content".getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
assertTrue("Directory should exist", fs.exists(testDir)); |
||||
|
assertTrue("File should exist", fs.exists(testFile)); |
||||
|
|
||||
|
// Recursive delete |
||||
|
assertTrue("Recursive delete should succeed", fs.delete(testDir, true)); |
||||
|
assertFalse("Directory should not exist after delete", fs.exists(testDir)); |
||||
|
assertFalse("File should not exist after delete", fs.exists(testFile)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testAppend() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/appendtest.txt"); |
||||
|
String initialContent = "initial"; |
||||
|
String appendContent = " appended"; |
||||
|
|
||||
|
// Create initial file |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write(initialContent.getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
// Append |
||||
|
FSDataOutputStream appendOut = fs.append(testFile, 4096, null); |
||||
|
assertNotNull("Append stream should not be null", appendOut); |
||||
|
appendOut.write(appendContent.getBytes()); |
||||
|
appendOut.close(); |
||||
|
|
||||
|
// Verify combined content |
||||
|
FSDataInputStream in = fs.open(testFile); |
||||
|
byte[] buffer = new byte[initialContent.length() + appendContent.length()]; |
||||
|
int bytesRead = in.read(buffer); |
||||
|
in.close(); |
||||
|
|
||||
|
String expected = initialContent + appendContent; |
||||
|
assertEquals("Should read all bytes", expected.length(), bytesRead); |
||||
|
assertEquals("Content should match", expected, new String(buffer)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testSetWorkingDirectory() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path originalWd = fs.getWorkingDirectory(); |
||||
|
assertEquals("Original working directory should be /", "/", originalWd.toUri().getPath()); |
||||
|
|
||||
|
Path newWd = new Path(TEST_ROOT); |
||||
|
fs.mkdirs(newWd); |
||||
|
fs.setWorkingDirectory(newWd); |
||||
|
|
||||
|
Path currentWd = fs.getWorkingDirectory(); |
||||
|
assertTrue("Working directory should be updated", |
||||
|
currentWd.toUri().getPath().contains(TEST_ROOT)); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testSetPermission() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/permtest.txt"); |
||||
|
|
||||
|
// Create file |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write("permission test".getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
// Set permission |
||||
|
FsPermission newPerm = new FsPermission((short) 0644); |
||||
|
fs.setPermission(testFile, newPerm); |
||||
|
|
||||
|
FileStatus status = fs.getFileStatus(testFile); |
||||
|
assertNotNull("Permission should not be null", status.getPermission()); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testSetOwner() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path testFile = new Path(TEST_ROOT + "/ownertest.txt"); |
||||
|
|
||||
|
// Create file |
||||
|
FSDataOutputStream out = fs.create(testFile); |
||||
|
out.write("owner test".getBytes()); |
||||
|
out.close(); |
||||
|
|
||||
|
// Set owner - this may not fail even if not fully implemented |
||||
|
fs.setOwner(testFile, "testuser", "testgroup"); |
||||
|
|
||||
|
// Just verify the call doesn't throw an exception |
||||
|
FileStatus status = fs.getFileStatus(testFile); |
||||
|
assertNotNull("FileStatus should not be null", status); |
||||
|
} |
||||
|
|
||||
|
@Test |
||||
|
public void testRenameToExistingDirectory() throws Exception { |
||||
|
if (!TESTS_ENABLED) { |
||||
|
System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
Path srcFile = new Path(TEST_ROOT + "/movefile.txt"); |
||||
|
Path dstDir = new Path(TEST_ROOT + "/movedir"); |
||||
|
|
||||
|
// Create source file and destination directory |
||||
|
FSDataOutputStream out = fs.create(srcFile); |
||||
|
out.write("move test".getBytes()); |
||||
|
out.close(); |
||||
|
fs.mkdirs(dstDir); |
||||
|
|
||||
|
// Rename file to existing directory (should move file into directory) |
||||
|
assertTrue("Rename to directory should succeed", fs.rename(srcFile, dstDir)); |
||||
|
|
||||
|
// File should be moved into the directory |
||||
|
Path expectedLocation = new Path(dstDir, srcFile.getName()); |
||||
|
assertTrue("File should exist in destination directory", fs.exists(expectedLocation)); |
||||
|
assertFalse("Source file should not exist", fs.exists(srcFile)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue