From c14e513964ff708b7ace352b7e86198b3ebe6827 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Nov 2025 21:22:18 -0800 Subject: [PATCH] chore(deps): bump org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0 in /other/java/hdfs3 (#7512) * chore(deps): bump org.apache.hadoop:hadoop-common in /other/java/hdfs3 Bumps org.apache.hadoop:hadoop-common from 3.2.4 to 3.4.0. --- updated-dependencies: - dependency-name: org.apache.hadoop:hadoop-common dependency-version: 3.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add java client unit tests * Update dependency-reduced-pom.xml * add java integration tests * fix * fix buffer --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: chrislu --- .github/workflows/java_integration_tests.yml | 170 +++++++ .github/workflows/java_unit_tests.yml | 64 +++ .../client/FilerClientIntegrationTest.java | 323 ++++++++++++++ .../client/SeaweedStreamIntegrationTest.java | 417 ++++++++++++++++++ other/java/hdfs2/README.md | 190 ++++++++ other/java/hdfs2/pom.xml | 19 + .../hdfs/SeaweedFileSystemConfigTest.java | 90 ++++ .../seaweed/hdfs/SeaweedFileSystemTest.java | 379 ++++++++++++++++ other/java/hdfs3/README.md | 190 ++++++++ other/java/hdfs3/dependency-reduced-pom.xml | 263 ++++++++++- other/java/hdfs3/pom.xml | 21 +- .../hdfs/SeaweedFileSystemConfigTest.java | 90 ++++ .../seaweed/hdfs/SeaweedFileSystemTest.java | 379 ++++++++++++++++ 13 files changed, 2585 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/java_integration_tests.yml create mode 100644 .github/workflows/java_unit_tests.yml create mode 100644 other/java/client/src/test/java/seaweedfs/client/FilerClientIntegrationTest.java create mode 100644 other/java/client/src/test/java/seaweedfs/client/SeaweedStreamIntegrationTest.java create mode 100644 other/java/hdfs2/README.md create mode 100644 other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java create mode 100644 other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java create mode 100644 other/java/hdfs3/README.md create mode 100644 other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java create mode 100644 other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java diff --git a/.github/workflows/java_integration_tests.yml b/.github/workflows/java_integration_tests.yml new file mode 100644 index 000000000..9b86d8e69 --- /dev/null +++ b/.github/workflows/java_integration_tests.yml @@ -0,0 +1,170 @@ +name: Java Client Integration Tests + +on: + push: + branches: [ master ] + paths: + - 'other/java/**' + - 'weed/**' + - '.github/workflows/java_integration_tests.yml' + pull_request: + branches: [ master ] + paths: + - 'other/java/**' + - 'weed/**' + - '.github/workflows/java_integration_tests.yml' + +jobs: + test: + name: Java Integration Tests + runs-on: ubuntu-latest + + strategy: + matrix: + java: ['11', '17'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + id: go + + - name: Set up Java + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + + - name: Build SeaweedFS + run: | + cd weed + go install -buildvcs=false + weed version + + - name: Start SeaweedFS Server + run: | + # Create clean data directory + export WEED_DATA_DIR="/tmp/seaweedfs-java-tests-$(date +%s)" + mkdir -p "$WEED_DATA_DIR" + + # Start SeaweedFS with optimized settings for CI + weed server -dir="$WEED_DATA_DIR" \ + -master.raftHashicorp \ + -master.electionTimeout=1s \ + -master.volumeSizeLimitMB=100 \ + -volume.max=100 \ + -volume.preStopSeconds=1 \ + -master.peers=none \ + -filer -filer.maxMB=64 \ + -master.port=9333 \ + -volume.port=8080 \ + -filer.port=8888 \ + -metricsPort=9324 > seaweedfs.log 2>&1 & + + SERVER_PID=$! + echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV + echo "WEED_DATA_DIR=$WEED_DATA_DIR" >> $GITHUB_ENV + echo "SeaweedFS server started with PID: $SERVER_PID" + + - name: Wait for SeaweedFS Components + run: | + echo "Waiting for SeaweedFS components to start..." + + # Wait for master + for i in {1..30}; do + if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then + echo "✓ Master server is ready" + break + fi + echo "Waiting for master server... ($i/30)" + sleep 2 + done + + # Wait for volume + for i in {1..30}; do + if curl -s http://localhost:8080/status > /dev/null 2>&1; then + echo "✓ Volume server is ready" + break + fi + echo "Waiting for volume server... ($i/30)" + sleep 2 + done + + # Wait for filer + for i in {1..30}; do + if curl -s http://localhost:8888/ > /dev/null 2>&1; then + echo "✓ Filer is ready" + break + fi + echo "Waiting for filer... ($i/30)" + sleep 2 + done + + echo "✓ All SeaweedFS components are ready!" + + # Display cluster status + echo "Cluster status:" + curl -s http://localhost:9333/cluster/status | head -20 + + - name: Build and Install SeaweedFS Client + working-directory: other/java/client + run: | + mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Run Client Unit Tests + working-directory: other/java/client + run: | + mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest + + - name: Run Client Integration Tests + working-directory: other/java/client + env: + SEAWEEDFS_TEST_ENABLED: true + run: | + mvn test -Dtest=*IntegrationTest + + - name: Run HDFS2 Configuration Tests + working-directory: other/java/hdfs2 + run: | + mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Run HDFS3 Configuration Tests + working-directory: other/java/hdfs3 + run: | + mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Display logs on failure + if: failure() + run: | + echo "=== SeaweedFS Server Log ===" + tail -100 seaweedfs.log || echo "No server log" + echo "" + echo "=== Cluster Status ===" + curl -s http://localhost:9333/cluster/status || echo "Cannot reach cluster" + echo "" + echo "=== Process Status ===" + ps aux | grep weed || echo "No weed processes" + + - name: Cleanup + if: always() + run: | + # Stop server using stored PID + if [ -n "$SERVER_PID" ]; then + echo "Stopping SeaweedFS server (PID: $SERVER_PID)" + kill -9 $SERVER_PID 2>/dev/null || true + fi + + # Fallback: kill any remaining weed processes + pkill -f "weed server" || true + + # Clean up data directory + if [ -n "$WEED_DATA_DIR" ]; then + echo "Cleaning up data directory: $WEED_DATA_DIR" + rm -rf "$WEED_DATA_DIR" || true + fi + diff --git a/.github/workflows/java_unit_tests.yml b/.github/workflows/java_unit_tests.yml new file mode 100644 index 000000000..e79499b04 --- /dev/null +++ b/.github/workflows/java_unit_tests.yml @@ -0,0 +1,64 @@ +name: Java Client Unit Tests + +on: + push: + branches: [ master ] + paths: + - 'other/java/**' + - '.github/workflows/java_unit_tests.yml' + pull_request: + branches: [ master ] + paths: + - 'other/java/**' + - '.github/workflows/java_unit_tests.yml' + +jobs: + test: + name: Java Unit Tests + runs-on: ubuntu-latest + + strategy: + matrix: + java: ['8', '11', '17', '21'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'maven' + + - name: Build and Install SeaweedFS Client + working-directory: other/java/client + run: | + mvn clean install -DskipTests -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Run Client Unit Tests + working-directory: other/java/client + run: | + mvn test -Dtest=SeaweedReadTest,SeaweedCipherTest + + - name: Run HDFS2 Configuration Tests + working-directory: other/java/hdfs2 + run: | + mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Run HDFS3 Configuration Tests + working-directory: other/java/hdfs3 + run: | + mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true + + - name: Upload Test Reports + if: always() + uses: actions/upload-artifact@v5 + with: + name: test-reports-java-${{ matrix.java }} + path: | + other/java/client/target/surefire-reports/ + other/java/hdfs2/target/surefire-reports/ + other/java/hdfs3/target/surefire-reports/ + diff --git a/other/java/client/src/test/java/seaweedfs/client/FilerClientIntegrationTest.java b/other/java/client/src/test/java/seaweedfs/client/FilerClientIntegrationTest.java new file mode 100644 index 000000000..1015653bd --- /dev/null +++ b/other/java/client/src/test/java/seaweedfs/client/FilerClientIntegrationTest.java @@ -0,0 +1,323 @@ +package seaweedfs.client; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Integration tests for FilerClient. + * + * These tests verify FilerClient operations against a running SeaweedFS filer + * instance. + * + * Prerequisites: + * - SeaweedFS master, volume server, and filer must be running + * - Default ports: filer HTTP 8888, filer gRPC 18888 + * + * To run tests: + * export SEAWEEDFS_TEST_ENABLED=true + * mvn test -Dtest=FilerClientIntegrationTest + */ +public class FilerClientIntegrationTest { + + private FilerClient filerClient; + private static final String TEST_ROOT = "/test-client-integration"; + private static final boolean TESTS_ENABLED = "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); + + @Before + public void setUp() throws Exception { + if (!TESTS_ENABLED) { + return; + } + + filerClient = new FilerClient("localhost", 18888); + + // Clean up any existing test directory + if (filerClient.exists(TEST_ROOT)) { + filerClient.rm(TEST_ROOT, true, true); + } + + // Create test root directory + filerClient.mkdirs(TEST_ROOT, 0755); + } + + @After + public void tearDown() throws Exception { + if (!TESTS_ENABLED || filerClient == null) { + return; + } + + try { + // Clean up test directory + if (filerClient.exists(TEST_ROOT)) { + filerClient.rm(TEST_ROOT, true, true); + } + } finally { + filerClient.shutdown(); + } + } + + @Test + public void testMkdirs() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testDir = TEST_ROOT + "/testdir"; + boolean success = filerClient.mkdirs(testDir, 0755); + + assertTrue("Directory creation should succeed", success); + assertTrue("Directory should exist", filerClient.exists(testDir)); + } + + @Test + public void testTouch() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testFile = TEST_ROOT + "/testfile.txt"; + boolean success = filerClient.touch(testFile, 0644); + + assertTrue("Touch should succeed", success); + assertTrue("File should exist", filerClient.exists(testFile)); + } + + @Test + public void testExists() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + assertTrue("Root should exist", filerClient.exists("/")); + assertTrue("Test root should exist", filerClient.exists(TEST_ROOT)); + assertFalse("Non-existent path should not exist", + filerClient.exists(TEST_ROOT + "/nonexistent")); + } + + @Test + public void testListEntries() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + // Create some test files and directories + filerClient.touch(TEST_ROOT + "/file1.txt", 0644); + filerClient.touch(TEST_ROOT + "/file2.txt", 0644); + filerClient.mkdirs(TEST_ROOT + "/subdir", 0755); + + List entries = filerClient.listEntries(TEST_ROOT); + + assertNotNull("Entries should not be null", entries); + assertEquals("Should have 3 entries", 3, entries.size()); + } + + @Test + public void testListEntriesWithPrefix() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + // Create test files + filerClient.touch(TEST_ROOT + "/test1.txt", 0644); + filerClient.touch(TEST_ROOT + "/test2.txt", 0644); + filerClient.touch(TEST_ROOT + "/other.txt", 0644); + + List entries = filerClient.listEntries(TEST_ROOT, "test", "", 100, false); + + assertNotNull("Entries should not be null", entries); + assertEquals("Should have 2 entries starting with 'test'", 2, entries.size()); + } + + @Test + public void testDeleteFile() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testFile = TEST_ROOT + "/deleteme.txt"; + filerClient.touch(testFile, 0644); + + assertTrue("File should exist before delete", filerClient.exists(testFile)); + + boolean success = filerClient.rm(testFile, false, true); + + assertTrue("Delete should succeed", success); + assertFalse("File should not exist after delete", filerClient.exists(testFile)); + } + + @Test + public void testDeleteDirectoryRecursive() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testDir = TEST_ROOT + "/deletedir"; + filerClient.mkdirs(testDir, 0755); + filerClient.touch(testDir + "/file.txt", 0644); + + assertTrue("Directory should exist", filerClient.exists(testDir)); + assertTrue("File should exist", filerClient.exists(testDir + "/file.txt")); + + boolean success = filerClient.rm(testDir, true, true); + + assertTrue("Delete should succeed", success); + assertFalse("Directory should not exist after delete", filerClient.exists(testDir)); + } + + @Test + public void testRename() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String srcFile = TEST_ROOT + "/source.txt"; + String dstFile = TEST_ROOT + "/destination.txt"; + + filerClient.touch(srcFile, 0644); + assertTrue("Source file should exist", filerClient.exists(srcFile)); + + boolean success = filerClient.mv(srcFile, dstFile); + + assertTrue("Rename should succeed", success); + assertFalse("Source file should not exist after rename", filerClient.exists(srcFile)); + assertTrue("Destination file should exist after rename", filerClient.exists(dstFile)); + } + + @Test + public void testGetEntry() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testFile = TEST_ROOT + "/getentry.txt"; + filerClient.touch(testFile, 0644); + + FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "getentry.txt"); + + assertNotNull("Entry should not be null", entry); + assertEquals("Entry name should match", "getentry.txt", entry.getName()); + assertFalse("Entry should not be a directory", entry.getIsDirectory()); + } + + @Test + public void testGetEntryForDirectory() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testDir = TEST_ROOT + "/testsubdir"; + filerClient.mkdirs(testDir, 0755); + + FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "testsubdir"); + + assertNotNull("Entry should not be null", entry); + assertEquals("Entry name should match", "testsubdir", entry.getName()); + assertTrue("Entry should be a directory", entry.getIsDirectory()); + } + + @Test + public void testCreateAndListNestedDirectories() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String nestedPath = TEST_ROOT + "/level1/level2/level3"; + boolean success = filerClient.mkdirs(nestedPath, 0755); + + assertTrue("Nested directory creation should succeed", success); + assertTrue("Nested directory should exist", filerClient.exists(nestedPath)); + + // Verify each level exists + assertTrue("Level 1 should exist", filerClient.exists(TEST_ROOT + "/level1")); + assertTrue("Level 2 should exist", filerClient.exists(TEST_ROOT + "/level1/level2")); + assertTrue("Level 3 should exist", filerClient.exists(nestedPath)); + } + + @Test + public void testMultipleFilesInDirectory() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testDir = TEST_ROOT + "/multifiles"; + filerClient.mkdirs(testDir, 0755); + + // Create 10 files + for (int i = 0; i < 10; i++) { + filerClient.touch(testDir + "/file" + i + ".txt", 0644); + } + + List entries = filerClient.listEntries(testDir); + + assertNotNull("Entries should not be null", entries); + assertEquals("Should have 10 files", 10, entries.size()); + } + + @Test + public void testRenameDirectory() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String srcDir = TEST_ROOT + "/sourcedir"; + String dstDir = TEST_ROOT + "/destdir"; + + filerClient.mkdirs(srcDir, 0755); + filerClient.touch(srcDir + "/file.txt", 0644); + + boolean success = filerClient.mv(srcDir, dstDir); + + assertTrue("Directory rename should succeed", success); + assertFalse("Source directory should not exist", filerClient.exists(srcDir)); + assertTrue("Destination directory should exist", filerClient.exists(dstDir)); + assertTrue("File should exist in destination", filerClient.exists(dstDir + "/file.txt")); + } + + @Test + public void testLookupNonExistentEntry() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + FilerProto.Entry entry = filerClient.lookupEntry(TEST_ROOT, "nonexistent.txt"); + + assertNull("Entry for non-existent file should be null", entry); + } + + @Test + public void testEmptyDirectory() { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String emptyDir = TEST_ROOT + "/emptydir"; + filerClient.mkdirs(emptyDir, 0755); + + List entries = filerClient.listEntries(emptyDir); + + assertNotNull("Entries should not be null", entries); + assertTrue("Empty directory should have no entries", entries.isEmpty()); + } +} diff --git a/other/java/client/src/test/java/seaweedfs/client/SeaweedStreamIntegrationTest.java b/other/java/client/src/test/java/seaweedfs/client/SeaweedStreamIntegrationTest.java new file mode 100644 index 000000000..f384e059f --- /dev/null +++ b/other/java/client/src/test/java/seaweedfs/client/SeaweedStreamIntegrationTest.java @@ -0,0 +1,417 @@ +package seaweedfs.client; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Random; + +import static org.junit.Assert.*; + +/** + * Integration tests for SeaweedInputStream and SeaweedOutputStream. + * + * These tests verify stream operations against a running SeaweedFS instance. + * + * Prerequisites: + * - SeaweedFS master, volume server, and filer must be running + * - Default ports: filer HTTP 8888, filer gRPC 18888 + * + * To run tests: + * export SEAWEEDFS_TEST_ENABLED=true + * mvn test -Dtest=SeaweedStreamIntegrationTest + */ +public class SeaweedStreamIntegrationTest { + + private FilerClient filerClient; + private static final String TEST_ROOT = "/test-stream-integration"; + private static final boolean TESTS_ENABLED = + "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); + + @Before + public void setUp() throws Exception { + if (!TESTS_ENABLED) { + return; + } + + filerClient = new FilerClient("localhost", 18888); + + // Clean up any existing test directory + if (filerClient.exists(TEST_ROOT)) { + filerClient.rm(TEST_ROOT, true, true); + } + + // Create test root directory + filerClient.mkdirs(TEST_ROOT, 0755); + } + + @After + public void tearDown() throws Exception { + if (!TESTS_ENABLED || filerClient == null) { + return; + } + + try { + // Clean up test directory + if (filerClient.exists(TEST_ROOT)) { + filerClient.rm(TEST_ROOT, true, true); + } + } finally { + filerClient.shutdown(); + } + } + + @Test + public void testWriteAndReadSmallFile() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/small.txt"; + String testContent = "Hello, SeaweedFS!"; + + // Write file + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + // Verify file exists + assertTrue("File should exist", filerClient.exists(testPath)); + + // Read file + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + assertNotNull("Entry should not be null", entry); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + byte[] buffer = new byte[testContent.length()]; + int bytesRead = inputStream.read(buffer); + inputStream.close(); + + assertEquals("Should read all bytes", testContent.length(), bytesRead); + assertEquals("Content should match", testContent, new String(buffer, StandardCharsets.UTF_8)); + } + + @Test + public void testWriteAndReadLargeFile() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/large.bin"; + int fileSize = 10 * 1024 * 1024; // 10 MB + + // Generate random data + byte[] originalData = new byte[fileSize]; + new Random(42).nextBytes(originalData); // Use seed for reproducibility + + // Write file + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(originalData); + outputStream.close(); + + // Verify file exists + assertTrue("File should exist", filerClient.exists(testPath)); + + // Read file + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + assertNotNull("Entry should not be null", entry); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + + // Read file in chunks to handle large files properly + byte[] readData = new byte[fileSize]; + int totalRead = 0; + int bytesRead; + byte[] buffer = new byte[8192]; // Read in 8KB chunks + + while ((bytesRead = inputStream.read(buffer)) > 0) { + System.arraycopy(buffer, 0, readData, totalRead, bytesRead); + totalRead += bytesRead; + } + inputStream.close(); + + assertEquals("Should read all bytes", fileSize, totalRead); + assertArrayEquals("Content should match", originalData, readData); + } + + @Test + public void testWriteInChunks() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/chunked.txt"; + String[] chunks = {"First chunk. ", "Second chunk. ", "Third chunk."}; + + // Write file in chunks + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + for (String chunk : chunks) { + outputStream.write(chunk.getBytes(StandardCharsets.UTF_8)); + } + outputStream.close(); + + // Read and verify + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + byte[] buffer = new byte[1024]; + int bytesRead = inputStream.read(buffer); + inputStream.close(); + + String expected = String.join("", chunks); + String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + + assertEquals("Content should match", expected, actual); + } + + @Test + public void testReadWithOffset() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/offset.txt"; + String testContent = "0123456789ABCDEFGHIJ"; + + // Write file + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + // Read with offset + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + inputStream.seek(10); // Skip first 10 bytes + + byte[] buffer = new byte[10]; + int bytesRead = inputStream.read(buffer); + inputStream.close(); + + assertEquals("Should read 10 bytes", 10, bytesRead); + assertEquals("Should read from offset", "ABCDEFGHIJ", + new String(buffer, StandardCharsets.UTF_8)); + } + + @Test + public void testReadPartial() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/partial.txt"; + String testContent = "The quick brown fox jumps over the lazy dog"; + + // Write file + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + // Read partial + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + + // Read only "quick brown" + inputStream.seek(4); + byte[] buffer = new byte[11]; + int bytesRead = inputStream.read(buffer); + inputStream.close(); + + assertEquals("Should read 11 bytes", 11, bytesRead); + assertEquals("Should read partial content", "quick brown", + new String(buffer, StandardCharsets.UTF_8)); + } + + @Test + public void testEmptyFile() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/empty.txt"; + + // Write empty file + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.close(); + + // Verify file exists + assertTrue("File should exist", filerClient.exists(testPath)); + + // Read empty file + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + assertNotNull("Entry should not be null", entry); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + byte[] buffer = new byte[100]; + int bytesRead = inputStream.read(buffer); + inputStream.close(); + + assertEquals("Should read 0 bytes from empty file", -1, bytesRead); + } + + @Test + public void testOverwriteFile() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/overwrite.txt"; + String originalContent = "Original content"; + String newContent = "New content that overwrites the original"; + + // Write original file + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(originalContent.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + // Overwrite file + outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(newContent.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + // Read and verify + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + byte[] buffer = new byte[1024]; + int bytesRead = inputStream.read(buffer); + inputStream.close(); + + String actual = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8); + assertEquals("Should have new content", newContent, actual); + } + + @Test + public void testMultipleReads() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/multireads.txt"; + String testContent = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + // Write file + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + // Read in multiple small chunks + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + + StringBuilder result = new StringBuilder(); + byte[] buffer = new byte[5]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) > 0) { + result.append(new String(buffer, 0, bytesRead, StandardCharsets.UTF_8)); + } + inputStream.close(); + + assertEquals("Should read entire content", testContent, result.toString()); + } + + @Test + public void testBinaryData() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/binary.bin"; + byte[] binaryData = new byte[256]; + for (int i = 0; i < 256; i++) { + binaryData[i] = (byte) i; + } + + // Write binary file + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(binaryData); + outputStream.close(); + + // Read and verify + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + byte[] readData = new byte[256]; + int bytesRead = inputStream.read(readData); + inputStream.close(); + + assertEquals("Should read all bytes", 256, bytesRead); + assertArrayEquals("Binary data should match", binaryData, readData); + } + + @Test + public void testFlush() throws IOException { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + String testPath = TEST_ROOT + "/flush.txt"; + String testContent = "Content to flush"; + + // Write file with flush + SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, testPath); + outputStream.write(testContent.getBytes(StandardCharsets.UTF_8)); + outputStream.flush(); // Explicitly flush + outputStream.close(); + + // Verify file was written + assertTrue("File should exist after flush", filerClient.exists(testPath)); + + // Read and verify + FilerProto.Entry entry = filerClient.lookupEntry( + SeaweedOutputStream.getParentDirectory(testPath), + SeaweedOutputStream.getFileName(testPath) + ); + + SeaweedInputStream inputStream = new SeaweedInputStream(filerClient, testPath, entry); + byte[] buffer = new byte[testContent.length()]; + int bytesRead = inputStream.read(buffer); + inputStream.close(); + + assertEquals("Content should match", testContent, + new String(buffer, 0, bytesRead, StandardCharsets.UTF_8)); + } +} + diff --git a/other/java/hdfs2/README.md b/other/java/hdfs2/README.md new file mode 100644 index 000000000..e98b06506 --- /dev/null +++ b/other/java/hdfs2/README.md @@ -0,0 +1,190 @@ +# SeaweedFS Hadoop2 Client + +Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 2.x/3.x. + +## Building + +```bash +mvn clean install +``` + +## Testing + +This project includes two types of tests: + +### 1. Configuration Tests (No SeaweedFS Required) + +These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance: + +```bash +mvn test -Dtest=SeaweedFileSystemConfigTest +``` + +### 2. Integration Tests (Requires SeaweedFS) + +These tests verify actual FileSystem operations against a running SeaweedFS instance. + +#### Prerequisites + +1. Start SeaweedFS with default ports: + ```bash + # Terminal 1: Start master + weed master + + # Terminal 2: Start volume server + weed volume -mserver=localhost:9333 + + # Terminal 3: Start filer + weed filer -master=localhost:9333 + ``` + +2. Verify services are running: + - Master: http://localhost:9333 + - Filer HTTP: http://localhost:8888 + - Filer gRPC: localhost:18888 + +#### Running Integration Tests + +```bash +# Enable integration tests +export SEAWEEDFS_TEST_ENABLED=true + +# Run all tests +mvn test + +# Run specific test +mvn test -Dtest=SeaweedFileSystemTest +``` + +### Test Configuration + +Integration tests can be configured via environment variables or system properties: + +- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false) +- Tests use these default connection settings: + - Filer Host: localhost + - Filer HTTP Port: 8888 + - Filer gRPC Port: 18888 + +### Running Tests with Custom Configuration + +To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration: + +```java +conf.set("fs.seaweed.filer.host", "your-host"); +conf.setInt("fs.seaweed.filer.port", 8888); +conf.setInt("fs.seaweed.filer.port.grpc", 18888); +``` + +## Test Coverage + +The test suite covers: + +- **Configuration & Initialization** + - URI parsing and configuration + - Default values + - Configuration overrides + - Working directory management + +- **File Operations** + - Create files + - Read files + - Write files + - Append to files + - Delete files + +- **Directory Operations** + - Create directories + - List directory contents + - Delete directories (recursive and non-recursive) + +- **Metadata Operations** + - Get file status + - Set permissions + - Set owner/group + - Rename files and directories + +## Usage in Hadoop + +1. Copy the built JAR to your Hadoop classpath: + ```bash + cp target/seaweedfs-hadoop2-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/ + ``` + +2. Configure `core-site.xml`: + ```xml + + + fs.seaweedfs.impl + seaweed.hdfs.SeaweedFileSystem + + + fs.seaweed.filer.host + localhost + + + fs.seaweed.filer.port + 8888 + + + fs.seaweed.filer.port.grpc + 18888 + + + ``` + +3. Use SeaweedFS with Hadoop commands: + ```bash + hadoop fs -ls seaweedfs://localhost:8888/ + hadoop fs -mkdir seaweedfs://localhost:8888/test + hadoop fs -put local.txt seaweedfs://localhost:8888/test/ + ``` + +## Continuous Integration + +For CI environments, tests can be run in two modes: + +1. **Configuration Tests Only** (default, no SeaweedFS required): + ```bash + mvn test -Dtest=SeaweedFileSystemConfigTest + ``` + +2. **Full Integration Tests** (requires SeaweedFS): + ```bash + # Start SeaweedFS in CI environment + # Then run: + export SEAWEEDFS_TEST_ENABLED=true + mvn test + ``` + +## Troubleshooting + +### Tests are skipped + +If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set": +```bash +export SEAWEEDFS_TEST_ENABLED=true +``` + +### Connection refused errors + +Ensure SeaweedFS is running and accessible: +```bash +curl http://localhost:8888/ +``` + +### gRPC errors + +Verify the gRPC port is accessible: +```bash +# Should show the port is listening +netstat -an | grep 18888 +``` + +## Contributing + +When adding new features, please include: +1. Configuration tests (no SeaweedFS required) +2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard) +3. Documentation updates + diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index 949b88c35..7b4c2507d 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -171,6 +171,25 @@ ${hadoop.version} provided + + junit + junit + 4.13.1 + test + + + org.mockito + mockito-core + 3.12.4 + test + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + test + test-jar + diff --git a/other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java b/other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java new file mode 100644 index 000000000..bcc08b8e2 --- /dev/null +++ b/other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java @@ -0,0 +1,90 @@ +package seaweed.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Unit tests for SeaweedFileSystem configuration that don't require a running SeaweedFS instance. + * + * These tests verify basic properties and constants. + */ +public class SeaweedFileSystemConfigTest { + + private SeaweedFileSystem fs; + private Configuration conf; + + @Before + public void setUp() { + fs = new SeaweedFileSystem(); + conf = new Configuration(); + } + + @Test + public void testScheme() { + assertEquals("seaweedfs", fs.getScheme()); + } + + @Test + public void testConstants() { + // Test that constants are defined correctly + assertEquals("fs.seaweed.filer.host", SeaweedFileSystem.FS_SEAWEED_FILER_HOST); + assertEquals("fs.seaweed.filer.port", SeaweedFileSystem.FS_SEAWEED_FILER_PORT); + assertEquals("fs.seaweed.filer.port.grpc", SeaweedFileSystem.FS_SEAWEED_FILER_PORT_GRPC); + assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); + assertEquals("fs.seaweed.buffer.size", SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE); + assertEquals(4 * 1024 * 1024, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); + assertEquals("fs.seaweed.replication", SeaweedFileSystem.FS_SEAWEED_REPLICATION); + assertEquals("fs.seaweed.volume.server.access", SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS); + assertEquals("fs.seaweed.filer.cn", SeaweedFileSystem.FS_SEAWEED_FILER_CN); + } + + @Test + public void testWorkingDirectoryPathOperations() { + // Test path operations that don't require initialization + Path testPath = new Path("/test/path"); + assertTrue("Path should be absolute", testPath.isAbsolute()); + assertEquals("/test/path", testPath.toUri().getPath()); + + Path childPath = new Path(testPath, "child"); + assertEquals("/test/path/child", childPath.toUri().getPath()); + } + + @Test + public void testConfigurationProperties() { + // Test that configuration can be set and read + conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_HOST, "testhost"); + assertEquals("testhost", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_HOST)); + + conf.setInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 9999); + assertEquals(9999, conf.getInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 0)); + + conf.setInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 8 * 1024 * 1024); + assertEquals(8 * 1024 * 1024, conf.getInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 0)); + + conf.set(SeaweedFileSystem.FS_SEAWEED_REPLICATION, "001"); + assertEquals("001", conf.get(SeaweedFileSystem.FS_SEAWEED_REPLICATION)); + + conf.set(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS, "publicUrl"); + assertEquals("publicUrl", conf.get(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS)); + + conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_CN, "test-cn"); + assertEquals("test-cn", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_CN)); + } + + @Test + public void testDefaultBufferSize() { + // Test default buffer size constant + int expected = 4 * 1024 * 1024; // 4MB + assertEquals(expected, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); + } + + @Test + public void testDefaultPort() { + // Test default port constant + assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); + } +} diff --git a/other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java b/other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java new file mode 100644 index 000000000..ec43b3481 --- /dev/null +++ b/other/java/hdfs2/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java @@ -0,0 +1,379 @@ +package seaweed.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; + +import static org.junit.Assert.*; + +/** + * Unit tests for SeaweedFileSystem. + * + * These tests verify basic FileSystem operations against a SeaweedFS backend. + * Note: These tests require a running SeaweedFS filer instance. + * + * To run tests, ensure SeaweedFS is running with default ports: + * - Filer HTTP: 8888 + * - Filer gRPC: 18888 + * + * Set environment variable SEAWEEDFS_TEST_ENABLED=true to enable these tests. + */ +public class SeaweedFileSystemTest { + + private SeaweedFileSystem fs; + private Configuration conf; + private static final String TEST_ROOT = "/test-hdfs2"; + private static final boolean TESTS_ENABLED = + "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); + + @Before + public void setUp() throws Exception { + if (!TESTS_ENABLED) { + return; + } + + conf = new Configuration(); + conf.set("fs.seaweed.filer.host", "localhost"); + conf.setInt("fs.seaweed.filer.port", 8888); + conf.setInt("fs.seaweed.filer.port.grpc", 18888); + + fs = new SeaweedFileSystem(); + URI uri = new URI("seaweedfs://localhost:8888/"); + fs.initialize(uri, conf); + + // Clean up any existing test directory + Path testPath = new Path(TEST_ROOT); + if (fs.exists(testPath)) { + fs.delete(testPath, true); + } + } + + @After + public void tearDown() throws Exception { + if (!TESTS_ENABLED || fs == null) { + return; + } + + // Clean up test directory + Path testPath = new Path(TEST_ROOT); + if (fs.exists(testPath)) { + fs.delete(testPath, true); + } + + fs.close(); + } + + @Test + public void testInitialization() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + assertNotNull(fs); + assertEquals("seaweedfs", fs.getScheme()); + assertNotNull(fs.getUri()); + assertEquals("/", fs.getWorkingDirectory().toUri().getPath()); + } + + @Test + public void testMkdirs() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testDir = new Path(TEST_ROOT + "/testdir"); + assertTrue("Failed to create directory", fs.mkdirs(testDir)); + assertTrue("Directory should exist", fs.exists(testDir)); + + FileStatus status = fs.getFileStatus(testDir); + assertTrue("Path should be a directory", status.isDirectory()); + } + + @Test + public void testCreateAndReadFile() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/testfile.txt"); + String testContent = "Hello, SeaweedFS!"; + + // Create and write to file + FSDataOutputStream out = fs.create(testFile, FsPermission.getDefault(), + false, 4096, (short) 1, 4 * 1024 * 1024, null); + assertNotNull("Output stream should not be null", out); + out.write(testContent.getBytes()); + out.close(); + + // Verify file exists + assertTrue("File should exist", fs.exists(testFile)); + + // Read and verify content + FSDataInputStream in = fs.open(testFile, 4096); + assertNotNull("Input stream should not be null", in); + byte[] buffer = new byte[testContent.length()]; + int bytesRead = in.read(buffer); + in.close(); + + assertEquals("Should read all bytes", testContent.length(), bytesRead); + assertEquals("Content should match", testContent, new String(buffer)); + } + + @Test + public void testFileStatus() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/statustest.txt"); + String content = "test content"; + + FSDataOutputStream out = fs.create(testFile); + out.write(content.getBytes()); + out.close(); + + FileStatus status = fs.getFileStatus(testFile); + assertNotNull("FileStatus should not be null", status); + assertFalse("Should not be a directory", status.isDirectory()); + assertTrue("Should be a file", status.isFile()); + assertEquals("File length should match", content.length(), status.getLen()); + assertNotNull("Path should not be null", status.getPath()); + } + + @Test + public void testListStatus() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testDir = new Path(TEST_ROOT + "/listtest"); + fs.mkdirs(testDir); + + // Create multiple files + for (int i = 0; i < 3; i++) { + Path file = new Path(testDir, "file" + i + ".txt"); + FSDataOutputStream out = fs.create(file); + out.write(("content" + i).getBytes()); + out.close(); + } + + FileStatus[] statuses = fs.listStatus(testDir); + assertNotNull("List should not be null", statuses); + assertEquals("Should have 3 files", 3, statuses.length); + } + + @Test + public void testRename() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path srcFile = new Path(TEST_ROOT + "/source.txt"); + Path dstFile = new Path(TEST_ROOT + "/destination.txt"); + String content = "rename test"; + + // Create source file + FSDataOutputStream out = fs.create(srcFile); + out.write(content.getBytes()); + out.close(); + + assertTrue("Source file should exist", fs.exists(srcFile)); + + // Rename + assertTrue("Rename should succeed", fs.rename(srcFile, dstFile)); + + // Verify + assertFalse("Source file should not exist", fs.exists(srcFile)); + assertTrue("Destination file should exist", fs.exists(dstFile)); + + // Verify content preserved + FSDataInputStream in = fs.open(dstFile); + byte[] buffer = new byte[content.length()]; + in.read(buffer); + in.close(); + assertEquals("Content should be preserved", content, new String(buffer)); + } + + @Test + public void testDelete() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/deletetest.txt"); + + // Create file + FSDataOutputStream out = fs.create(testFile); + out.write("delete me".getBytes()); + out.close(); + + assertTrue("File should exist before delete", fs.exists(testFile)); + + // Delete + assertTrue("Delete should succeed", fs.delete(testFile, false)); + assertFalse("File should not exist after delete", fs.exists(testFile)); + } + + @Test + public void testDeleteDirectory() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testDir = new Path(TEST_ROOT + "/deletedir"); + Path testFile = new Path(testDir, "file.txt"); + + // Create directory with file + fs.mkdirs(testDir); + FSDataOutputStream out = fs.create(testFile); + out.write("content".getBytes()); + out.close(); + + assertTrue("Directory should exist", fs.exists(testDir)); + assertTrue("File should exist", fs.exists(testFile)); + + // Recursive delete + assertTrue("Recursive delete should succeed", fs.delete(testDir, true)); + assertFalse("Directory should not exist after delete", fs.exists(testDir)); + assertFalse("File should not exist after delete", fs.exists(testFile)); + } + + @Test + public void testAppend() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/appendtest.txt"); + String initialContent = "initial"; + String appendContent = " appended"; + + // Create initial file + FSDataOutputStream out = fs.create(testFile); + out.write(initialContent.getBytes()); + out.close(); + + // Append + FSDataOutputStream appendOut = fs.append(testFile, 4096, null); + assertNotNull("Append stream should not be null", appendOut); + appendOut.write(appendContent.getBytes()); + appendOut.close(); + + // Verify combined content + FSDataInputStream in = fs.open(testFile); + byte[] buffer = new byte[initialContent.length() + appendContent.length()]; + int bytesRead = in.read(buffer); + in.close(); + + String expected = initialContent + appendContent; + assertEquals("Should read all bytes", expected.length(), bytesRead); + assertEquals("Content should match", expected, new String(buffer)); + } + + @Test + public void testSetWorkingDirectory() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path originalWd = fs.getWorkingDirectory(); + assertEquals("Original working directory should be /", "/", originalWd.toUri().getPath()); + + Path newWd = new Path(TEST_ROOT); + fs.mkdirs(newWd); + fs.setWorkingDirectory(newWd); + + Path currentWd = fs.getWorkingDirectory(); + assertTrue("Working directory should be updated", + currentWd.toUri().getPath().contains(TEST_ROOT)); + } + + @Test + public void testSetPermission() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/permtest.txt"); + + // Create file + FSDataOutputStream out = fs.create(testFile); + out.write("permission test".getBytes()); + out.close(); + + // Set permission + FsPermission newPerm = new FsPermission((short) 0644); + fs.setPermission(testFile, newPerm); + + FileStatus status = fs.getFileStatus(testFile); + assertNotNull("Permission should not be null", status.getPermission()); + } + + @Test + public void testSetOwner() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/ownertest.txt"); + + // Create file + FSDataOutputStream out = fs.create(testFile); + out.write("owner test".getBytes()); + out.close(); + + // Set owner - this may not fail even if not fully implemented + fs.setOwner(testFile, "testuser", "testgroup"); + + // Just verify the call doesn't throw an exception + FileStatus status = fs.getFileStatus(testFile); + assertNotNull("FileStatus should not be null", status); + } + + @Test + public void testRenameToExistingDirectory() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path srcFile = new Path(TEST_ROOT + "/movefile.txt"); + Path dstDir = new Path(TEST_ROOT + "/movedir"); + + // Create source file and destination directory + FSDataOutputStream out = fs.create(srcFile); + out.write("move test".getBytes()); + out.close(); + fs.mkdirs(dstDir); + + // Rename file to existing directory (should move file into directory) + assertTrue("Rename to directory should succeed", fs.rename(srcFile, dstDir)); + + // File should be moved into the directory + Path expectedLocation = new Path(dstDir, srcFile.getName()); + assertTrue("File should exist in destination directory", fs.exists(expectedLocation)); + assertFalse("Source file should not exist", fs.exists(srcFile)); + } +} + diff --git a/other/java/hdfs3/README.md b/other/java/hdfs3/README.md new file mode 100644 index 000000000..f1afee264 --- /dev/null +++ b/other/java/hdfs3/README.md @@ -0,0 +1,190 @@ +# SeaweedFS Hadoop3 Client + +Hadoop FileSystem implementation for SeaweedFS, compatible with Hadoop 3.x. + +## Building + +```bash +mvn clean install +``` + +## Testing + +This project includes two types of tests: + +### 1. Configuration Tests (No SeaweedFS Required) + +These tests verify configuration handling and initialization logic without requiring a running SeaweedFS instance: + +```bash +mvn test -Dtest=SeaweedFileSystemConfigTest +``` + +### 2. Integration Tests (Requires SeaweedFS) + +These tests verify actual FileSystem operations against a running SeaweedFS instance. + +#### Prerequisites + +1. Start SeaweedFS with default ports: + ```bash + # Terminal 1: Start master + weed master + + # Terminal 2: Start volume server + weed volume -mserver=localhost:9333 + + # Terminal 3: Start filer + weed filer -master=localhost:9333 + ``` + +2. Verify services are running: + - Master: http://localhost:9333 + - Filer HTTP: http://localhost:8888 + - Filer gRPC: localhost:18888 + +#### Running Integration Tests + +```bash +# Enable integration tests +export SEAWEEDFS_TEST_ENABLED=true + +# Run all tests +mvn test + +# Run specific test +mvn test -Dtest=SeaweedFileSystemTest +``` + +### Test Configuration + +Integration tests can be configured via environment variables or system properties: + +- `SEAWEEDFS_TEST_ENABLED`: Set to `true` to enable integration tests (default: false) +- Tests use these default connection settings: + - Filer Host: localhost + - Filer HTTP Port: 8888 + - Filer gRPC Port: 18888 + +### Running Tests with Custom Configuration + +To test against a different SeaweedFS instance, modify the test code or use Hadoop configuration: + +```java +conf.set("fs.seaweed.filer.host", "your-host"); +conf.setInt("fs.seaweed.filer.port", 8888); +conf.setInt("fs.seaweed.filer.port.grpc", 18888); +``` + +## Test Coverage + +The test suite covers: + +- **Configuration & Initialization** + - URI parsing and configuration + - Default values + - Configuration overrides + - Working directory management + +- **File Operations** + - Create files + - Read files + - Write files + - Append to files + - Delete files + +- **Directory Operations** + - Create directories + - List directory contents + - Delete directories (recursive and non-recursive) + +- **Metadata Operations** + - Get file status + - Set permissions + - Set owner/group + - Rename files and directories + +## Usage in Hadoop + +1. Copy the built JAR to your Hadoop classpath: + ```bash + cp target/seaweedfs-hadoop3-client-*.jar $HADOOP_HOME/share/hadoop/common/lib/ + ``` + +2. Configure `core-site.xml`: + ```xml + + + fs.seaweedfs.impl + seaweed.hdfs.SeaweedFileSystem + + + fs.seaweed.filer.host + localhost + + + fs.seaweed.filer.port + 8888 + + + fs.seaweed.filer.port.grpc + 18888 + + + ``` + +3. Use SeaweedFS with Hadoop commands: + ```bash + hadoop fs -ls seaweedfs://localhost:8888/ + hadoop fs -mkdir seaweedfs://localhost:8888/test + hadoop fs -put local.txt seaweedfs://localhost:8888/test/ + ``` + +## Continuous Integration + +For CI environments, tests can be run in two modes: + +1. **Configuration Tests Only** (default, no SeaweedFS required): + ```bash + mvn test -Dtest=SeaweedFileSystemConfigTest + ``` + +2. **Full Integration Tests** (requires SeaweedFS): + ```bash + # Start SeaweedFS in CI environment + # Then run: + export SEAWEEDFS_TEST_ENABLED=true + mvn test + ``` + +## Troubleshooting + +### Tests are skipped + +If you see "Skipping test - SEAWEEDFS_TEST_ENABLED not set": +```bash +export SEAWEEDFS_TEST_ENABLED=true +``` + +### Connection refused errors + +Ensure SeaweedFS is running and accessible: +```bash +curl http://localhost:8888/ +``` + +### gRPC errors + +Verify the gRPC port is accessible: +```bash +# Should show the port is listening +netstat -an | grep 18888 +``` + +## Contributing + +When adding new features, please include: +1. Configuration tests (no SeaweedFS required) +2. Integration tests (with SEAWEEDFS_TEST_ENABLED guard) +3. Documentation updates + diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index decf55a59..d3c2751a5 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -140,7 +140,7 @@ org.apache.hadoop hadoop-client - 3.2.4 + 3.4.0 provided @@ -172,9 +172,17 @@ org.apache.hadoop hadoop-common - 3.2.4 + 3.4.0 provided + + hadoop-shaded-protobuf_3_21 + org.apache.hadoop.thirdparty + + + hadoop-shaded-guava + org.apache.hadoop.thirdparty + commons-cli commons-cli @@ -200,8 +208,8 @@ javax.servlet - javax.activation-api - javax.activation + jakarta.activation-api + jakarta.activation jetty-server @@ -233,7 +241,11 @@ jersey-json - com.sun.jersey + com.github.pjfanning + + + jettison + org.codehaus.jettison jersey-server @@ -288,19 +300,248 @@ org.apache.curator - htrace-core4 - org.apache.htrace + zookeeper + org.apache.zookeeper + + + netty-handler + io.netty + + + netty-transport-native-epoll + io.netty + + + metrics-core + io.dropwizard.metrics + + + commons-compress + org.apache.commons + + + bcprov-jdk15on + org.bouncycastle + + + kerb-core + org.apache.kerby + + + jackson-databind + com.fasterxml.jackson.core + + + stax2-api + org.codehaus.woodstox + + + woodstox-core + com.fasterxml.woodstox + + + dnsjava + dnsjava + + + snappy-java + org.xerial.snappy + + + hadoop-annotations + org.apache.hadoop + + + + + junit + junit + 4.13.1 + test + + + hamcrest-core + org.hamcrest + + + + + org.mockito + mockito-core + 3.12.4 + test + + + byte-buddy + net.bytebuddy + + + byte-buddy-agent + net.bytebuddy + + + objenesis + org.objenesis + + + + + org.apache.hadoop + hadoop-common + 3.4.0 + test-jar + test + + + hadoop-shaded-protobuf_3_21 + org.apache.hadoop.thirdparty + + + hadoop-shaded-guava + org.apache.hadoop.thirdparty + + + commons-cli + commons-cli + + + commons-math3 + org.apache.commons + + + commons-io + commons-io + + + commons-net + commons-net + + + commons-collections + commons-collections + + + javax.servlet-api + javax.servlet + + + jakarta.activation-api + jakarta.activation + + + jetty-server + org.eclipse.jetty + + + jetty-util + org.eclipse.jetty + + + jetty-servlet + org.eclipse.jetty + + + jetty-webapp + org.eclipse.jetty + + + jsp-api + javax.servlet.jsp + + + jersey-core + com.sun.jersey + + + jersey-servlet + com.sun.jersey + + + jersey-json + com.github.pjfanning + + + jettison + org.codehaus.jettison + + + jersey-server + com.sun.jersey + + + reload4j + ch.qos.reload4j + + + commons-beanutils + commons-beanutils + + + commons-configuration2 + org.apache.commons + + + commons-lang3 + org.apache.commons + + + commons-text + org.apache.commons + + + slf4j-reload4j + org.slf4j + + + avro + org.apache.avro + + + re2j + com.google.re2j + + + hadoop-auth + org.apache.hadoop + + + jsch + com.jcraft + + + curator-client + org.apache.curator + + + curator-recipes + org.apache.curator zookeeper org.apache.zookeeper + + netty-handler + io.netty + + + netty-transport-native-epoll + io.netty + + + metrics-core + io.dropwizard.metrics + commons-compress org.apache.commons - kerb-simplekdc + bcprov-jdk15on + org.bouncycastle + + + kerb-core org.apache.kerby @@ -319,6 +560,10 @@ dnsjava dnsjava + + snappy-java + org.xerial.snappy + hadoop-annotations org.apache.hadoop @@ -328,6 +573,6 @@ 3.80 - 3.2.4 + 3.4.0 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 3faba03be..061d4d700 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -6,7 +6,7 @@ 3.80 - 3.2.4 + 3.4.0 com.seaweedfs @@ -171,6 +171,25 @@ ${hadoop.version} provided + + junit + junit + 4.13.1 + test + + + org.mockito + mockito-core + 3.12.4 + test + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + test + test-jar + diff --git a/other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java b/other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java new file mode 100644 index 000000000..bcc08b8e2 --- /dev/null +++ b/other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemConfigTest.java @@ -0,0 +1,90 @@ +package seaweed.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Unit tests for SeaweedFileSystem configuration that don't require a running SeaweedFS instance. + * + * These tests verify basic properties and constants. + */ +public class SeaweedFileSystemConfigTest { + + private SeaweedFileSystem fs; + private Configuration conf; + + @Before + public void setUp() { + fs = new SeaweedFileSystem(); + conf = new Configuration(); + } + + @Test + public void testScheme() { + assertEquals("seaweedfs", fs.getScheme()); + } + + @Test + public void testConstants() { + // Test that constants are defined correctly + assertEquals("fs.seaweed.filer.host", SeaweedFileSystem.FS_SEAWEED_FILER_HOST); + assertEquals("fs.seaweed.filer.port", SeaweedFileSystem.FS_SEAWEED_FILER_PORT); + assertEquals("fs.seaweed.filer.port.grpc", SeaweedFileSystem.FS_SEAWEED_FILER_PORT_GRPC); + assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); + assertEquals("fs.seaweed.buffer.size", SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE); + assertEquals(4 * 1024 * 1024, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); + assertEquals("fs.seaweed.replication", SeaweedFileSystem.FS_SEAWEED_REPLICATION); + assertEquals("fs.seaweed.volume.server.access", SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS); + assertEquals("fs.seaweed.filer.cn", SeaweedFileSystem.FS_SEAWEED_FILER_CN); + } + + @Test + public void testWorkingDirectoryPathOperations() { + // Test path operations that don't require initialization + Path testPath = new Path("/test/path"); + assertTrue("Path should be absolute", testPath.isAbsolute()); + assertEquals("/test/path", testPath.toUri().getPath()); + + Path childPath = new Path(testPath, "child"); + assertEquals("/test/path/child", childPath.toUri().getPath()); + } + + @Test + public void testConfigurationProperties() { + // Test that configuration can be set and read + conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_HOST, "testhost"); + assertEquals("testhost", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_HOST)); + + conf.setInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 9999); + assertEquals(9999, conf.getInt(SeaweedFileSystem.FS_SEAWEED_FILER_PORT, 0)); + + conf.setInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 8 * 1024 * 1024); + assertEquals(8 * 1024 * 1024, conf.getInt(SeaweedFileSystem.FS_SEAWEED_BUFFER_SIZE, 0)); + + conf.set(SeaweedFileSystem.FS_SEAWEED_REPLICATION, "001"); + assertEquals("001", conf.get(SeaweedFileSystem.FS_SEAWEED_REPLICATION)); + + conf.set(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS, "publicUrl"); + assertEquals("publicUrl", conf.get(SeaweedFileSystem.FS_SEAWEED_VOLUME_SERVER_ACCESS)); + + conf.set(SeaweedFileSystem.FS_SEAWEED_FILER_CN, "test-cn"); + assertEquals("test-cn", conf.get(SeaweedFileSystem.FS_SEAWEED_FILER_CN)); + } + + @Test + public void testDefaultBufferSize() { + // Test default buffer size constant + int expected = 4 * 1024 * 1024; // 4MB + assertEquals(expected, SeaweedFileSystem.FS_SEAWEED_DEFAULT_BUFFER_SIZE); + } + + @Test + public void testDefaultPort() { + // Test default port constant + assertEquals(8888, SeaweedFileSystem.FS_SEAWEED_DEFAULT_PORT); + } +} diff --git a/other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java b/other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java new file mode 100644 index 000000000..4ccb21a56 --- /dev/null +++ b/other/java/hdfs3/src/test/java/seaweed/hdfs/SeaweedFileSystemTest.java @@ -0,0 +1,379 @@ +package seaweed.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; + +import static org.junit.Assert.*; + +/** + * Unit tests for SeaweedFileSystem. + * + * These tests verify basic FileSystem operations against a SeaweedFS backend. + * Note: These tests require a running SeaweedFS filer instance. + * + * To run tests, ensure SeaweedFS is running with default ports: + * - Filer HTTP: 8888 + * - Filer gRPC: 18888 + * + * Set environment variable SEAWEEDFS_TEST_ENABLED=true to enable these tests. + */ +public class SeaweedFileSystemTest { + + private SeaweedFileSystem fs; + private Configuration conf; + private static final String TEST_ROOT = "/test-hdfs3"; + private static final boolean TESTS_ENABLED = + "true".equalsIgnoreCase(System.getenv("SEAWEEDFS_TEST_ENABLED")); + + @Before + public void setUp() throws Exception { + if (!TESTS_ENABLED) { + return; + } + + conf = new Configuration(); + conf.set("fs.seaweed.filer.host", "localhost"); + conf.setInt("fs.seaweed.filer.port", 8888); + conf.setInt("fs.seaweed.filer.port.grpc", 18888); + + fs = new SeaweedFileSystem(); + URI uri = new URI("seaweedfs://localhost:8888/"); + fs.initialize(uri, conf); + + // Clean up any existing test directory + Path testPath = new Path(TEST_ROOT); + if (fs.exists(testPath)) { + fs.delete(testPath, true); + } + } + + @After + public void tearDown() throws Exception { + if (!TESTS_ENABLED || fs == null) { + return; + } + + // Clean up test directory + Path testPath = new Path(TEST_ROOT); + if (fs.exists(testPath)) { + fs.delete(testPath, true); + } + + fs.close(); + } + + @Test + public void testInitialization() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + assertNotNull(fs); + assertEquals("seaweedfs", fs.getScheme()); + assertNotNull(fs.getUri()); + assertEquals("/", fs.getWorkingDirectory().toUri().getPath()); + } + + @Test + public void testMkdirs() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testDir = new Path(TEST_ROOT + "/testdir"); + assertTrue("Failed to create directory", fs.mkdirs(testDir)); + assertTrue("Directory should exist", fs.exists(testDir)); + + FileStatus status = fs.getFileStatus(testDir); + assertTrue("Path should be a directory", status.isDirectory()); + } + + @Test + public void testCreateAndReadFile() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/testfile.txt"); + String testContent = "Hello, SeaweedFS!"; + + // Create and write to file + FSDataOutputStream out = fs.create(testFile, FsPermission.getDefault(), + false, 4096, (short) 1, 4 * 1024 * 1024, null); + assertNotNull("Output stream should not be null", out); + out.write(testContent.getBytes()); + out.close(); + + // Verify file exists + assertTrue("File should exist", fs.exists(testFile)); + + // Read and verify content + FSDataInputStream in = fs.open(testFile, 4096); + assertNotNull("Input stream should not be null", in); + byte[] buffer = new byte[testContent.length()]; + int bytesRead = in.read(buffer); + in.close(); + + assertEquals("Should read all bytes", testContent.length(), bytesRead); + assertEquals("Content should match", testContent, new String(buffer)); + } + + @Test + public void testFileStatus() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/statustest.txt"); + String content = "test content"; + + FSDataOutputStream out = fs.create(testFile); + out.write(content.getBytes()); + out.close(); + + FileStatus status = fs.getFileStatus(testFile); + assertNotNull("FileStatus should not be null", status); + assertFalse("Should not be a directory", status.isDirectory()); + assertTrue("Should be a file", status.isFile()); + assertEquals("File length should match", content.length(), status.getLen()); + assertNotNull("Path should not be null", status.getPath()); + } + + @Test + public void testListStatus() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testDir = new Path(TEST_ROOT + "/listtest"); + fs.mkdirs(testDir); + + // Create multiple files + for (int i = 0; i < 3; i++) { + Path file = new Path(testDir, "file" + i + ".txt"); + FSDataOutputStream out = fs.create(file); + out.write(("content" + i).getBytes()); + out.close(); + } + + FileStatus[] statuses = fs.listStatus(testDir); + assertNotNull("List should not be null", statuses); + assertEquals("Should have 3 files", 3, statuses.length); + } + + @Test + public void testRename() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path srcFile = new Path(TEST_ROOT + "/source.txt"); + Path dstFile = new Path(TEST_ROOT + "/destination.txt"); + String content = "rename test"; + + // Create source file + FSDataOutputStream out = fs.create(srcFile); + out.write(content.getBytes()); + out.close(); + + assertTrue("Source file should exist", fs.exists(srcFile)); + + // Rename + assertTrue("Rename should succeed", fs.rename(srcFile, dstFile)); + + // Verify + assertFalse("Source file should not exist", fs.exists(srcFile)); + assertTrue("Destination file should exist", fs.exists(dstFile)); + + // Verify content preserved + FSDataInputStream in = fs.open(dstFile); + byte[] buffer = new byte[content.length()]; + in.read(buffer); + in.close(); + assertEquals("Content should be preserved", content, new String(buffer)); + } + + @Test + public void testDelete() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/deletetest.txt"); + + // Create file + FSDataOutputStream out = fs.create(testFile); + out.write("delete me".getBytes()); + out.close(); + + assertTrue("File should exist before delete", fs.exists(testFile)); + + // Delete + assertTrue("Delete should succeed", fs.delete(testFile, false)); + assertFalse("File should not exist after delete", fs.exists(testFile)); + } + + @Test + public void testDeleteDirectory() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testDir = new Path(TEST_ROOT + "/deletedir"); + Path testFile = new Path(testDir, "file.txt"); + + // Create directory with file + fs.mkdirs(testDir); + FSDataOutputStream out = fs.create(testFile); + out.write("content".getBytes()); + out.close(); + + assertTrue("Directory should exist", fs.exists(testDir)); + assertTrue("File should exist", fs.exists(testFile)); + + // Recursive delete + assertTrue("Recursive delete should succeed", fs.delete(testDir, true)); + assertFalse("Directory should not exist after delete", fs.exists(testDir)); + assertFalse("File should not exist after delete", fs.exists(testFile)); + } + + @Test + public void testAppend() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/appendtest.txt"); + String initialContent = "initial"; + String appendContent = " appended"; + + // Create initial file + FSDataOutputStream out = fs.create(testFile); + out.write(initialContent.getBytes()); + out.close(); + + // Append + FSDataOutputStream appendOut = fs.append(testFile, 4096, null); + assertNotNull("Append stream should not be null", appendOut); + appendOut.write(appendContent.getBytes()); + appendOut.close(); + + // Verify combined content + FSDataInputStream in = fs.open(testFile); + byte[] buffer = new byte[initialContent.length() + appendContent.length()]; + int bytesRead = in.read(buffer); + in.close(); + + String expected = initialContent + appendContent; + assertEquals("Should read all bytes", expected.length(), bytesRead); + assertEquals("Content should match", expected, new String(buffer)); + } + + @Test + public void testSetWorkingDirectory() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path originalWd = fs.getWorkingDirectory(); + assertEquals("Original working directory should be /", "/", originalWd.toUri().getPath()); + + Path newWd = new Path(TEST_ROOT); + fs.mkdirs(newWd); + fs.setWorkingDirectory(newWd); + + Path currentWd = fs.getWorkingDirectory(); + assertTrue("Working directory should be updated", + currentWd.toUri().getPath().contains(TEST_ROOT)); + } + + @Test + public void testSetPermission() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/permtest.txt"); + + // Create file + FSDataOutputStream out = fs.create(testFile); + out.write("permission test".getBytes()); + out.close(); + + // Set permission + FsPermission newPerm = new FsPermission((short) 0644); + fs.setPermission(testFile, newPerm); + + FileStatus status = fs.getFileStatus(testFile); + assertNotNull("Permission should not be null", status.getPermission()); + } + + @Test + public void testSetOwner() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path testFile = new Path(TEST_ROOT + "/ownertest.txt"); + + // Create file + FSDataOutputStream out = fs.create(testFile); + out.write("owner test".getBytes()); + out.close(); + + // Set owner - this may not fail even if not fully implemented + fs.setOwner(testFile, "testuser", "testgroup"); + + // Just verify the call doesn't throw an exception + FileStatus status = fs.getFileStatus(testFile); + assertNotNull("FileStatus should not be null", status); + } + + @Test + public void testRenameToExistingDirectory() throws Exception { + if (!TESTS_ENABLED) { + System.out.println("Skipping test - SEAWEEDFS_TEST_ENABLED not set"); + return; + } + + Path srcFile = new Path(TEST_ROOT + "/movefile.txt"); + Path dstDir = new Path(TEST_ROOT + "/movedir"); + + // Create source file and destination directory + FSDataOutputStream out = fs.create(srcFile); + out.write("move test".getBytes()); + out.close(); + fs.mkdirs(dstDir); + + // Rename file to existing directory (should move file into directory) + assertTrue("Rename to directory should succeed", fs.rename(srcFile, dstDir)); + + // File should be moved into the directory + Path expectedLocation = new Path(dstDir, srcFile.getName()); + assertTrue("File should exist in destination directory", fs.exists(expectedLocation)); + assertFalse("Source file should not exist", fs.exists(srcFile)); + } +} +