mirror of https://github.com/trapexit/mergerfs.git
Antonio SJ Musumeci
1 month ago
11 changed files with 442 additions and 292 deletions
-
10libfuse/include/fuse.h
-
5libfuse/lib/fuse.cpp
-
2libfuse/lib/fuse_lowlevel.cpp
-
15src/fileinfo.hpp
-
124src/fuse_create.cpp
-
242src/fuse_open.cpp
-
84src/fuse_release.cpp
-
12src/state.hpp
-
103tests/TEST_io_passthrough_create_open
-
132tests/TEST_io_passthrough_open_race
-
5tests/tests.cpp
@ -0,0 +1,103 @@ |
|||
#!/usr/bin/env python3 |
|||
|
|||
import os |
|||
import sys |
|||
import tempfile |
|||
import time |
|||
|
|||
# Test create + write + read (passthrough.io=rw) |
|||
(fd, filepath) = tempfile.mkstemp(dir=sys.argv[1]) |
|||
|
|||
test_data = b"passthrough io test data for create\n" * 100 |
|||
|
|||
# Write to newly created file |
|||
bytes_written = os.write(fd, test_data) |
|||
if bytes_written != len(test_data): |
|||
print("create write failed: expected {} bytes, wrote {}".format(len(test_data), bytes_written)) |
|||
sys.exit(1) |
|||
|
|||
# Seek and read back |
|||
os.lseek(fd, 0, os.SEEK_SET) |
|||
read_data = os.read(fd, len(test_data)) |
|||
if read_data != test_data: |
|||
print("create read failed: data mismatch") |
|||
sys.exit(1) |
|||
|
|||
os.close(fd) |
|||
|
|||
# Test open existing file + write + read |
|||
fd = os.open(filepath, os.O_RDWR) |
|||
|
|||
# Read existing data |
|||
os.lseek(fd, 0, os.SEEK_SET) |
|||
read_data = os.read(fd, len(test_data)) |
|||
if read_data != test_data: |
|||
print("open read failed: data mismatch") |
|||
sys.exit(1) |
|||
|
|||
# Write more data at end |
|||
os.lseek(fd, 0, os.SEEK_END) |
|||
more_data = b"additional passthrough data for open\n" * 50 |
|||
bytes_written = os.write(fd, more_data) |
|||
if bytes_written != len(more_data): |
|||
print("open write failed: expected {} bytes, wrote {}".format(len(more_data), bytes_written)) |
|||
sys.exit(1) |
|||
|
|||
# Verify all data |
|||
os.lseek(fd, 0, os.SEEK_SET) |
|||
all_data = os.read(fd, len(test_data) + len(more_data)) |
|||
if all_data != test_data + more_data: |
|||
print("open final read failed: data mismatch") |
|||
sys.exit(1) |
|||
|
|||
# Test multiple opens of same file (single backing_id feature) |
|||
# When a file is already open with passthrough, subsequent opens |
|||
# must reuse the same backing_id rather than creating new ones |
|||
fd2 = os.open(filepath, os.O_RDWR) |
|||
|
|||
# Read from second fd - should see same data |
|||
os.lseek(fd2, 0, os.SEEK_SET) |
|||
read_data2 = os.read(fd2, len(test_data) + len(more_data)) |
|||
if read_data2 != test_data + more_data: |
|||
print("second open read failed: data mismatch") |
|||
os.close(fd) |
|||
os.close(fd2) |
|||
sys.exit(1) |
|||
|
|||
# Write from second fd |
|||
os.lseek(fd2, 0, os.SEEK_END) |
|||
extra_data = b"data from second file descriptor\n" * 25 |
|||
bytes_written = os.write(fd2, extra_data) |
|||
if bytes_written != len(extra_data): |
|||
print("second open write failed: expected {} bytes, wrote {}".format(len(extra_data), bytes_written)) |
|||
os.close(fd) |
|||
os.close(fd2) |
|||
sys.exit(1) |
|||
|
|||
# Verify write is visible from first fd (shared backing) |
|||
os.lseek(fd, 0, os.SEEK_SET) |
|||
combined_data = os.read(fd, len(test_data) + len(more_data) + len(extra_data)) |
|||
expected_data = test_data + more_data + extra_data |
|||
if combined_data != expected_data: |
|||
print("cross-fd read failed: data mismatch (writes from fd2 not visible on fd)") |
|||
os.close(fd) |
|||
os.close(fd2) |
|||
sys.exit(1) |
|||
|
|||
# Open a third fd while others are still open |
|||
fd3 = os.open(filepath, os.O_RDONLY) |
|||
os.lseek(fd3, 0, os.SEEK_SET) |
|||
read_data3 = os.read(fd3, len(expected_data)) |
|||
if read_data3 != expected_data: |
|||
print("third open read failed: data mismatch") |
|||
os.close(fd) |
|||
os.close(fd2) |
|||
os.close(fd3) |
|||
sys.exit(1) |
|||
|
|||
os.close(fd3) |
|||
os.close(fd2) |
|||
os.close(fd) |
|||
|
|||
# Cleanup |
|||
os.unlink(filepath) |
|||
@ -0,0 +1,132 @@ |
|||
#!/usr/bin/env python3 |
|||
|
|||
import os |
|||
import sys |
|||
import tempfile |
|||
import threading |
|||
import time |
|||
|
|||
NUM_THREADS = 50 |
|||
TEST_DATA = b"race condition test data\n" |
|||
|
|||
def open_and_operate(filepath, barrier, results, index): |
|||
""" |
|||
Wait at barrier, then open file, write, read, and store result. |
|||
""" |
|||
try: |
|||
# Wait for all threads to be ready |
|||
barrier.wait() |
|||
|
|||
# All threads try to open simultaneously |
|||
fd = os.open(filepath, os.O_RDWR) |
|||
|
|||
# Write thread-specific data at a unique offset |
|||
offset = index * len(TEST_DATA) |
|||
os.lseek(fd, offset, os.SEEK_SET) |
|||
bytes_written = os.write(fd, TEST_DATA) |
|||
|
|||
if bytes_written != len(TEST_DATA): |
|||
results[index] = ("write_error", "expected {} bytes, wrote {}".format(len(TEST_DATA), bytes_written)) |
|||
os.close(fd) |
|||
return |
|||
|
|||
# Read back what we wrote |
|||
os.lseek(fd, offset, os.SEEK_SET) |
|||
read_data = os.read(fd, len(TEST_DATA)) |
|||
|
|||
if read_data != TEST_DATA: |
|||
results[index] = ("read_error", "data mismatch at offset {}".format(offset)) |
|||
os.close(fd) |
|||
return |
|||
|
|||
results[index] = ("success", fd) |
|||
|
|||
except Exception as e: |
|||
results[index] = ("exception", str(e)) |
|||
|
|||
# Create test file with enough space for all threads |
|||
(fd, filepath) = tempfile.mkstemp(dir=sys.argv[1]) |
|||
|
|||
# Pre-allocate file to avoid issues with concurrent writes extending file |
|||
total_size = NUM_THREADS * len(TEST_DATA) |
|||
os.ftruncate(fd, total_size) |
|||
os.close(fd) |
|||
|
|||
# Set up synchronization |
|||
barrier = threading.Barrier(NUM_THREADS) |
|||
results = [None] * NUM_THREADS |
|||
threads = [] |
|||
|
|||
# Create and start all threads |
|||
for i in range(NUM_THREADS): |
|||
t = threading.Thread(target=open_and_operate, args=(filepath, barrier, results, i)) |
|||
threads.append(t) |
|||
|
|||
for t in threads: |
|||
t.start() |
|||
|
|||
for t in threads: |
|||
t.join() |
|||
|
|||
# Check results |
|||
failed = False |
|||
fds_to_close = [] |
|||
|
|||
for i, result in enumerate(results): |
|||
if result is None: |
|||
print("thread {} returned no result".format(i)) |
|||
failed = True |
|||
elif result[0] == "success": |
|||
fds_to_close.append(result[1]) |
|||
else: |
|||
print("thread {} failed: {} - {}".format(i, result[0], result[1])) |
|||
failed = True |
|||
|
|||
if failed: |
|||
for fd in fds_to_close: |
|||
os.close(fd) |
|||
os.unlink(filepath) |
|||
sys.exit(1) |
|||
|
|||
# Verify all data is consistent by reading through one fd |
|||
if fds_to_close: |
|||
verify_fd = fds_to_close[0] |
|||
os.lseek(verify_fd, 0, os.SEEK_SET) |
|||
all_data = os.read(verify_fd, total_size) |
|||
|
|||
expected_data = TEST_DATA * NUM_THREADS |
|||
if all_data != expected_data: |
|||
print("final verification failed: data mismatch") |
|||
print("expected {} bytes, got {} bytes".format(len(expected_data), len(all_data))) |
|||
for fd in fds_to_close: |
|||
os.close(fd) |
|||
os.unlink(filepath) |
|||
sys.exit(1) |
|||
|
|||
# Test cross-fd visibility: write from one fd, read from another |
|||
if len(fds_to_close) >= 2: |
|||
fd_a = fds_to_close[0] |
|||
fd_b = fds_to_close[1] |
|||
|
|||
# Write new data from fd_a |
|||
new_data = b"cross-fd visibility test\n" |
|||
os.lseek(fd_a, 0, os.SEEK_SET) |
|||
os.write(fd_a, new_data) |
|||
|
|||
# Read from fd_b - should see the new data |
|||
os.lseek(fd_b, 0, os.SEEK_SET) |
|||
read_back = os.read(fd_b, len(new_data)) |
|||
|
|||
if read_back != new_data: |
|||
print("cross-fd visibility failed: write from fd_a not visible on fd_b") |
|||
for fd in fds_to_close: |
|||
os.close(fd) |
|||
os.unlink(filepath) |
|||
sys.exit(1) |
|||
|
|||
# Close all file descriptors |
|||
for fd in fds_to_close: |
|||
os.close(fd) |
|||
|
|||
# Cleanup |
|||
os.unlink(filepath) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue