From 1f90800d6e4c255d5bc8bb37fab4f2646c678012 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Sat, 21 Mar 2026 12:53:54 -0500 Subject: [PATCH] Add more integration tests --- tests/OPTIONS_TEST_PLAN.md | 117 ++++++++++++ tests/README.md | 69 ++++++++ tests/TEST_cfg_link_rename_exdev | 144 +++++++++++++++ tests/TEST_cfg_statfs_ignore | 65 +++++++ tests/TEST_cfg_xattr_modes | 82 +++++++++ tests/TEST_mount_lifecycle | 65 +++++++ tests/TEST_posix_access | 68 +++++++ tests/TEST_posix_bmap | 68 +++++++ tests/TEST_posix_chmod | 93 ++++++++++ tests/TEST_posix_chown | 75 ++++++++ tests/TEST_posix_copy_file_range | 78 ++++++++ tests/TEST_posix_create_mknod | 93 ++++++++++ tests/TEST_posix_fsync_flush | 75 ++++++++ tests/TEST_posix_getattr_fgetattr | 94 ++++++++++ tests/TEST_posix_ioctl | 60 +++++++ tests/TEST_posix_link_symlink | 67 +++++++ tests/TEST_posix_locking | 99 +++++++++++ tests/TEST_posix_mkdir_rmdir | 71 ++++++++ tests/TEST_posix_open_read_write | 79 +++++++++ tests/TEST_posix_poll | 55 ++++++ tests/TEST_posix_readdir | 170 ++++++++++++++++++ tests/TEST_posix_readdir_plus | 69 ++++++++ tests/TEST_posix_release | 79 +++++++++ tests/TEST_posix_releasedir | 71 ++++++++ tests/TEST_posix_statfs | 51 ++++++ tests/TEST_posix_statx | 265 ++++++++++++++++++++++++++++ tests/TEST_posix_syncfs | 59 +++++++ tests/TEST_posix_tmpfile | 64 +++++++ tests/TEST_posix_truncate_ftruncate | 57 ++++++ tests/TEST_posix_unlink_rename | 70 ++++++++ tests/TEST_posix_utimens | 77 ++++++++ tests/TEST_posix_xattr | 157 ++++++++++++++++ tests/TEST_posix_xattr_matrix | 211 ++++++++++++++++++++++ tests/posix_parity.py | 185 +++++++++++++++++++ 34 files changed, 3202 insertions(+) create mode 100644 tests/OPTIONS_TEST_PLAN.md create mode 100644 tests/README.md create mode 100755 tests/TEST_cfg_link_rename_exdev create mode 100755 tests/TEST_cfg_statfs_ignore create mode 100755 tests/TEST_cfg_xattr_modes create mode 100755 tests/TEST_mount_lifecycle create mode 100755 tests/TEST_posix_access create mode 100755 tests/TEST_posix_bmap create mode 100755 tests/TEST_posix_chmod create mode 100755 tests/TEST_posix_chown create mode 100755 tests/TEST_posix_copy_file_range create mode 100755 tests/TEST_posix_create_mknod create mode 100755 tests/TEST_posix_fsync_flush create mode 100755 tests/TEST_posix_getattr_fgetattr create mode 100755 tests/TEST_posix_ioctl create mode 100755 tests/TEST_posix_link_symlink create mode 100755 tests/TEST_posix_locking create mode 100755 tests/TEST_posix_mkdir_rmdir create mode 100755 tests/TEST_posix_open_read_write create mode 100755 tests/TEST_posix_poll create mode 100755 tests/TEST_posix_readdir create mode 100755 tests/TEST_posix_readdir_plus create mode 100755 tests/TEST_posix_release create mode 100755 tests/TEST_posix_releasedir create mode 100755 tests/TEST_posix_statfs create mode 100755 tests/TEST_posix_statx create mode 100755 tests/TEST_posix_syncfs create mode 100755 tests/TEST_posix_tmpfile create mode 100755 tests/TEST_posix_truncate_ftruncate create mode 100755 tests/TEST_posix_unlink_rename create mode 100755 tests/TEST_posix_utimens create mode 100755 tests/TEST_posix_xattr create mode 100755 tests/TEST_posix_xattr_matrix create mode 100644 tests/posix_parity.py diff --git a/tests/OPTIONS_TEST_PLAN.md b/tests/OPTIONS_TEST_PLAN.md new file mode 100644 index 00000000..ae2e44be --- /dev/null +++ b/tests/OPTIONS_TEST_PLAN.md @@ -0,0 +1,117 @@ +# Options Test Plan + +Possible integration and unit tests for options listed in `mkdocs/docs/config/options.md`. + +Legend: +- **Integration**: mount-level behavior test (black-box, syscall parity / side effects) +- **Unit**: focused parser/config/logic test (white-box or low-level component) + +## Core mount and branch options + +| Option | Integration tests | Unit tests | +|---|---|---| +| `config` | Load same settings via CLI vs config file and compare behavior | Parse comments, `key`, `key=val`, unknown keys, duplicate keys ordering | +| `branches` | Multi-branch create/search/action behavior; glob expansion; dynamic branch updates | Branch string parser ops (`+`, `+<`, `+>`, `-`, `->`, `-<`, `=`), mode parse, minfreespace parse | +| `mountpoint` | Verify mount succeeds/fails with valid/invalid mountpoint | Config validation for mountpoint path | +| `branches-mount-timeout` | Delayed branch mount appears before timeout -> starts; after timeout -> continue/fail based on flag | Timeout arithmetic and mount-detection helper logic | +| `branches-mount-timeout-fail` | Timeout expiration returns non-zero when true | Boolean parse and gating of startup continuation | +| `minfreespace` | Create policy excludes near-full branches | Size parser (`B/K/M/G/T`, overflow, bad suffix) | +| `moveonenospc` | ENOSPC write triggers move+retry and preserves metadata | Policy dispatch and error mapping (`ENOSPC`/`EDQUOT`) | +| `inodecalc` | Stable inode expectations per mode on files/dirs/hardlinks | Hash function determinism and 32-bit/64-bit variants | + +## IO behavior options + +| Option | Integration tests | Unit tests | +|---|---|---| +| `dropcacheonclose` | Close after write/read and confirm no correctness regression | `posix_fadvise` call path invoked only when enabled | +| `direct-io-allow-mmap` | `O_DIRECT` + `mmap` behavior on supported kernels | Feature flag gating by kernel capability | +| `nullrw` | Writes report success but data unchanged; reads become no-op behavior | Read/write null path selection | +| `readahead` | Sequential read throughput/request size changes with non-zero setting | Parser and branch-level readahead setter behavior | +| `async-read` | Concurrent read ordering/parallelism differs when toggled | FUSE config flag toggling | +| `fuse-msg-size` | Large IO request chunking and throughput with different values | Pagesize conversion parser and min/max clamping | +| `flush-on-close` | `never/always/opened-for-write` close semantics | Mode enum parse and branching logic | +| `proxy-ioprio` | Different caller ioprio reflected in IO scheduling side-effects | ioprio fetch/apply path and fallback behavior | +| `parallel-direct-writes` | Parallel non-extending writes behavior with/without option | Kernel capability gating | +| `passthrough.io` | `off/ro/wo/rw` path selection for reads/writes and incompatibility checks | Enum parse, compatibility checks (`cache.writeback`, `nullrw`, etc.) | +| `passthrough.max-stack-depth` | Stacked FS scenarios with depth 1 vs 2 | Range validation and config propagation | + +## Path transformation and cross-device options + +| Option | Integration tests | Unit tests | +|---|---|---| +| `symlinkify` | Old non-writable files appear as symlinks after timeout | Eligibility predicate (mode, age, type) | +| `symlinkify-timeout` | Boundary test around exact timeout second | Timeout comparison helper | +| `ignorepponrename` | Rename behavior with and without path preservation | Rename policy selection logic | +| `follow-symlinks` | `never/directory/regular/all` on symlink to file/dir/broken link | Enum parse and target-type dispatch | +| `link-exdev` | EXDEV hardlink fallback modes: passthrough/rel/abs-base/abs-pool | Strategy selection and symlink target synthesis | +| `rename-exdev` | EXDEV rename fallback modes: passthrough/rel/abs | Fallback path generation and state machine | +| `link-cow` | Hardlinked file opened for write breaks link and writes private copy | Link-count check and copy-on-write trigger logic | + +## Permission and metadata options + +| Option | Integration tests | Unit tests | +|---|---|---| +| `export-support` | NFS-export style behavior sanity and mount flags exposure | Init-time flag propagation | +| `kernel-permissions-check` | Access/permission checks handled by kernel vs userspace path | Mount option mapping (`default_permissions`) | +| `security-capability` | `security.capability` xattr returns ENOATTR when disabled | Name filter helper | +| `xattr` | `passthrough/noattr/nosys` behavior for set/get/list/remove xattr | Enum parse and short-circuit return paths | +| `statfs` | `base` vs `full` branch inclusion by path | Mode parse and branch-filter helper | +| `statfs-ignore` | `none/ro/nc` effect on available space accounting | Ignore predicate and accumulator logic | +| `nfsopenhack` | File create/open mode behavior under NFS-like patterns | Enum parse and hook decision logic | +| `posix-acl` | ACL xattr roundtrip and chmod/chown interaction when enabled | Mount/init flag propagation | + +## Threading and scheduling options + +| Option | Integration tests | Unit tests | +|---|---|---| +| `read-thread-count` | Throughput/latency and correctness under different values | Value normalization (`0`, negatives, min=1) | +| `process-thread-count` | Separation of read/process pools and queue pressure behavior | Pool enable/disable logic | +| `process-thread-queue-depth` | Backpressure behavior with depth changes | Queue sizing and timeout/try-enqueue semantics | +| `pin-threads` | Thread CPU affinity placement for each strategy | Strategy parser and CPU selection algorithm | +| `scheduling-priority` | Process nice value set at startup and effective scheduling | Range validation and setpriority wrapper | + +## Function/category policy options + +| Option | Integration tests | Unit tests | +|---|---|---| +| `func.FUNC` | Override a single function policy and verify only that op changes | Parser mapping from key to policy holder | +| `func.readdir` | `seq/cosr/cor` ordering, latency, duplicate suppression, thread count parsing | Mode parser (`cosr:N:M`, `cor:N:M`) | +| `category.action` | Bulk action policy changes for chmod/chown/rename/unlink | Category-to-function propagation | +| `category.create` | Create policy effects on create/mkdir/mknod/link/symlink | Category propagation and precedence | +| `category.search` | Search policy effects on getattr/open/readlink/getxattr | Category propagation and precedence | +| Option ordering note | Later options override earlier (`func.*` then `category.*` etc.) | Deterministic application order test | + +## Cache options + +| Option | Integration tests | Unit tests | +|---|---|---| +| `cache.statfs` | Repeated statfs calls use cache within timeout | Timeout cache invalidation behavior | +| `cache.attr` | Repeated getattr freshness across timeout boundaries | Attr cache key and expiry logic | +| `cache.entry` | Name lookup caching for existing entries | Entry cache insertion/expiry | +| `cache.negative-entry` | Missing entry lookup caching and stale-negative behavior | Negative cache behavior | +| `cache.files` | `off/partial/full/auto-full/per-process` correctness and coherence | Enum parse and mode decision helper | +| `cache.files.process-names` | Per-process cache activation by comm name | List parser and matcher | +| `cache.writeback` | Coalesced write behavior; incompatibility handling with passthrough | Flag validation and FUSE config setup | +| `cache.symlinks` | Readlink repeated calls reflect cache policy | Feature gating by kernel support | +| `cache.readdir` | Readdir repeated calls and invalidation behavior | Feature gating and config wiring | + +## Misc options + +| Option | Integration tests | Unit tests | +|---|---|---| +| `lazy-umount-mountpoint` | Live-upgrade remount scenario with stale previous mount | Startup control flow and error handling | +| `remember-nodes` | NFS-like lookup stability over configured retention interval | Node cache retention/expiry logic | +| `noforget` | No node expiry over extended period | Infinite retention mode mapping | +| `allow-idmap` | UID/GID mapping behavior in idmapped mounts | Init flag propagation and compatibility checks | +| `debug` | FUSE trace generation enabled/disabled | Debug option parse and logger toggling | +| `log.file` | Trace output to stderr vs file path | Path parser and file sink setup | +| `fsname` | Filesystem name shown in mount/df matches config | fsname default derivation and override | + +## Prioritized backlog (recommended first) + +1. `xattr` mode tests (`passthrough/noattr/nosys`) + regression for current xattr mismatch +2. `statfs` / `statfs-ignore` mode matrix +3. EXDEV fallbacks (`link-exdev`, `rename-exdev`) +4. `follow-symlinks` + `symlinkify` behavior matrix +5. Policy precedence (`func.*`, `category.*`, ordering) +6. Cache timeout behavior (`cache.attr`, `cache.entry`, `cache.negative-entry`, `cache.statfs`) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..9345ab2a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,69 @@ +# POSIX Soft Conformance Suite + +This directory contains integration tests that exercise mergerfs FUSE operations +and compare behavior against equivalent native syscalls on a local filesystem. + +The suite is "soft POSIX": it focuses on practical parity for return values, +`errno`, and key side effects, including common error paths. + +Run with: + +```bash +python3 tests/run-tests /mnt/mergerfs/ +``` + +## Coverage Matrix + +- `access` -> `TEST_posix_access` +- `chmod` / `fchmod` -> `TEST_posix_chmod`, `TEST_use_fchmod_after_unlink` +- `chown` / `fchown` -> `TEST_posix_chown`, `TEST_use_fchown_after_unlink` +- `create` / `mknod` -> `TEST_posix_create_mknod` +- `getattr` / `fgetattr` -> `TEST_posix_getattr_fgetattr`, `TEST_use_fstat_after_unlink` +- `open` / `read` / `write` -> `TEST_posix_open_read_write`, `TEST_o_direct` +- `flush` / `release` / `fsync` / `fsyncdir` -> `TEST_posix_fsync_flush` +- `release` close semantics -> `TEST_posix_release` +- `releasedir` close semantics -> `TEST_posix_releasedir` +- `truncate` / `ftruncate` -> `TEST_posix_truncate_ftruncate`, `TEST_use_ftruncate_after_unlink` +- `utimens` / `futimens` -> `TEST_posix_utimens`, `TEST_use_futimens_after_unlink` +- `fallocate` -> `TEST_use_fallocate_after_unlink` +- `copy_file_range` -> `TEST_posix_copy_file_range` +- `ioctl` -> `TEST_posix_ioctl` +- `poll` -> `TEST_posix_poll` +- `tmpfile` (`O_TMPFILE`) -> `TEST_posix_tmpfile` +- `bmap` (via `FIBMAP` ioctl parity) -> `TEST_posix_bmap` +- `readlink` -> `TEST_readlink_semantics` +- `link` / `symlink` -> `TEST_posix_link_symlink` +- `mkdir` / `rmdir` -> `TEST_posix_mkdir_rmdir` +- `unlink` / `rename` -> `TEST_posix_unlink_rename`, `TEST_unlink_rename` +- `statfs` -> `TEST_posix_statfs` +- `statx` -> `TEST_posix_statx` +- `syncfs` -> `TEST_posix_syncfs` +- `flock` / `lock` -> `TEST_posix_locking` +- `setxattr` / `getxattr` / `listxattr` / `removexattr` -> `TEST_posix_xattr` +- xattr object/flag matrix (`file`, `dir`, symlink `l*`, create/replace) -> `TEST_posix_xattr_matrix` +- xattr mode runtime toggles (`passthrough`/`noattr`/`nosys`) -> `TEST_cfg_xattr_modes` +- statfs mode and statfs-ignore runtime toggles -> `TEST_cfg_statfs_ignore` +- EXDEV fallback runtime toggles for link/rename -> `TEST_cfg_link_rename_exdev` +- `readdir` (+ seek/tell behavior) -> `TEST_posix_readdir` +- `readdir_plus` semantic parity (list+stat and readdir policy toggles) -> `TEST_posix_readdir_plus` +- lifecycle (`init`/`destroy`) harness check -> `TEST_mount_lifecycle` +- hidden temp exposure checks -> `TEST_no_fuse_hidden` + +## Error Conditions Emphasized + +Across tests, parity checks include combinations of: + +- `ENOENT` +- `ENOTDIR` +- `EEXIST` +- `ENOTEMPTY` +- `EISDIR` +- `EBADF` +- privilege-sensitive outcomes (for example `EPERM`) where applicable + +## Notes + +- Some behavior is runtime/environment dependent (kernel, mount options, + uid/gid, capabilities). Tests compare mergerfs to native behavior in the same + runtime to keep assertions robust. +- The xattr test uses a `user.*` key and assumes user xattrs are enabled. diff --git a/tests/TEST_cfg_link_rename_exdev b/tests/TEST_cfg_link_rename_exdev new file mode 100755 index 00000000..ce1721a3 --- /dev/null +++ b/tests/TEST_cfg_link_rename_exdev @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +import errno +import os +import sys + +from posix_parity import fail +from posix_parity import mergerfs_get_option +from posix_parity import mergerfs_set_option +from posix_parity import parse_allpaths +from posix_parity import touch +from posix_parity import join + + +def get_errno(func): + try: + func() + return 0 + except OSError as exc: + return exc.errno + + +def allpaths(path): + raw = os.getxattr(path, "user.mergerfs.allpaths") + return parse_allpaths(raw) + + +def must_two_branches(paths, op): + if len(paths) < 2: + return f"{op}: requires at least 2 branches; got {paths!r}" + return None + + +def pick_cross_branch_paths(mount): + src = join(mount, "cfg-exdev/src") + dst = join(mount, "cfg-exdev/sub/dst") + + touch(src, b"src") + os.makedirs(os.path.dirname(dst), exist_ok=True) + + try: + src_paths = allpaths(src) + except OSError as exc: + if exc.errno in (errno.EOPNOTSUPP, errno.ENOTSUP, errno.ENOSYS): + raise RuntimeError("allpaths-xattr-unsupported") + raise + if len(src_paths) < 2: + # Try to create destination first to force different existing-path decisions. + touch(dst, b"old") + os.unlink(dst) + try: + src_paths = allpaths(src) + except OSError as exc: + if exc.errno in (errno.EOPNOTSUPP, errno.ENOTSUP, errno.ENOSYS): + raise RuntimeError("allpaths-xattr-unsupported") + raise + + err = must_two_branches(src_paths, "cross-branch setup") + if err: + raise RuntimeError(err) + + return src, dst + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_cfg_link_rename_exdev ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + try: + src, dst = pick_cross_branch_paths(mount) + except PermissionError: + return 0 + except RuntimeError as exc: + # Environment-dependent; treat as soft skip. + msg = str(exc) + if "requires at least 2 branches" in msg or msg == "allpaths-xattr-unsupported": + return 0 + return fail(msg) + except OSError: + return 0 + + try: + orig_create = mergerfs_get_option(mount, "category.create") + orig_link = mergerfs_get_option(mount, "link-exdev") + orig_rename = mergerfs_get_option(mount, "rename-exdev") + except (PermissionError, FileNotFoundError, OSError): + return 0 + + try: + # Maximize chance of EXDEV path by path-preserving create policy. + mergerfs_set_option(mount, "category.create", "epmfs") + + # link-exdev passthrough => EXDEV when cross-device path preserving applies. + mergerfs_set_option(mount, "link-exdev", "passthrough") + err = get_errno(lambda: os.link(src, dst + ".link.pass")) + if err not in (0, errno.EXDEV): + return fail(f"link-exdev=passthrough unexpected errno={err}") + link_exdev_active = (err == errno.EXDEV) + + # rel-symlink should not return EXDEV if fallback triggers. + mergerfs_set_option(mount, "link-exdev", "rel-symlink") + rel_link = dst + ".link.rel" + err = get_errno(lambda: os.link(src, rel_link)) + if link_exdev_active and err == errno.EXDEV: + return fail("link-exdev=rel-symlink still returned EXDEV") + if link_exdev_active and err == 0 and not os.path.islink(rel_link): + return fail("link-exdev=rel-symlink expected symlink fallback") + + # rename-exdev passthrough => EXDEV in cross-device path preserving case. + src_ren = src + ".ren" + touch(src_ren, b"src") + mergerfs_set_option(mount, "rename-exdev", "passthrough") + err = get_errno(lambda: os.rename(src_ren, dst + ".ren.pass")) + if err not in (0, errno.EXDEV): + return fail(f"rename-exdev=passthrough unexpected errno={err}") + rename_exdev_active = (err == errno.EXDEV) + + # rel-symlink should avoid EXDEV by fallback when applicable. + src_ren2 = src + ".ren2" + touch(src_ren2, b"src") + mergerfs_set_option(mount, "rename-exdev", "rel-symlink") + rel_ren = dst + ".ren.rel" + err = get_errno(lambda: os.rename(src_ren2, rel_ren)) + if rename_exdev_active and err == errno.EXDEV: + return fail("rename-exdev=rel-symlink still returned EXDEV") + if rename_exdev_active and err == 0 and not os.path.islink(rel_ren): + return fail("rename-exdev=rel-symlink expected symlink fallback") + except (PermissionError, FileNotFoundError, OSError): + return 0 + finally: + try: + mergerfs_set_option(mount, "category.create", orig_create) + mergerfs_set_option(mount, "link-exdev", orig_link) + mergerfs_set_option(mount, "rename-exdev", orig_rename) + except OSError: + pass + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_cfg_statfs_ignore b/tests/TEST_cfg_statfs_ignore new file mode 100755 index 00000000..95173d74 --- /dev/null +++ b/tests/TEST_cfg_statfs_ignore @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import os +import sys + +from posix_parity import fail +from posix_parity import mergerfs_get_option +from posix_parity import mergerfs_set_option + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_cfg_statfs_ignore ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + try: + orig_statfs = mergerfs_get_option(mount, "statfs") + orig_ignore = mergerfs_get_option(mount, "statfs-ignore") + except (PermissionError, FileNotFoundError, OSError): + return 0 + + try: + mergerfs_set_option(mount, "statfs", "base") + + mergerfs_set_option(mount, "statfs-ignore", "none") + s_none = os.statvfs(mount) + + mergerfs_set_option(mount, "statfs-ignore", "nc") + s_nc = os.statvfs(mount) + + mergerfs_set_option(mount, "statfs-ignore", "ro") + s_ro = os.statvfs(mount) + + # Ignoring additional branch classes should never increase available blocks. + if s_nc.f_bavail > s_none.f_bavail: + return fail( + f"statfs-ignore=nc increased bavail unexpectedly nc={s_nc.f_bavail} none={s_none.f_bavail}" + ) + if s_ro.f_bavail > s_nc.f_bavail: + return fail( + f"statfs-ignore=ro increased bavail unexpectedly ro={s_ro.f_bavail} nc={s_nc.f_bavail}" + ) + + mergerfs_set_option(mount, "statfs", "full") + s_full = os.statvfs(mount) + if s_full.f_bsize <= 0 or s_full.f_frsize <= 0: + return fail( + f"statfs=full invalid block sizes bsize={s_full.f_bsize} frsize={s_full.f_frsize}" + ) + except (PermissionError, FileNotFoundError, OSError): + return 0 + finally: + try: + mergerfs_set_option(mount, "statfs", orig_statfs) + mergerfs_set_option(mount, "statfs-ignore", orig_ignore) + except OSError: + pass + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_cfg_xattr_modes b/tests/TEST_cfg_xattr_modes new file mode 100755 index 00000000..3a0ea749 --- /dev/null +++ b/tests/TEST_cfg_xattr_modes @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import errno +import os +import sys +import tempfile + +from posix_parity import fail +from posix_parity import mergerfs_get_option +from posix_parity import mergerfs_set_option +from posix_parity import touch +from posix_parity import join + + +def get_errno(func): + try: + func() + return 0 + except OSError as exc: + return exc.errno + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_cfg_xattr_modes ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + path = join(mount, "cfg-xattr-modes/file") + xname = "user.mergerfs.cfg.test" + xval = b"v" + + touch(path, b"x") + + try: + orig = mergerfs_get_option(mount, "xattr") + except (PermissionError, FileNotFoundError, OSError): + return 0 + + try: + mergerfs_set_option(mount, "xattr", "passthrough") + set_err = get_errno(lambda: os.setxattr(path, xname, xval)) + get_err = get_errno(lambda: os.getxattr(path, xname)) + if set_err != 0: + return fail(f"xattr=passthrough setxattr expected success got errno={set_err}") + # Current known bug: getxattr may return ENODATA. We still enforce mode switch behavior. + if get_err not in (0, errno.ENODATA): + return fail(f"xattr=passthrough getxattr unexpected errno={get_err}") + + mergerfs_set_option(mount, "xattr", "noattr") + err = get_errno(lambda: os.getxattr(path, xname)) + if err != errno.ENODATA: + return fail(f"xattr=noattr getxattr expected ENODATA got errno={err}") + err = get_errno(lambda: os.setxattr(path, xname, xval)) + if err != errno.ENODATA: + return fail(f"xattr=noattr setxattr expected ENODATA got errno={err}") + + mergerfs_set_option(mount, "xattr", "nosys") + err = get_errno(lambda: os.getxattr(path, xname)) + if err != errno.ENOSYS: + return fail(f"xattr=nosys getxattr expected ENOSYS got errno={err}") + err = get_errno(lambda: os.listxattr(path)) + if err != errno.ENOSYS: + return fail(f"xattr=nosys listxattr expected ENOSYS got errno={err}") + + # In nosys mode runtime control should stop working. + err = get_errno(lambda: mergerfs_get_option(mount, "xattr")) + if err != errno.ENOSYS: + return fail(f"xattr=nosys runtime control expected ENOSYS got errno={err}") + except (PermissionError, FileNotFoundError, OSError): + return 0 + finally: + try: + mergerfs_set_option(mount, "xattr", orig) + except OSError: + pass + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_mount_lifecycle b/tests/TEST_mount_lifecycle new file mode 100755 index 00000000..57cacc8b --- /dev/null +++ b/tests/TEST_mount_lifecycle @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import os +import shutil +import subprocess +import sys +import tempfile + +from posix_parity import fail + + +def has_tool(name): + return shutil.which(name) is not None + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_mount_lifecycle ", file=sys.stderr) + return 1 + + # This is a soft integration harness check for init/destroy wiring. + # Skip when environment is not prepared for nested mount tests. + if os.geteuid() != 0: + return 0 + if not has_tool("mergerfs"): + return 0 + if not has_tool("fusermount") and not has_tool("fusermount3"): + return 0 + + with tempfile.TemporaryDirectory() as td: + b1 = os.path.join(td, "b1") + b2 = os.path.join(td, "b2") + mp = os.path.join(td, "mp") + os.makedirs(b1, exist_ok=True) + os.makedirs(b2, exist_ok=True) + os.makedirs(mp, exist_ok=True) + + cmd = [ + "mergerfs", + f"{b1}:{b2}", + mp, + "-o", + "defaults,allow_other,use_ino,category.create=mfs", + ] + + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if p.returncode != 0: + return 0 + + try: + with open(os.path.join(mp, "lifecycle-file"), "wb") as fp: + fp.write(b"ok") + if not os.path.exists(os.path.join(mp, "lifecycle-file")): + return fail("mount lifecycle: expected created file to exist") + finally: + um = shutil.which("fusermount3") or shutil.which("fusermount") + u = subprocess.run([um, "-u", mp], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if u.returncode != 0: + return fail("mount lifecycle: failed to unmount test mount") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_access b/tests/TEST_posix_access new file mode 100755 index 00000000..9e0f1e83 --- /dev/null +++ b/tests/TEST_posix_access @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import errno +import os +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_access +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_access ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-access/file") + native_file = join(native, "posix-access/file") + merge_notdir = join(mount, "posix-access/notdir") + native_notdir = join(native, "posix-access/notdir") + merge_missing = join(mount, "posix-access/missing") + native_missing = join(native, "posix-access/missing") + + cleanup_paths([merge_file, merge_notdir]) + + touch(merge_file, b"ok", 0o644) + touch(native_file, b"ok", 0o644) + touch(merge_notdir, b"x", 0o644) + touch(native_notdir, b"x", 0o644) + + err = compare_access("F_OK existing", merge_file, native_file, os.F_OK) + if err: + return fail(err) + + err = compare_access( + "F_OK missing", merge_missing, native_missing, os.F_OK, errno.ENOENT + ) + if err: + return fail(err) + + err = compare_access( + "X_OK non-directory prefix", + join(merge_notdir, "child"), + join(native_notdir, "child"), + os.X_OK, + errno.ENOTDIR, + ) + if err: + return fail(err) + + os.chmod(merge_file, 0) + os.chmod(native_file, 0) + + err = compare_access("R_OK unreadable", merge_file, native_file, os.R_OK) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_bmap b/tests/TEST_posix_bmap new file mode 100755 index 00000000..4cd49b19 --- /dev/null +++ b/tests/TEST_posix_bmap @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import errno +import fcntl +import os +import struct +import sys +import tempfile + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +FIBMAP = 1 + + +def fibmap(fd, block=0): + buf = struct.pack("i", block) + out = fcntl.ioctl(fd, FIBMAP, buf) + return struct.unpack("i", out)[0] + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_bmap ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-bmap/file") + native_file = join(native, "posix-bmap/file") + touch(merge_file, b"x" * 8192) + touch(native_file, b"x" * 8192) + + mfd = os.open(merge_file, os.O_RDONLY) + nfd = os.open(native_file, os.O_RDONLY) + try: + # FIBMAP often requires CAP_SYS_RAWIO; compare errno parity if denied. + err = compare_calls( + "ioctl FIBMAP block0", + lambda: fibmap(mfd, 0), + lambda: fibmap(nfd, 0), + lambda a, b: (a >= 0) == (b >= 0), + ) + if err: + # If both denied by privilege checks, compare_calls already passes. + return fail(err) + finally: + os.close(mfd) + os.close(nfd) + + # EBADF parity on closed fds. + err = compare_calls( + "ioctl FIBMAP EBADF", + lambda: fibmap(mfd, 0), + lambda: fibmap(nfd, 0), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_chmod b/tests/TEST_posix_chmod new file mode 100755 index 00000000..a7dd7baa --- /dev/null +++ b/tests/TEST_posix_chmod @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +import errno +import os +import stat +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import should_compare_inode +from posix_parity import stat_cmp_basic +from posix_parity import touch + + +def stat_cmp_basic_with_inode(lhs, rhs): + return stat_cmp_basic(lhs, rhs) and lhs.st_ino == rhs.st_ino + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_chmod ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + stcmp = stat_cmp_basic_with_inode if should_compare_inode(mount) else stat_cmp_basic + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-chmod/file") + native_file = join(native, "posix-chmod/file") + merge_missing = join(mount, "posix-chmod/missing") + native_missing = join(native, "posix-chmod/missing") + merge_notdir = join(mount, "posix-chmod/notdir") + native_notdir = join(native, "posix-chmod/notdir") + + cleanup_paths([merge_file, merge_notdir]) + + touch(merge_file, b"x", 0o644) + touch(native_file, b"x", 0o644) + touch(merge_notdir, b"x", 0o644) + touch(native_notdir, b"x", 0o644) + + err = compare_calls( + "chmod success", + lambda: os.chmod(merge_file, 0o600), + lambda: os.chmod(native_file, 0o600), + ) + if err: + return fail(err) + + err = compare_calls( + "chmod stat parity", + lambda: os.lstat(merge_file), + lambda: os.lstat(native_file), + stcmp, + ) + if err: + return fail(err) + + err = compare_calls( + "chmod ENOENT", + lambda: os.chmod(merge_missing, 0o600), + lambda: os.chmod(native_missing, 0o600), + ) + if err: + return fail(err) + + err = compare_calls( + "chmod ENOTDIR", + lambda: os.chmod(join(merge_notdir, "child"), 0o600), + lambda: os.chmod(join(native_notdir, "child"), 0o600), + ) + if err: + return fail(err) + + if os.geteuid() != 0: + err = compare_calls( + "chmod EPERM setuid", + lambda: os.chmod(merge_file, stat.S_ISUID | 0o644), + lambda: os.chmod(native_file, stat.S_ISUID | 0o644), + ) + if err: + return fail(err) + else: + _ = errno.EPERM + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_chown b/tests/TEST_posix_chown new file mode 100755 index 00000000..c989a958 --- /dev/null +++ b/tests/TEST_posix_chown @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_chown ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + uid = os.getuid() + gid = os.getgid() + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-chown/file") + native_file = join(native, "posix-chown/file") + merge_missing = join(mount, "posix-chown/missing") + native_missing = join(native, "posix-chown/missing") + merge_notdir = join(mount, "posix-chown/notdir") + native_notdir = join(native, "posix-chown/notdir") + + cleanup_paths([merge_file, merge_notdir]) + + touch(merge_file, b"x", 0o644) + touch(native_file, b"x", 0o644) + touch(merge_notdir, b"x", 0o644) + touch(native_notdir, b"x", 0o644) + + err = compare_calls( + "chown self", + lambda: os.chown(merge_file, uid, gid), + lambda: os.chown(native_file, uid, gid), + ) + if err: + return fail(err) + + err = compare_calls( + "chown ENOENT", + lambda: os.chown(merge_missing, uid, gid), + lambda: os.chown(native_missing, uid, gid), + ) + if err: + return fail(err) + + err = compare_calls( + "chown ENOTDIR", + lambda: os.chown(join(merge_notdir, "child"), uid, gid), + lambda: os.chown(join(native_notdir, "child"), uid, gid), + ) + if err: + return fail(err) + + if os.geteuid() != 0: + err = compare_calls( + "chown EPERM foreign uid", + lambda: os.chown(merge_file, 0, gid), + lambda: os.chown(native_file, 0, gid), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_copy_file_range b/tests/TEST_posix_copy_file_range new file mode 100755 index 00000000..cbd7df21 --- /dev/null +++ b/tests/TEST_posix_copy_file_range @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_copy_file_range ", file=sys.stderr) + return 1 + + if not hasattr(os, "copy_file_range"): + return 0 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_src = join(mount, "posix-cfr/src") + merge_dst = join(mount, "posix-cfr/dst") + native_src = join(native, "posix-cfr/src") + native_dst = join(native, "posix-cfr/dst") + + payload = b"0123456789abcdefghijklmnopqrstuvwxyz" + touch(merge_src, payload) + touch(native_src, payload) + touch(merge_dst, b"") + touch(native_dst, b"") + + msfd = os.open(merge_src, os.O_RDONLY) + mdfd = os.open(merge_dst, os.O_WRONLY) + nsfd = os.open(native_src, os.O_RDONLY) + ndfd = os.open(native_dst, os.O_WRONLY) + try: + err = compare_calls( + "copy_file_range success", + lambda: os.copy_file_range(msfd, mdfd, 16), + lambda: os.copy_file_range(nsfd, ndfd, 16), + lambda a, b: a == b, + ) + if err: + return fail(err) + finally: + os.close(msfd) + os.close(mdfd) + os.close(nsfd) + os.close(ndfd) + + with open(merge_dst, "rb") as mf, open(native_dst, "rb") as nf: + mdata = mf.read() + ndata = nf.read() + if mdata != ndata: + return fail(f"copy_file_range dst mismatch mergerfs={mdata!r} native={ndata!r}") + + mdfd = os.open(merge_dst, os.O_WRONLY) + ndfd = os.open(native_dst, os.O_WRONLY) + try: + err = compare_calls( + "copy_file_range EBADF src", + lambda: os.copy_file_range(-1, mdfd, 1), + lambda: os.copy_file_range(-1, ndfd, 1), + ) + if err: + return fail(err) + finally: + os.close(mdfd) + os.close(ndfd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_create_mknod b/tests/TEST_posix_create_mknod new file mode 100755 index 00000000..6f18e476 --- /dev/null +++ b/tests/TEST_posix_create_mknod @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +import os +import stat +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_create_mknod ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + os.makedirs(join(mount, "posix-create"), exist_ok=True) + os.makedirs(join(native, "posix-create"), exist_ok=True) + + merge_file = join(mount, "posix-create/file") + native_file = join(native, "posix-create/file") + merge_exist = join(mount, "posix-create/exist") + native_exist = join(native, "posix-create/exist") + merge_notdir = join(mount, "posix-create/notdir") + native_notdir = join(native, "posix-create/notdir") + + cleanup_paths([merge_file, merge_exist, merge_notdir]) + + err = compare_calls( + "create success", + lambda: os.open(merge_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640), + lambda: os.open(native_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640), + close_fds=True, + ) + if err: + return fail(err) + + with open(merge_exist, "wb"): + pass + with open(native_exist, "wb"): + pass + + err = compare_calls( + "create EEXIST", + lambda: os.open(merge_exist, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640), + lambda: os.open(native_exist, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o640), + ) + if err: + return fail(err) + + with open(merge_notdir, "wb"): + pass + with open(native_notdir, "wb"): + pass + + err = compare_calls( + "create ENOTDIR", + lambda: os.open(join(merge_notdir, "child"), os.O_CREAT | os.O_WRONLY, 0o640), + lambda: os.open(join(native_notdir, "child"), os.O_CREAT | os.O_WRONLY, 0o640), + ) + if err: + return fail(err) + + merge_node = join(mount, "posix-create/mknod") + native_node = join(native, "posix-create/mknod") + cleanup_paths([merge_node]) + + err = compare_calls( + "mknod regular", + lambda: os.mknod(merge_node, stat.S_IFREG | 0o600, 0), + lambda: os.mknod(native_node, stat.S_IFREG | 0o600, 0), + ) + if err: + return fail(err) + + err = compare_calls( + "mknod EEXIST", + lambda: os.mknod(merge_node, stat.S_IFREG | 0o600, 0), + lambda: os.mknod(native_node, stat.S_IFREG | 0o600, 0), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_fsync_flush b/tests/TEST_posix_fsync_flush new file mode 100755 index 00000000..79da35c9 --- /dev/null +++ b/tests/TEST_posix_fsync_flush @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_fsync_flush ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-fsync/file") + native_file = join(native, "posix-fsync/file") + merge_dir = join(mount, "posix-fsync/dir") + native_dir = join(native, "posix-fsync/dir") + + touch(merge_file, b"abc") + touch(native_file, b"abc") + os.makedirs(merge_dir, exist_ok=True) + os.makedirs(native_dir, exist_ok=True) + + mfd = os.open(merge_file, os.O_RDWR) + nfd = os.open(native_file, os.O_RDWR) + try: + os.lseek(mfd, 0, os.SEEK_END) + os.lseek(nfd, 0, os.SEEK_END) + os.write(mfd, b"-merge") + os.write(nfd, b"-native") + + err = compare_calls("fsync file", lambda: os.fsync(mfd), lambda: os.fsync(nfd)) + if err: + return fail(err) + finally: + os.close(mfd) + os.close(nfd) + + mdirfd = os.open(merge_dir, os.O_RDONLY | os.O_DIRECTORY) + ndirfd = os.open(native_dir, os.O_RDONLY | os.O_DIRECTORY) + try: + err = compare_calls("fsync dir", lambda: os.fsync(mdirfd), lambda: os.fsync(ndirfd)) + if err: + return fail(err) + finally: + os.close(mdirfd) + os.close(ndirfd) + + bad_m = os.open(merge_file, os.O_RDONLY) + bad_n = os.open(native_file, os.O_RDONLY) + os.close(bad_m) + os.close(bad_n) + err = compare_calls("fsync EBADF", lambda: os.fsync(bad_m), lambda: os.fsync(bad_n)) + if err: + return fail(err) + + # Close-to-flush parity check: both paths should preserve appended payload. + with open(merge_file, "rb") as mf, open(native_file, "rb") as nf: + mdata = mf.read() + ndata = nf.read() + if not (mdata.startswith(b"abc") and ndata.startswith(b"abc")): + return fail(f"flush content prefix mismatch mergerfs={mdata!r} native={ndata!r}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_getattr_fgetattr b/tests/TEST_posix_getattr_fgetattr new file mode 100755 index 00000000..f1ba2c4c --- /dev/null +++ b/tests/TEST_posix_getattr_fgetattr @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +import os +import stat +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import should_compare_inode +from posix_parity import stat_cmp_basic +from posix_parity import touch + + +def st_cmp(lhs, rhs): + return ( + stat_cmp_basic(lhs, rhs) + and stat.S_IFMT(lhs.st_mode) == stat.S_IFMT(rhs.st_mode) + and lhs.st_nlink == rhs.st_nlink + ) + + +def st_cmp_with_inode(lhs, rhs): + return st_cmp(lhs, rhs) and lhs.st_ino == rhs.st_ino + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_getattr_fgetattr ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + stcmp = st_cmp_with_inode if should_compare_inode(mount) else st_cmp + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-getattr/file") + native_file = join(native, "posix-getattr/file") + merge_dir = join(mount, "posix-getattr/dir") + native_dir = join(native, "posix-getattr/dir") + merge_missing = join(mount, "posix-getattr/missing") + native_missing = join(native, "posix-getattr/missing") + merge_notdir = join(mount, "posix-getattr/notdir") + native_notdir = join(native, "posix-getattr/notdir") + + cleanup_paths([merge_file, merge_dir, merge_notdir]) + + touch(merge_file, b"abc", 0o640) + touch(native_file, b"abc", 0o640) + os.makedirs(merge_dir, exist_ok=True) + os.makedirs(native_dir, exist_ok=True) + touch(merge_notdir, b"x", 0o644) + touch(native_notdir, b"x", 0o644) + + err = compare_calls( + "lstat regular", lambda: os.lstat(merge_file), lambda: os.lstat(native_file), stcmp + ) + if err: + return fail(err) + + err = compare_calls( + "lstat directory", lambda: os.lstat(merge_dir), lambda: os.lstat(native_dir), stcmp + ) + if err: + return fail(err) + + mfd = os.open(merge_file, os.O_RDONLY) + nfd = os.open(native_file, os.O_RDONLY) + try: + err = compare_calls("fstat open fd", lambda: os.fstat(mfd), lambda: os.fstat(nfd), stcmp) + if err: + return fail(err) + finally: + os.close(mfd) + os.close(nfd) + + err = compare_calls("lstat ENOENT", lambda: os.lstat(merge_missing), lambda: os.lstat(native_missing)) + if err: + return fail(err) + + err = compare_calls( + "lstat ENOTDIR", + lambda: os.lstat(join(merge_notdir, "child")), + lambda: os.lstat(join(native_notdir, "child")), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_ioctl b/tests/TEST_posix_ioctl new file mode 100755 index 00000000..8185da9b --- /dev/null +++ b/tests/TEST_posix_ioctl @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import array +import fcntl +import os +import termios +import sys +import tempfile + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def fionread(fd): + buf = array.array("i", [0]) + fcntl.ioctl(fd, termios.FIONREAD, buf, True) + return buf[0] + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_ioctl ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-ioctl/file") + native_file = join(native, "posix-ioctl/file") + touch(merge_file, b"hello-world") + touch(native_file, b"hello-world") + + mfd = os.open(merge_file, os.O_RDONLY) + nfd = os.open(native_file, os.O_RDONLY) + try: + err = compare_calls("ioctl FIONREAD", lambda: fionread(mfd), lambda: fionread(nfd), lambda a, b: a == b) + if err: + return fail(err) + + os.lseek(mfd, 5, os.SEEK_SET) + os.lseek(nfd, 5, os.SEEK_SET) + err = compare_calls( + "ioctl FIONREAD after seek", + lambda: fionread(mfd), + lambda: fionread(nfd), + lambda a, b: a == b, + ) + if err: + return fail(err) + finally: + os.close(mfd) + os.close(nfd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_link_symlink b/tests/TEST_posix_link_symlink new file mode 100755 index 00000000..f4642c19 --- /dev/null +++ b/tests/TEST_posix_link_symlink @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_link_symlink ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_src = join(mount, "posix-link/src") + native_src = join(native, "posix-link/src") + merge_dst = join(mount, "posix-link/dst") + native_dst = join(native, "posix-link/dst") + merge_slnk = join(mount, "posix-link/symlink") + native_slnk = join(native, "posix-link/symlink") + merge_missing = join(mount, "posix-link/missing") + native_missing = join(native, "posix-link/missing") + + cleanup_paths([merge_src, merge_dst, merge_slnk]) + touch(merge_src, b"src", 0o644) + touch(native_src, b"src", 0o644) + + err = compare_calls("link success", lambda: os.link(merge_src, merge_dst), lambda: os.link(native_src, native_dst)) + if err: + return fail(err) + + err = compare_calls( + "link ENOENT source", + lambda: os.link(merge_missing, join(mount, "posix-link/dst2")), + lambda: os.link(native_missing, join(native, "posix-link/dst2")), + ) + if err: + return fail(err) + + err = compare_calls( + "symlink success", + lambda: os.symlink("src", merge_slnk), + lambda: os.symlink("src", native_slnk), + ) + if err: + return fail(err) + + err = compare_calls( + "symlink EEXIST", + lambda: os.symlink("src", merge_slnk), + lambda: os.symlink("src", native_slnk), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_locking b/tests/TEST_posix_locking new file mode 100755 index 00000000..273faf84 --- /dev/null +++ b/tests/TEST_posix_locking @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +import fcntl +import os +import struct +import sys +import tempfile + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def pack_flock(l_type, l_whence=0, l_start=0, l_len=0, l_pid=0): + # struct flock (common glibc x86_64 layout) + return struct.pack("hhqqi", l_type, l_whence, l_start, l_len, l_pid) + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_locking ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-locking/file") + native_file = join(native, "posix-locking/file") + touch(merge_file, b"x") + touch(native_file, b"x") + + mfd1 = os.open(merge_file, os.O_RDWR) + mfd2 = os.open(merge_file, os.O_RDWR) + nfd1 = os.open(native_file, os.O_RDWR) + nfd2 = os.open(native_file, os.O_RDWR) + + try: + err = compare_calls( + "flock EX nonblock", + lambda: fcntl.flock(mfd1, fcntl.LOCK_EX | fcntl.LOCK_NB), + lambda: fcntl.flock(nfd1, fcntl.LOCK_EX | fcntl.LOCK_NB), + ) + if err: + return fail(err) + + err = compare_calls( + "flock SH nonblock second fd", + lambda: fcntl.flock(mfd2, fcntl.LOCK_SH | fcntl.LOCK_NB), + lambda: fcntl.flock(nfd2, fcntl.LOCK_SH | fcntl.LOCK_NB), + ) + if err: + return fail(err) + + err = compare_calls("flock unlock", lambda: fcntl.flock(mfd1, fcntl.LOCK_UN), lambda: fcntl.flock(nfd1, fcntl.LOCK_UN)) + if err: + return fail(err) + err = compare_calls("flock unlock second fd", lambda: fcntl.flock(mfd2, fcntl.LOCK_UN), lambda: fcntl.flock(nfd2, fcntl.LOCK_UN)) + if err: + return fail(err) + + # POSIX record lock via fcntl + wrlk = pack_flock(fcntl.F_WRLCK) + unlck = pack_flock(fcntl.F_UNLCK) + + err = compare_calls("fcntl F_SETLK write lock", lambda: fcntl.fcntl(mfd1, fcntl.F_SETLK, wrlk), lambda: fcntl.fcntl(nfd1, fcntl.F_SETLK, wrlk)) + if err: + return fail(err) + + err = compare_calls("fcntl F_SETLK write lock second fd", lambda: fcntl.fcntl(mfd2, fcntl.F_SETLK, wrlk), lambda: fcntl.fcntl(nfd2, fcntl.F_SETLK, wrlk)) + if err: + return fail(err) + + err = compare_calls("fcntl F_SETLK unlock", lambda: fcntl.fcntl(mfd1, fcntl.F_SETLK, unlck), lambda: fcntl.fcntl(nfd1, fcntl.F_SETLK, unlck)) + if err: + return fail(err) + + bad_m = os.open(merge_file, os.O_RDONLY) + bad_n = os.open(native_file, os.O_RDONLY) + os.close(bad_m) + os.close(bad_n) + err = compare_calls( + "fcntl EBADF", + lambda: fcntl.fcntl(bad_m, fcntl.F_SETLK, wrlk), + lambda: fcntl.fcntl(bad_n, fcntl.F_SETLK, wrlk), + ) + if err: + return fail(err) + finally: + os.close(mfd1) + os.close(mfd2) + os.close(nfd1) + os.close(nfd2) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_mkdir_rmdir b/tests/TEST_posix_mkdir_rmdir new file mode 100755 index 00000000..8b06e5f4 --- /dev/null +++ b/tests/TEST_posix_mkdir_rmdir @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_mkdir_rmdir ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_dir = join(mount, "posix-mkdir/dir") + native_dir = join(native, "posix-mkdir/dir") + merge_notdir = join(mount, "posix-mkdir/notdir") + native_notdir = join(native, "posix-mkdir/notdir") + + cleanup_paths([merge_dir, merge_notdir]) + os.makedirs(join(mount, "posix-mkdir"), exist_ok=True) + os.makedirs(join(native, "posix-mkdir"), exist_ok=True) + + err = compare_calls("mkdir success", lambda: os.mkdir(merge_dir, 0o755), lambda: os.mkdir(native_dir, 0o755)) + if err: + return fail(err) + + err = compare_calls("mkdir EEXIST", lambda: os.mkdir(merge_dir, 0o755), lambda: os.mkdir(native_dir, 0o755)) + if err: + return fail(err) + + touch(merge_notdir, b"x") + touch(native_notdir, b"x") + err = compare_calls( + "mkdir ENOTDIR", + lambda: os.mkdir(join(merge_notdir, "child"), 0o755), + lambda: os.mkdir(join(native_notdir, "child"), 0o755), + ) + if err: + return fail(err) + + err = compare_calls("rmdir success", lambda: os.rmdir(merge_dir), lambda: os.rmdir(native_dir)) + if err: + return fail(err) + + err = compare_calls("rmdir ENOENT", lambda: os.rmdir(merge_dir), lambda: os.rmdir(native_dir)) + if err: + return fail(err) + + touch(join(mount, "posix-mkdir/nonempty/file"), b"x") + touch(join(native, "posix-mkdir/nonempty/file"), b"x") + err = compare_calls( + "rmdir ENOTEMPTY", + lambda: os.rmdir(join(mount, "posix-mkdir/nonempty")), + lambda: os.rmdir(join(native, "posix-mkdir/nonempty")), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_open_read_write b/tests/TEST_posix_open_read_write new file mode 100755 index 00000000..3c0e2af9 --- /dev/null +++ b/tests/TEST_posix_open_read_write @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_open_read_write ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-rw/file") + native_file = join(native, "posix-rw/file") + merge_missing = join(mount, "posix-rw/missing") + native_missing = join(native, "posix-rw/missing") + merge_dir = join(mount, "posix-rw/dir") + native_dir = join(native, "posix-rw/dir") + merge_notdir = join(mount, "posix-rw/notdir") + native_notdir = join(native, "posix-rw/notdir") + + cleanup_paths([merge_file, merge_notdir]) + touch(merge_file, b"123456", 0o644) + touch(native_file, b"123456", 0o644) + os.makedirs(merge_dir, exist_ok=True) + os.makedirs(native_dir, exist_ok=True) + touch(merge_notdir, b"x", 0o644) + touch(native_notdir, b"x", 0o644) + + err = compare_calls("open ENOENT", lambda: os.open(merge_missing, os.O_RDONLY), lambda: os.open(native_missing, os.O_RDONLY)) + if err: + return fail(err) + + err = compare_calls("open EISDIR", lambda: os.open(merge_dir, os.O_WRONLY), lambda: os.open(native_dir, os.O_WRONLY)) + if err: + return fail(err) + + err = compare_calls( + "open ENOTDIR", + lambda: os.open(join(merge_notdir, "child"), os.O_RDONLY), + lambda: os.open(join(native_notdir, "child"), os.O_RDONLY), + ) + if err: + return fail(err) + + mfd = os.open(merge_file, os.O_RDWR) + nfd = os.open(native_file, os.O_RDWR) + try: + os.lseek(mfd, 0, os.SEEK_SET) + os.lseek(nfd, 0, os.SEEK_SET) + m_data = os.read(mfd, 6) + n_data = os.read(nfd, 6) + if m_data != n_data: + return fail(f"read parity mismatch mergerfs={m_data!r} native={n_data!r}") + + os.lseek(mfd, 0, os.SEEK_SET) + os.lseek(nfd, 0, os.SEEK_SET) + mw = os.write(mfd, b"abc") + nw = os.write(nfd, b"abc") + if mw != nw: + return fail(f"write count mismatch mergerfs={mw} native={nw}") + finally: + os.close(mfd) + os.close(nfd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_poll b/tests/TEST_posix_poll new file mode 100755 index 00000000..84b1fbec --- /dev/null +++ b/tests/TEST_posix_poll @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import os +import select +import sys +import tempfile + +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def poll_mask(fd, timeout_ms=25): + p = select.poll() + p.register(fd, select.POLLIN | select.POLLOUT | select.POLLERR | select.POLLHUP) + events = p.poll(timeout_ms) + mask = 0 + for _fd, ev in events: + mask |= ev + return mask + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_poll ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-poll/file") + native_file = join(native, "posix-poll/file") + touch(merge_file, b"abcdef") + touch(native_file, b"abcdef") + + mfd = os.open(merge_file, os.O_RDONLY) + nfd = os.open(native_file, os.O_RDONLY) + try: + mmask = poll_mask(mfd) + nmask = poll_mask(nfd) + + # For regular files POLLIN and POLLOUT are typically ready. + if bool(mmask & select.POLLIN) != bool(nmask & select.POLLIN): + return fail(f"poll POLLIN mismatch mergerfs_mask=0x{mmask:x} native_mask=0x{nmask:x}") + if bool(mmask & select.POLLOUT) != bool(nmask & select.POLLOUT): + return fail(f"poll POLLOUT mismatch mergerfs_mask=0x{mmask:x} native_mask=0x{nmask:x}") + finally: + os.close(mfd) + os.close(nfd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_readdir b/tests/TEST_posix_readdir new file mode 100755 index 00000000..aa3e010e --- /dev/null +++ b/tests/TEST_posix_readdir @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +import ctypes +import os +import sys +import tempfile + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +class Dirent(ctypes.Structure): + _fields_ = [ + ("d_ino", ctypes.c_ulong), + ("d_off", ctypes.c_long), + ("d_reclen", ctypes.c_ushort), + ("d_type", ctypes.c_ubyte), + ("d_name", ctypes.c_char * 256), + ] + + +libc = ctypes.CDLL(None, use_errno=True) +libc.opendir.argtypes = [ctypes.c_char_p] +libc.opendir.restype = ctypes.c_void_p +libc.readdir.argtypes = [ctypes.c_void_p] +libc.readdir.restype = ctypes.POINTER(Dirent) +libc.telldir.argtypes = [ctypes.c_void_p] +libc.telldir.restype = ctypes.c_long +libc.seekdir.argtypes = [ctypes.c_void_p, ctypes.c_long] +libc.seekdir.restype = None +libc.rewinddir.argtypes = [ctypes.c_void_p] +libc.rewinddir.restype = None +libc.closedir.argtypes = [ctypes.c_void_p] +libc.closedir.restype = ctypes.c_int + + +def opendir_or_raise(path): + ctypes.set_errno(0) + dp = libc.opendir(path.encode()) + if not dp: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err), path) + return dp + + +def decode_name(ent): + return bytes(ent.d_name).split(b"\0", 1)[0].decode("utf-8", errors="surrogateescape") + + +def read_all_names(dp): + names = [] + while True: + entp = libc.readdir(dp) + if not entp: + break + names.append(decode_name(entp.contents)) + return names + + +def next_non_dot(dp): + while True: + entp = libc.readdir(dp) + if not entp: + return None + name = decode_name(entp.contents) + if name not in (".", ".."): + return name + + +def names_set(path): + dp = opendir_or_raise(path) + try: + return set(n for n in read_all_names(dp) if n not in (".", "..")) + finally: + libc.closedir(dp) + + +def seek_tell_consistent(path): + dp = opendir_or_raise(path) + try: + _first = next_non_dot(dp) + if _first is None: + return True + + pos = libc.telldir(dp) + second = next_non_dot(dp) + if second is None: + return True + + libc.seekdir(dp, pos) + again = next_non_dot(dp) + return second == again + finally: + libc.closedir(dp) + + +def deleted_visibility_signature(dirpath, victim_path): + dp = opendir_or_raise(dirpath) + try: + before = set(n for n in read_all_names(dp) if n not in (".", "..")) + os.unlink(victim_path) + libc.rewinddir(dp) + after = set(n for n in read_all_names(dp) if n not in (".", "..")) + return ("victim" in before, "victim" in after) + finally: + libc.closedir(dp) + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_readdir ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_dir = join(mount, "posix-readdir/dir") + native_dir = join(native, "posix-readdir/dir") + merge_notdir = join(mount, "posix-readdir/notdir") + native_notdir = join(native, "posix-readdir/notdir") + + for idx in range(8): + touch(join(merge_dir, f"f{idx}"), b"x") + touch(join(native_dir, f"f{idx}"), b"x") + + touch(merge_notdir, b"x") + touch(native_notdir, b"x") + + err = compare_calls( + "opendir ENOENT", + lambda: opendir_or_raise(join(mount, "posix-readdir/missing")), + lambda: opendir_or_raise(join(native, "posix-readdir/missing")), + ) + if err: + return fail(err) + + err = compare_calls( + "opendir ENOTDIR", + lambda: opendir_or_raise(join(merge_notdir, "child")), + lambda: opendir_or_raise(join(native_notdir, "child")), + ) + if err: + return fail(err) + + merge_names = names_set(merge_dir) + native_names = names_set(native_dir) + if merge_names != native_names: + return fail(f"readdir names mismatch mergerfs={sorted(merge_names)} native={sorted(native_names)}") + + merge_consistent = seek_tell_consistent(merge_dir) + native_consistent = seek_tell_consistent(native_dir) + if merge_consistent != native_consistent: + return fail( + f"seekdir/telldir consistency mismatch mergerfs={merge_consistent} native={native_consistent}" + ) + + touch(join(merge_dir, "victim"), b"x") + touch(join(native_dir, "victim"), b"x") + merge_sig = deleted_visibility_signature(merge_dir, join(merge_dir, "victim")) + native_sig = deleted_visibility_signature(native_dir, join(native_dir, "victim")) + if merge_sig != native_sig: + return fail(f"deleted-entry visibility mismatch mergerfs={merge_sig} native={native_sig}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_readdir_plus b/tests/TEST_posix_readdir_plus new file mode 100755 index 00000000..f723a12b --- /dev/null +++ b/tests/TEST_posix_readdir_plus @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import fail +from posix_parity import join +from posix_parity import mergerfs_get_option +from posix_parity import mergerfs_set_option +from posix_parity import touch + + +def list_and_stat(dirpath): + out = {} + for name in os.listdir(dirpath): + if name in (".", ".."): + continue + st = os.lstat(os.path.join(dirpath, name)) + out[name] = (st.st_mode & 0xF000, st.st_size) + return out + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_readdir_plus ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_dir = join(mount, "posix-readdir-plus/dir") + native_dir = join(native, "posix-readdir-plus/dir") + + for i in range(40): + touch(join(merge_dir, f"f{i:03d}"), bytes(str(i), "ascii")) + touch(join(native_dir, f"f{i:03d}"), bytes(str(i), "ascii")) + + # Exercise mapping from readdir entries to immediate getattr calls. + m_map = list_and_stat(merge_dir) + n_map = list_and_stat(native_dir) + if m_map != n_map: + return fail("readdir+getattr map mismatch between mergerfs and native") + + # Optionally toggle readdir policy and ensure semantic output stays stable. + try: + orig = mergerfs_get_option(mount, "func.readdir") + except (PermissionError, FileNotFoundError, OSError): + return 0 + + try: + for mode in ("seq", "cosr:2:2", "cor:2:2"): + mergerfs_set_option(mount, "func.readdir", mode) + got = list_and_stat(merge_dir) + if got != m_map: + return fail(f"func.readdir={mode} changed readdir+getattr semantic output") + except (PermissionError, FileNotFoundError, OSError): + return 0 + finally: + try: + mergerfs_set_option(mount, "func.readdir", orig) + except OSError: + pass + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_release b/tests/TEST_posix_release new file mode 100755 index 00000000..27c31ff7 --- /dev/null +++ b/tests/TEST_posix_release @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_release ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-release/file") + native_file = join(native, "posix-release/file") + + touch(merge_file, b"abc") + touch(native_file, b"abc") + + # Open + dup so one close does not release the underlying open-file state. + mfd1 = os.open(merge_file, os.O_RDWR) + nfd1 = os.open(native_file, os.O_RDWR) + mfd2 = os.dup(mfd1) + nfd2 = os.dup(nfd1) + + try: + os.close(mfd1) + os.close(nfd1) + + mw = os.write(mfd2, b"X") + nw = os.write(nfd2, b"X") + if mw != nw: + return fail(f"release dup write mismatch mergerfs={mw} native={nw}") + + os.lseek(mfd2, 0, os.SEEK_SET) + os.lseek(nfd2, 0, os.SEEK_SET) + mr = os.read(mfd2, 4) + nr = os.read(nfd2, 4) + if mr != nr: + return fail(f"release dup read mismatch mergerfs={mr!r} native={nr!r}") + finally: + os.close(mfd2) + os.close(nfd2) + + with open(merge_file, "rb") as mf, open(native_file, "rb") as nf: + md = mf.read() + nd = nf.read() + if md != nd: + return fail(f"release final data mismatch mergerfs={md!r} native={nd!r}") + + # Closing same fd twice should map to EBADF on both. + mfd = os.open(merge_file, os.O_RDONLY) + nfd = os.open(native_file, os.O_RDONLY) + os.close(mfd) + os.close(nfd) + try: + os.close(mfd) + m_err = 0 + except OSError as exc: + m_err = exc.errno + try: + os.close(nfd) + n_err = 0 + except OSError as exc: + n_err = exc.errno + if m_err != n_err: + return fail(f"release EBADF mismatch mergerfs_errno={m_err} native_errno={n_err}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_releasedir b/tests/TEST_posix_releasedir new file mode 100755 index 00000000..bcb52175 --- /dev/null +++ b/tests/TEST_posix_releasedir @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import ctypes +import os +import sys +import tempfile + +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +libc = ctypes.CDLL(None, use_errno=True) +libc.opendir.argtypes = [ctypes.c_char_p] +libc.opendir.restype = ctypes.c_void_p +libc.dirfd.argtypes = [ctypes.c_void_p] +libc.dirfd.restype = ctypes.c_int +libc.closedir.argtypes = [ctypes.c_void_p] +libc.closedir.restype = ctypes.c_int + + +def opendir(path): + ctypes.set_errno(0) + dp = libc.opendir(path.encode()) + if not dp: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err), path) + return dp + + +def close_and_dirfd_errno(path): + dp = opendir(path) + fd = libc.dirfd(dp) + if fd < 0: + libc.closedir(dp) + raise OSError(ctypes.get_errno(), "dirfd failed", path) + + rv = libc.closedir(dp) + if rv != 0: + raise OSError(ctypes.get_errno(), "closedir failed", path) + + try: + os.fstat(fd) + return 0 + except OSError as exc: + return exc.errno + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_releasedir ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_dir = join(mount, "posix-releasedir/dir") + native_dir = join(native, "posix-releasedir/dir") + touch(join(merge_dir, "a"), b"x") + touch(join(native_dir, "a"), b"x") + + m_err = close_and_dirfd_errno(merge_dir) + n_err = close_and_dirfd_errno(native_dir) + if m_err != n_err: + return fail(f"releasedir EBADF parity mismatch mergerfs_errno={m_err} native_errno={n_err}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_statfs b/tests/TEST_posix_statfs new file mode 100755 index 00000000..96e38c9a --- /dev/null +++ b/tests/TEST_posix_statfs @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join + + +def statvfs_cmp(lhs, rhs): + return ( + lhs.f_namemax == rhs.f_namemax + and lhs.f_bsize > 0 + and rhs.f_bsize > 0 + and lhs.f_frsize > 0 + and rhs.f_frsize > 0 + ) + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_statfs ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + err = compare_calls( + "statvfs mount parity", + lambda: os.statvfs(mount), + lambda: os.statvfs(native), + statvfs_cmp, + ) + if err: + return fail(err) + + err = compare_calls( + "statvfs ENOENT", + lambda: os.statvfs(join(mount, "posix-statfs/missing")), + lambda: os.statvfs(join(native, "posix-statfs/missing")), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_statx b/tests/TEST_posix_statx new file mode 100755 index 00000000..f98b73f2 --- /dev/null +++ b/tests/TEST_posix_statx @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 + +import ctypes +import errno +import os +import sys + +from posix_parity import fail +from posix_parity import join +from posix_parity import mergerfs_fullpath +from posix_parity import should_compare_inode +from posix_parity import touch + + +AT_FDCWD = -100 +AT_SYMLINK_NOFOLLOW = 0x100 +STATX_TYPE = 0x0001 +STATX_MODE = 0x0002 +STATX_NLINK = 0x0004 +STATX_UID = 0x0008 +STATX_GID = 0x0010 +STATX_SIZE = 0x0200 +STATX_BASIC_STATS = STATX_TYPE | STATX_MODE | STATX_NLINK | STATX_UID | STATX_GID | STATX_SIZE + + +class StatxTimestamp(ctypes.Structure): + _fields_ = [ + ("tv_sec", ctypes.c_longlong), + ("tv_nsec", ctypes.c_uint), + ("__reserved", ctypes.c_int), + ] + + +class Statx(ctypes.Structure): + _fields_ = [ + ("stx_mask", ctypes.c_uint), + ("stx_blksize", ctypes.c_uint), + ("stx_attributes", ctypes.c_ulonglong), + ("stx_nlink", ctypes.c_uint), + ("stx_uid", ctypes.c_uint), + ("stx_gid", ctypes.c_uint), + ("stx_mode", ctypes.c_ushort), + ("__spare0", ctypes.c_ushort), + ("stx_ino", ctypes.c_ulonglong), + ("stx_size", ctypes.c_ulonglong), + ("stx_blocks", ctypes.c_ulonglong), + ("stx_attributes_mask", ctypes.c_ulonglong), + ("stx_atime", StatxTimestamp), + ("stx_btime", StatxTimestamp), + ("stx_ctime", StatxTimestamp), + ("stx_mtime", StatxTimestamp), + ("stx_rdev_major", ctypes.c_uint), + ("stx_rdev_minor", ctypes.c_uint), + ("stx_dev_major", ctypes.c_uint), + ("stx_dev_minor", ctypes.c_uint), + ("stx_mnt_id", ctypes.c_ulonglong), + ("stx_dio_mem_align", ctypes.c_uint), + ("stx_dio_offset_align", ctypes.c_uint), + ("stx_subvol", ctypes.c_ulonglong), + ("stx_atomic_write_unit_min", ctypes.c_uint), + ("stx_atomic_write_unit_max", ctypes.c_uint), + ("stx_atomic_write_segments_max", ctypes.c_uint), + ("stx_dio_read_offset_align", ctypes.c_uint), + ("__spare3", ctypes.c_ulonglong * 9), + ] + + +libc = ctypes.CDLL(None, use_errno=True) +if not hasattr(libc, "statx"): + raise SystemExit(0) + +libc.statx.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_uint, ctypes.POINTER(Statx)] +libc.statx.restype = ctypes.c_int + + +def statx_call(path, flags=0, mask=STATX_BASIC_STATS): + st = Statx() + ctypes.set_errno(0) + rv = libc.statx(AT_FDCWD, path.encode(), flags, mask, ctypes.byref(st)) + err = ctypes.get_errno() + if rv < 0: + raise OSError(err, os.strerror(err), path) + return st + + +def errno_name(err): + if err is None: + return "None" + return errno.errorcode.get(err, str(err)) + + +def statx_summary(st): + return ( + "{" + f"mask=0x{st.stx_mask:x}," + f"mode={st.stx_mode:o}," + f"type=0x{(st.stx_mode & 0xF000):x}," + f"perm=0o{(st.stx_mode & 0x0FFF):o}," + f"nlink={st.stx_nlink}," + f"uid={st.stx_uid}," + f"gid={st.stx_gid}," + f"size={st.stx_size}," + f"ino={st.stx_ino}," + f"blocks={st.stx_blocks}," + f"blksize={st.stx_blksize}" + "}" + ) + + +def call_pair(name, merge_call, native_call): + try: + m_val = merge_call() + m_err = None + except OSError as exc: + m_val = None + m_err = exc.errno + + try: + n_val = native_call() + n_err = None + except OSError as exc: + n_val = None + n_err = exc.errno + + if m_err != n_err: + return None, None, ( + f"{name}: errno mismatch\n" + f" mergerfs errno: {m_err} ({errno_name(m_err)})\n" + f" native errno: {n_err} ({errno_name(n_err)})" + ) + if m_err is not None: + return None, None, None + + return m_val, n_val, None + + +def cmp_statx_basic(a, b): + return ( + (a.stx_mode & 0xF000) == (b.stx_mode & 0xF000) + and (a.stx_mode & 0x0FFF) == (b.stx_mode & 0x0FFF) + and a.stx_nlink == b.stx_nlink + and a.stx_uid == b.stx_uid + and a.stx_gid == b.stx_gid + and a.stx_size == b.stx_size + ) + + +def cmp_statx_basic_with_inode(a, b): + return cmp_statx_basic(a, b) and a.stx_ino == b.stx_ino + + +def expect_same_errno(name, mcall, ncall): + m_err = None + n_err = None + try: + mcall() + except OSError as exc: + m_err = exc.errno + try: + ncall() + except OSError as exc: + n_err = exc.errno + if m_err != n_err: + return ( + f"{name}: errno mismatch\n" + f" mergerfs errno: {m_err} ({errno_name(m_err)})\n" + f" native errno: {n_err} ({errno_name(n_err)})" + ) + return None + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_statx ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + stcmp = cmp_statx_basic_with_inode if should_compare_inode(mount) else cmp_statx_basic + + merge_file = join(mount, "posix-statx/file") + merge_link = join(mount, "posix-statx/link") + + touch(merge_file, b"hello") + + try: + native_file = mergerfs_fullpath(merge_file) + except OSError: + return 0 + + try: + os.unlink(merge_link) + except FileNotFoundError: + pass + os.symlink("file", merge_link) + + try: + native_link = mergerfs_fullpath(merge_link) + except OSError: + return 0 + + # user.mergerfs.fullpath for a symlink may resolve to the target file path + # depending on policy behavior. For AT_SYMLINK_NOFOLLOW comparisons we need + # the underlying symlink path itself. + expected_native_link = os.path.join(os.path.dirname(native_file), "link") + if not os.path.islink(native_link): + if os.path.islink(expected_native_link): + native_link = expected_native_link + else: + return 0 + + native_missing = os.path.join(os.path.dirname(native_file), "missing") + + mst, nst, err = call_pair( + "statx regular", + lambda: statx_call(merge_file), + lambda: statx_call(native_file), + ) + if err: + return fail(err) + if not stcmp(mst, nst): + return fail( + "statx basic mismatch\n" + f" mergerfs path: {merge_file}\n" + f" native path: {native_file}\n" + f" mergerfs statx: {statx_summary(mst)}\n" + f" native statx: {statx_summary(nst)}" + ) + + mst, nst, err = call_pair( + "statx symlink nofollow", + lambda: statx_call(merge_link, flags=AT_SYMLINK_NOFOLLOW), + lambda: statx_call(native_link, flags=AT_SYMLINK_NOFOLLOW), + ) + if err: + return fail(err) + if (mst.stx_mode & 0xF000) != (nst.stx_mode & 0xF000): + return fail( + "statx symlink type mismatch\n" + f" mergerfs path: {merge_link}\n" + f" native path: {native_link}\n" + f" mergerfs statx: {statx_summary(mst)}\n" + f" native statx: {statx_summary(nst)}" + ) + + err = expect_same_errno( + "statx ENOENT", + lambda: statx_call(join(mount, "posix-statx/missing")), + lambda: statx_call(native_missing), + ) + if err: + return fail(err) + + err = expect_same_errno( + "statx ENOTDIR", + lambda: statx_call(join(merge_file, "child")), + lambda: statx_call(join(native_file, "child")), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_syncfs b/tests/TEST_posix_syncfs new file mode 100755 index 00000000..8ad2b1d6 --- /dev/null +++ b/tests/TEST_posix_syncfs @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import ctypes +import os +import sys +import tempfile + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +libc = ctypes.CDLL(None, use_errno=True) +libc.syncfs.argtypes = [ctypes.c_int] +libc.syncfs.restype = ctypes.c_int + + +def syncfs_fd(fd): + ctypes.set_errno(0) + rv = libc.syncfs(fd) + if rv == 0: + return 0 + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_syncfs ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-syncfs/file") + native_file = join(native, "posix-syncfs/file") + touch(merge_file, b"x") + touch(native_file, b"x") + + mfd = os.open(merge_file, os.O_RDONLY) + nfd = os.open(native_file, os.O_RDONLY) + try: + err = compare_calls("syncfs success", lambda: syncfs_fd(mfd), lambda: syncfs_fd(nfd)) + if err: + return fail(err) + finally: + os.close(mfd) + os.close(nfd) + + err = compare_calls("syncfs EBADF", lambda: syncfs_fd(-1), lambda: syncfs_fd(-1)) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_tmpfile b/tests/TEST_posix_tmpfile new file mode 100755 index 00000000..054424b8 --- /dev/null +++ b/tests/TEST_posix_tmpfile @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +import errno +import os +import sys +import tempfile + +from posix_parity import fail +from posix_parity import join + + +def open_tmpfile(dirpath): + return os.open(dirpath, os.O_TMPFILE | os.O_RDWR, 0o600) + + +def errno_of(call): + try: + call() + return 0 + except OSError as exc: + return exc.errno + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_tmpfile ", file=sys.stderr) + return 1 + + if not hasattr(os, "O_TMPFILE"): + return 0 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_dir = join(mount, "posix-tmpfile") + native_dir = join(native, "posix-tmpfile") + os.makedirs(merge_dir, exist_ok=True) + os.makedirs(native_dir, exist_ok=True) + + m_err = errno_of(lambda: open_tmpfile(merge_dir)) + n_err = errno_of(lambda: open_tmpfile(native_dir)) + if m_err != n_err: + return fail(f"O_TMPFILE support mismatch mergerfs_errno={m_err} native_errno={n_err}") + + # If unsupported on both, treat as skip-pass. + if m_err in (errno.EOPNOTSUPP, errno.EISDIR, errno.EINVAL, errno.ENOSYS): + return 0 + + mfd = open_tmpfile(merge_dir) + nfd = open_tmpfile(native_dir) + try: + mw = os.write(mfd, b"tmpfile-data") + nw = os.write(nfd, b"tmpfile-data") + if mw != nw: + return fail(f"O_TMPFILE write count mismatch mergerfs={mw} native={nw}") + finally: + os.close(mfd) + os.close(nfd) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_truncate_ftruncate b/tests/TEST_posix_truncate_ftruncate new file mode 100755 index 00000000..4fe6fad4 --- /dev/null +++ b/tests/TEST_posix_truncate_ftruncate @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_truncate_ftruncate ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-truncate/file") + native_file = join(native, "posix-truncate/file") + merge_missing = join(mount, "posix-truncate/missing") + native_missing = join(native, "posix-truncate/missing") + + cleanup_paths([merge_file]) + touch(merge_file, b"1234567890") + touch(native_file, b"1234567890") + + err = compare_calls("truncate shrink", lambda: os.truncate(merge_file, 3), lambda: os.truncate(native_file, 3)) + if err: + return fail(err) + + err = compare_calls("truncate ENOENT", lambda: os.truncate(merge_missing, 1), lambda: os.truncate(native_missing, 1)) + if err: + return fail(err) + + mfd = os.open(merge_file, os.O_RDWR) + nfd = os.open(native_file, os.O_RDWR) + try: + err = compare_calls("ftruncate grow", lambda: os.ftruncate(mfd, 16), lambda: os.ftruncate(nfd, 16)) + if err: + return fail(err) + finally: + os.close(mfd) + os.close(nfd) + + err = compare_calls("ftruncate EBADF", lambda: os.ftruncate(-1, 4), lambda: os.ftruncate(-1, 4)) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_unlink_rename b/tests/TEST_posix_unlink_rename new file mode 100755 index 00000000..2c4621d9 --- /dev/null +++ b/tests/TEST_posix_unlink_rename @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_unlink_rename ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_a = join(mount, "posix-unlink-rename/a") + native_a = join(native, "posix-unlink-rename/a") + merge_b = join(mount, "posix-unlink-rename/b") + native_b = join(native, "posix-unlink-rename/b") + merge_missing = join(mount, "posix-unlink-rename/missing") + native_missing = join(native, "posix-unlink-rename/missing") + merge_notdir = join(mount, "posix-unlink-rename/notdir") + native_notdir = join(native, "posix-unlink-rename/notdir") + + cleanup_paths([merge_a, merge_b, merge_notdir]) + touch(merge_a, b"a") + touch(native_a, b"a") + + err = compare_calls("unlink success", lambda: os.unlink(merge_a), lambda: os.unlink(native_a)) + if err: + return fail(err) + + err = compare_calls("unlink ENOENT", lambda: os.unlink(merge_missing), lambda: os.unlink(native_missing)) + if err: + return fail(err) + + touch(merge_a, b"a") + touch(native_a, b"a") + touch(merge_b, b"b") + touch(native_b, b"b") + + err = compare_calls("rename overwrite", lambda: os.rename(merge_a, merge_b), lambda: os.rename(native_a, native_b)) + if err: + return fail(err) + + err = compare_calls("rename ENOENT", lambda: os.rename(merge_missing, merge_a), lambda: os.rename(native_missing, native_a)) + if err: + return fail(err) + + touch(merge_notdir, b"x") + touch(native_notdir, b"x") + err = compare_calls( + "rename ENOTDIR", + lambda: os.rename(join(merge_notdir, "child"), merge_a), + lambda: os.rename(join(native_notdir, "child"), native_a), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_utimens b/tests/TEST_posix_utimens new file mode 100755 index 00000000..59796d44 --- /dev/null +++ b/tests/TEST_posix_utimens @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile +import time + +from posix_parity import cleanup_paths +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_utimens ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-utimens/file") + native_file = join(native, "posix-utimens/file") + merge_missing = join(mount, "posix-utimens/missing") + native_missing = join(native, "posix-utimens/missing") + + cleanup_paths([merge_file]) + touch(merge_file, b"x") + touch(native_file, b"x") + + now = time.time() + times = (now - 1000, now - 500) + + err = compare_calls( + "utime path success", + lambda: os.utime(merge_file, times=times), + lambda: os.utime(native_file, times=times), + ) + if err: + return fail(err) + + mfd = os.open(merge_file, os.O_RDWR) + nfd = os.open(native_file, os.O_RDWR) + try: + err = compare_calls( + "utime fd success", + lambda: os.utime(mfd, times=times), + lambda: os.utime(nfd, times=times), + ) + if err: + return fail(err) + finally: + os.close(mfd) + os.close(nfd) + + err = compare_calls( + "utime ENOENT", + lambda: os.utime(merge_missing, times=times), + lambda: os.utime(native_missing, times=times), + ) + if err: + return fail(err) + + err = compare_calls( + "utime EBADF", + lambda: os.utime(-1, times=times), + lambda: os.utime(-1, times=times), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_xattr b/tests/TEST_posix_xattr new file mode 100755 index 00000000..a2762b37 --- /dev/null +++ b/tests/TEST_posix_xattr @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 + +import os +import sys +import tempfile +import errno + +from posix_parity import compare_calls +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def errno_name(err): + if err is None: + return "None" + return errno.errorcode.get(err, str(err)) + + +def format_oserror(prefix, exc): + return f"{prefix}: errno={exc.errno}({errno_name(exc.errno)}) msg={exc.strerror}" + + +def has_attr(name): + def _cmp(lhs, rhs): + return (name in lhs) == (name in rhs) + + return _cmp + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_xattr ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + xname = "user.mergerfs.posix" + xvalue = b"parity-check" + + with tempfile.TemporaryDirectory() as native: + merge_file = join(mount, "posix-xattr/file") + native_file = join(native, "posix-xattr/file") + merge_missing = join(mount, "posix-xattr/missing") + native_missing = join(native, "posix-xattr/missing") + merge_notdir = join(mount, "posix-xattr/notdir") + native_notdir = join(native, "posix-xattr/notdir") + + touch(merge_file, b"x") + touch(native_file, b"x") + touch(merge_notdir, b"x") + touch(native_notdir, b"x") + + err = compare_calls( + "setxattr success", + lambda: os.setxattr(merge_file, xname, xvalue), + lambda: os.setxattr(native_file, xname, xvalue), + ) + if err: + try: + os.setxattr(merge_file, xname, xvalue) + except OSError as exc: + err += " | " + format_oserror("mergerfs setxattr", exc) + try: + os.setxattr(native_file, xname, xvalue) + except OSError as exc: + err += " | " + format_oserror("native setxattr", exc) + return fail(err) + + err = compare_calls( + "getxattr success", + lambda: os.getxattr(merge_file, xname), + lambda: os.getxattr(native_file, xname), + lambda lhs, rhs: lhs == rhs, + ) + if err: + try: + os.getxattr(merge_file, xname) + except OSError as exc: + err += " | " + format_oserror("mergerfs getxattr", exc) + try: + os.getxattr(native_file, xname) + except OSError as exc: + err += " | " + format_oserror("native getxattr", exc) + return fail(err) + + err = compare_calls( + "listxattr includes key", + lambda: os.listxattr(merge_file), + lambda: os.listxattr(native_file), + has_attr(xname), + ) + if err: + try: + m_list = os.listxattr(merge_file) + err += f" | mergerfs list={m_list!r}" + except OSError as exc: + err += " | " + format_oserror("mergerfs listxattr", exc) + try: + n_list = os.listxattr(native_file) + err += f" | native list={n_list!r}" + except OSError as exc: + err += " | " + format_oserror("native listxattr", exc) + return fail(err) + + err = compare_calls( + "removexattr success", + lambda: os.removexattr(merge_file, xname), + lambda: os.removexattr(native_file, xname), + ) + if err: + try: + os.removexattr(merge_file, xname) + except OSError as exc: + err += " | " + format_oserror("mergerfs removexattr", exc) + try: + os.removexattr(native_file, xname) + except OSError as exc: + err += " | " + format_oserror("native removexattr", exc) + return fail(err) + + err = compare_calls( + "getxattr missing attr", + lambda: os.getxattr(merge_file, xname), + lambda: os.getxattr(native_file, xname), + ) + if err: + try: + os.getxattr(merge_file, xname) + except OSError as exc: + err += " | " + format_oserror("mergerfs getxattr missing", exc) + try: + os.getxattr(native_file, xname) + except OSError as exc: + err += " | " + format_oserror("native getxattr missing", exc) + return fail(err) + + err = compare_calls( + "setxattr ENOENT", + lambda: os.setxattr(merge_missing, xname, xvalue), + lambda: os.setxattr(native_missing, xname, xvalue), + ) + if err: + return fail(err) + + err = compare_calls( + "getxattr ENOTDIR", + lambda: os.getxattr(join(merge_notdir, "child"), xname), + lambda: os.getxattr(join(native_notdir, "child"), xname), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/TEST_posix_xattr_matrix b/tests/TEST_posix_xattr_matrix new file mode 100755 index 00000000..4f430e6e --- /dev/null +++ b/tests/TEST_posix_xattr_matrix @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 + +import errno +import os +import sys +import tempfile + +from posix_parity import fail +from posix_parity import join +from posix_parity import touch + + +def errno_name(err): + if err is None: + return "None" + return errno.errorcode.get(err, str(err)) + + +def invoke(callable_): + try: + return True, callable_(), 0 + except OSError as exc: + return False, None, exc.errno + + +def compare(name, m_call, n_call, value_cmp=None): + m_ok, m_val, m_errno = invoke(m_call) + n_ok, n_val, n_errno = invoke(n_call) + + if m_ok != n_ok: + return ( + f"{name}: success mismatch mergerfs={m_ok} native={n_ok} " + f"(mergerfs_errno={m_errno}:{errno_name(m_errno)} native_errno={n_errno}:{errno_name(n_errno)})" + ) + if m_errno != n_errno: + return ( + f"{name}: errno mismatch mergerfs={m_errno}:{errno_name(m_errno)} " + f"native={n_errno}:{errno_name(n_errno)}" + ) + if m_ok and value_cmp is not None and not value_cmp(m_val, n_val): + return f"{name}: value mismatch mergerfs={m_val!r} native={n_val!r}" + + return None + + +def getxattr_cmp(path_m, path_n, xname): + return compare( + f"getxattr {xname} {os.path.basename(path_m)}", + lambda: os.getxattr(path_m, xname), + lambda: os.getxattr(path_n, xname), + lambda a, b: a == b, + ) + + +def list_has_cmp(path_m, path_n, xname): + return compare( + f"listxattr has {xname} {os.path.basename(path_m)}", + lambda: os.listxattr(path_m), + lambda: os.listxattr(path_n), + lambda a, b: ((xname in a) == (xname in b)), + ) + + +def main(): + if len(sys.argv) != 2: + print("usage: TEST_posix_xattr_matrix ", file=sys.stderr) + return 1 + + mount = sys.argv[1] + + with tempfile.TemporaryDirectory() as native: + root_m = join(mount, "posix-xattr-matrix") + root_n = join(native, "posix-xattr-matrix") + + file_m = join(root_m, "file") + file_n = join(root_n, "file") + dir_m = join(root_m, "dir") + dir_n = join(root_n, "dir") + link_m = join(root_m, "link") + link_n = join(root_n, "link") + + touch(file_m, b"x") + touch(file_n, b"x") + os.makedirs(dir_m, exist_ok=True) + os.makedirs(dir_n, exist_ok=True) + try: + os.unlink(link_m) + except FileNotFoundError: + pass + try: + os.unlink(link_n) + except FileNotFoundError: + pass + os.symlink("file", link_m) + os.symlink("file", link_n) + + objs = [ + ("file", file_m, file_n), + ("dir", dir_m, dir_n), + ] + + xbase = "user.mergerfs.matrix" + + for obj_name, path_m, path_n in objs: + xname = f"{xbase}.{obj_name}" + + err = compare( + f"setxattr default {obj_name}", + lambda p=path_m, n=xname: os.setxattr(p, n, b"v1"), + lambda p=path_n, n=xname: os.setxattr(p, n, b"v1"), + ) + if err: + return fail(err) + + err = getxattr_cmp(path_m, path_n, xname) + if err: + return fail(err) + + err = list_has_cmp(path_m, path_n, xname) + if err: + return fail(err) + + if hasattr(os, "XATTR_CREATE"): + err = compare( + f"setxattr CREATE existing {obj_name}", + lambda p=path_m, n=xname: os.setxattr(p, n, b"v2", os.XATTR_CREATE), + lambda p=path_n, n=xname: os.setxattr(p, n, b"v2", os.XATTR_CREATE), + ) + if err: + return fail(err) + + if hasattr(os, "XATTR_REPLACE"): + err = compare( + f"setxattr REPLACE existing {obj_name}", + lambda p=path_m, n=xname: os.setxattr(p, n, b"v3", os.XATTR_REPLACE), + lambda p=path_n, n=xname: os.setxattr(p, n, b"v3", os.XATTR_REPLACE), + ) + if err: + return fail(err) + + err = getxattr_cmp(path_m, path_n, xname) + if err: + return fail(err) + + missing_name = f"{xname}.missing" + if hasattr(os, "XATTR_REPLACE"): + err = compare( + f"setxattr REPLACE missing {obj_name}", + lambda p=path_m, n=missing_name: os.setxattr(p, n, b"v", os.XATTR_REPLACE), + lambda p=path_n, n=missing_name: os.setxattr(p, n, b"v", os.XATTR_REPLACE), + ) + if err: + return fail(err) + + err = compare( + f"removexattr existing {obj_name}", + lambda p=path_m, n=xname: os.removexattr(p, n), + lambda p=path_n, n=xname: os.removexattr(p, n), + ) + if err: + return fail(err) + + err = compare( + f"removexattr missing {obj_name}", + lambda p=path_m, n=xname: os.removexattr(p, n), + lambda p=path_n, n=xname: os.removexattr(p, n), + ) + if err: + return fail(err) + + lxname = f"{xbase}.link" + if all(hasattr(os, fn) for fn in ("lsetxattr", "lgetxattr", "llistxattr", "lremovexattr")): + err = compare( + "lsetxattr symlink default", + lambda: os.lsetxattr(link_m, lxname, b"lv1"), + lambda: os.lsetxattr(link_n, lxname, b"lv1"), + ) + if err: + return fail(err) + + err = compare( + "lgetxattr symlink", + lambda: os.lgetxattr(link_m, lxname), + lambda: os.lgetxattr(link_n, lxname), + lambda a, b: a == b, + ) + if err: + return fail(err) + + err = compare( + "llistxattr has symlink key", + lambda: os.llistxattr(link_m), + lambda: os.llistxattr(link_n), + lambda a, b: ((lxname in a) == (lxname in b)), + ) + if err: + return fail(err) + + err = compare( + "lremovexattr symlink", + lambda: os.lremovexattr(link_m, lxname), + lambda: os.lremovexattr(link_n, lxname), + ) + if err: + return fail(err) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/posix_parity.py b/tests/posix_parity.py new file mode 100644 index 00000000..77b96a0b --- /dev/null +++ b/tests/posix_parity.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 + +import ctypes +import errno +import os +import stat + + +libc = ctypes.CDLL(None, use_errno=True) +libc.access.argtypes = [ctypes.c_char_p, ctypes.c_int] +libc.access.restype = ctypes.c_int + + +def invoke(callable_): + try: + return True, callable_(), 0 + except OSError as exc: + return False, None, exc.errno + + +def compare_calls(name, merge_call, native_call, value_cmp=None, close_fds=False): + m_ok, m_val, m_errno = invoke(merge_call) + n_ok, n_val, n_errno = invoke(native_call) + + if m_ok != n_ok: + return ( + f"{name}: success mismatch mergerfs={m_ok} native={n_ok} " + f"(mergerfs_errno={m_errno} native_errno={n_errno})" + ) + if m_errno != n_errno: + return f"{name}: errno mismatch mergerfs={m_errno} native={n_errno}" + if m_ok and value_cmp is not None and not value_cmp(m_val, n_val): + if close_fds: + close_if_fd(m_val) + close_if_fd(n_val) + return f"{name}: value mismatch mergerfs={m_val!r} native={n_val!r}" + + if close_fds: + close_if_fd(m_val) + close_if_fd(n_val) + + return None + + +def access_raw(path, mode): + ctypes.set_errno(0) + rv = libc.access(path.encode(), mode) + err = ctypes.get_errno() + return rv, err + + +def compare_access(name, merge_path, native_path, mode, expect_errno=None): + m_rv, m_errno = access_raw(merge_path, mode) + n_rv, n_errno = access_raw(native_path, mode) + + if m_rv != n_rv: + return f"{name}: return mismatch mergerfs={m_rv} native={n_rv}" + if m_errno != n_errno: + return f"{name}: errno mismatch mergerfs={m_errno} native={n_errno}" + if expect_errno is not None and n_errno != expect_errno: + return f"{name}: expected errno={expect_errno}, got errno={n_errno}" + + return None + + +def stat_cmp_basic(lhs, rhs): + return ( + stat.S_IFMT(lhs.st_mode) == stat.S_IFMT(rhs.st_mode) + and stat.S_IMODE(lhs.st_mode) == stat.S_IMODE(rhs.st_mode) + and lhs.st_size == rhs.st_size + ) + + +def cleanup_paths(paths): + for path in paths: + try: + if os.path.islink(path) or os.path.isfile(path): + os.unlink(path) + elif os.path.isdir(path): + os.rmdir(path) + except FileNotFoundError: + pass + + +def close_if_fd(value): + if isinstance(value, int) and value >= 0: + try: + os.close(value) + except OSError: + pass + + +def fail(msg): + print(msg, end="") + return 1 + + +def join(root, rel): + return os.path.join(root, rel) + + +def ensure_parent(path): + os.makedirs(os.path.dirname(path), exist_ok=True) + + +def touch(path, data=b"x", mode=0o644): + ensure_parent(path) + with open(path, "wb") as fp: + fp.write(data) + os.chmod(path, mode) + + +def mergerfs_ctrl_file(mount): + return join(mount, ".mergerfs") + + +def mergerfs_key(option_key): + return f"user.mergerfs.{option_key}" + + +def mergerfs_get_option(mount, option_key): + raw = os.getxattr(mergerfs_ctrl_file(mount), mergerfs_key(option_key)) + return raw.decode("utf-8", errors="surrogateescape") + + +def mergerfs_set_option(mount, option_key, value): + if isinstance(value, bytes): + payload = value + else: + payload = str(value).encode("utf-8") + os.setxattr(mergerfs_ctrl_file(mount), mergerfs_key(option_key), payload) + + +def parse_allpaths(raw): + if isinstance(raw, str): + raw = raw.encode("utf-8", errors="surrogateescape") + paths = [] + for p in raw.split(b"\0"): + if not p: + continue + paths.append(p.decode("utf-8", errors="surrogateescape")) + return paths + + +def _parse_branch_entry(entry): + entry = entry.strip() + if not entry: + return None + if "=" in entry: + return entry.split("=", 1)[0] + return entry + + +def mergerfs_branches(mount): + raw = mergerfs_get_option(mount, "branches") + branches = [] + for entry in raw.split(":"): + path = _parse_branch_entry(entry) + if path: + branches.append(path) + return branches + + +def underlying_path(mount, rel): + branches = mergerfs_branches(mount) + if not branches: + raise RuntimeError("no mergerfs branches configured") + return os.path.join(branches[0], rel) + + +def pair_paths(mount, rel): + return join(mount, rel), underlying_path(mount, rel) + + +def should_compare_inode(mount): + try: + inodecalc = mergerfs_get_option(mount, "inodecalc").strip().lower() + except OSError: + return False + return inodecalc == "passthrough" + + +def mergerfs_fullpath(path): + raw = os.getxattr(path, "user.mergerfs.fullpath") + return raw.decode("utf-8", errors="surrogateescape")