From 2615b85bdf4c093724c019b752382411a9273e1d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 9 Mar 2026 05:59:10 -0700 Subject: [PATCH] feat: add Rust volume server CI release workflows and installer script - rust_binaries_release.yml: builds linux/darwin/windows on tags - rust_binaries_dev.yml: builds dev binaries on master push - install.sh: universal installer for Go weed and Rust volume server --- .github/workflows/rust_binaries_dev.yml | 165 ++++++++++++ .github/workflows/rust_binaries_release.yml | 215 +++++++++++++++ install.sh | 275 ++++++++++++++++++++ 3 files changed, 655 insertions(+) create mode 100644 .github/workflows/rust_binaries_dev.yml create mode 100644 .github/workflows/rust_binaries_release.yml create mode 100755 install.sh diff --git a/.github/workflows/rust_binaries_dev.yml b/.github/workflows/rust_binaries_dev.yml new file mode 100644 index 000000000..4a9e6f896 --- /dev/null +++ b/.github/workflows/rust_binaries_dev.yml @@ -0,0 +1,165 @@ +name: "rust: build dev volume server binaries" + +on: + push: + branches: [ master ] + paths: + - 'seaweed-volume/**' + - '.github/workflows/rust_binaries_dev.yml' + +permissions: + contents: read + +jobs: + + cleanup: + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Delete old Rust volume dev assets + uses: mknejp/delete-release-assets@v1 + with: + token: ${{ github.token }} + tag: dev + fail-if-no-assets: false + assets: | + seaweed-volume-* + + build-rust-volume-dev-linux: + permissions: + contents: write + needs: cleanup + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + asset_suffix: linux-amd64 + + steps: + - uses: actions/checkout@v6 + + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-dev-${{ matrix.target }}- + + - name: Set BUILD_TIME + run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV" + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: cd seaweed-volume && cargo build --release + + - name: Package large disk binary + run: | + cp seaweed-volume/target/release/seaweed-volume seaweed-volume-large-disk + tar czf "seaweed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" seaweed-volume-large-disk + rm seaweed-volume-large-disk + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: cd seaweed-volume && cargo build --release --no-default-features + + - name: Package normal binary + run: | + cp seaweed-volume/target/release/seaweed-volume seaweed-volume-normal + tar czf "seaweed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" seaweed-volume-normal + rm seaweed-volume-normal + + - name: Upload dev release assets + uses: softprops/action-gh-release@v2 + with: + tag_name: dev + prerelease: true + files: | + seaweed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz + seaweed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-rust-volume-dev-darwin: + permissions: + contents: write + needs: build-rust-volume-dev-linux + runs-on: macos-latest + strategy: + matrix: + include: + - target: aarch64-apple-darwin + asset_suffix: darwin-arm64 + - target: x86_64-apple-darwin + asset_suffix: darwin-amd64 + + steps: + - uses: actions/checkout@v6 + + - name: Install protobuf compiler + run: brew install protobuf + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Cache cargo registry and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-dev-${{ matrix.target }}- + + - name: Set BUILD_TIME + run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV" + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} + + - name: Package large disk binary + run: | + cp seaweed-volume/target/${{ matrix.target }}/release/seaweed-volume seaweed-volume-large-disk + tar czf "seaweed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" seaweed-volume-large-disk + rm seaweed-volume-large-disk + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features + + - name: Package normal binary + run: | + cp seaweed-volume/target/${{ matrix.target }}/release/seaweed-volume seaweed-volume-normal + tar czf "seaweed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" seaweed-volume-normal + rm seaweed-volume-normal + + - name: Upload dev release assets + uses: softprops/action-gh-release@v2 + with: + tag_name: dev + prerelease: true + files: | + seaweed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz + seaweed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml new file mode 100644 index 000000000..47995e21e --- /dev/null +++ b/.github/workflows/rust_binaries_release.yml @@ -0,0 +1,215 @@ +name: "rust: build versioned volume server binaries" + +on: + push: + tags: + - '*' + + workflow_dispatch: + +permissions: + contents: read + +jobs: + + build-rust-volume-linux: + permissions: + contents: write + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + asset_suffix: linux_amd64 + - target: aarch64-unknown-linux-gnu + asset_suffix: linux_arm64 + cross: true + + steps: + - uses: actions/checkout@v6 + + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install cross-compilation tools + if: matrix.cross + run: | + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-release-${{ matrix.target }}- + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features + + - name: Package binaries + run: | + # Large disk (default, 5bytes feature) + cp seaweed-volume/target/${{ matrix.target }}/release/seaweed-volume seaweed-volume-large-disk + tar czf seaweed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz seaweed-volume-large-disk + rm seaweed-volume-large-disk + + # Normal volume size + cp seaweed-volume/target/${{ matrix.target }}/release/seaweed-volume seaweed-volume-normal + tar czf seaweed-volume_${{ matrix.asset_suffix }}.tar.gz seaweed-volume-normal + rm seaweed-volume-normal + + - name: Upload release assets + uses: softprops/action-gh-release@v2 + with: + files: | + seaweed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz + seaweed-volume_${{ matrix.asset_suffix }}.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-rust-volume-darwin: + permissions: + contents: write + runs-on: macos-latest + strategy: + matrix: + include: + - target: x86_64-apple-darwin + asset_suffix: darwin_amd64 + - target: aarch64-apple-darwin + asset_suffix: darwin_arm64 + + steps: + - uses: actions/checkout@v6 + + - name: Install protobuf compiler + run: brew install protobuf + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Cache cargo registry and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-release-${{ matrix.target }}- + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features + + - name: Package binaries + run: | + cp seaweed-volume/target/${{ matrix.target }}/release/seaweed-volume seaweed-volume-large-disk + tar czf seaweed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz seaweed-volume-large-disk + rm seaweed-volume-large-disk + + cp seaweed-volume/target/${{ matrix.target }}/release/seaweed-volume seaweed-volume-normal + tar czf seaweed-volume_${{ matrix.asset_suffix }}.tar.gz seaweed-volume-normal + rm seaweed-volume-normal + + - name: Upload release assets + uses: softprops/action-gh-release@v2 + with: + files: | + seaweed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz + seaweed-volume_${{ matrix.asset_suffix }}.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build-rust-volume-windows: + permissions: + contents: write + runs-on: windows-latest + + steps: + - uses: actions/checkout@v6 + + - name: Install protobuf compiler + run: choco install protoc -y + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-release-windows- + + - name: Build Rust volume server (large disk) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release + + - name: Build Rust volume server (normal) + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --no-default-features + + - name: Package binaries + shell: bash + run: | + cp seaweed-volume/target/release/seaweed-volume.exe seaweed-volume-large-disk.exe + 7z a seaweed-volume_large_disk_windows_amd64.zip seaweed-volume-large-disk.exe + rm seaweed-volume-large-disk.exe + + cp seaweed-volume/target/release/seaweed-volume.exe seaweed-volume-normal.exe + 7z a seaweed-volume_windows_amd64.zip seaweed-volume-normal.exe + rm seaweed-volume-normal.exe + + - name: Upload release assets + uses: softprops/action-gh-release@v2 + with: + files: | + seaweed-volume_large_disk_windows_amd64.zip + seaweed-volume_windows_amd64.zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/install.sh b/install.sh new file mode 100755 index 000000000..051e9bd7e --- /dev/null +++ b/install.sh @@ -0,0 +1,275 @@ +#!/bin/bash +# +# SeaweedFS Installer +# Downloads Go and/or Rust binaries from GitHub releases. +# +# Usage: +# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash +# curl -fsSL ... | bash -s -- --component volume-rust --large-disk +# curl -fsSL ... | bash -s -- --version v3.93 --dir /usr/local/bin +# +# Options: +# --component COMP Which binary to install: weed, volume-rust, all (default: weed) +# --version VER Release version tag (default: latest) +# --large-disk Use large disk variant (5-byte offset, 8TB max volume) +# --dir DIR Installation directory (default: /usr/local/bin) +# --help Show this help message + +set -euo pipefail + +REPO="seaweedfs/seaweedfs" +COMPONENT="weed" +VERSION="" +LARGE_DISK=false +INSTALL_DIR="/usr/local/bin" + +# Colors (if terminal supports them) +if [ -t 1 ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + BLUE='\033[0;34m' + NC='\033[0m' +else + RED='' GREEN='' YELLOW='' BLUE='' NC='' +fi + +info() { echo -e "${BLUE}[info]${NC} $*"; } +ok() { echo -e "${GREEN}[ok]${NC} $*"; } +warn() { echo -e "${YELLOW}[warn]${NC} $*"; } +error() { echo -e "${RED}[error]${NC} $*" >&2; exit 1; } + +usage() { + sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' + exit 0 +} + +# Parse arguments +while [ $# -gt 0 ]; do + case "$1" in + --component) COMPONENT="$2"; shift 2 ;; + --version) VERSION="$2"; shift 2 ;; + --large-disk) LARGE_DISK=true; shift ;; + --dir) INSTALL_DIR="$2"; shift 2 ;; + --help|-h) usage ;; + *) error "Unknown option: $1. Use --help for usage." ;; + esac +done + +# Detect OS and architecture +detect_platform() { + local os arch + + case "$(uname -s)" in + Linux*) os="linux" ;; + Darwin*) os="darwin" ;; + MINGW*|MSYS*|CYGWIN*) os="windows" ;; + FreeBSD*) os="freebsd" ;; + *) error "Unsupported OS: $(uname -s)" ;; + esac + + case "$(uname -m)" in + x86_64|amd64) arch="amd64" ;; + aarch64|arm64) arch="arm64" ;; + armv7l|armv6l) arch="arm" ;; + *) error "Unsupported architecture: $(uname -m)" ;; + esac + + echo "${os}" "${arch}" +} + +# Get latest release tag from GitHub API +get_latest_version() { + local url="https://api.github.com/repos/${REPO}/releases/latest" + if command -v curl &>/dev/null; then + curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' + elif command -v wget &>/dev/null; then + wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' + else + error "Neither curl nor wget found. Please install one." + fi +} + +# Download a file +download() { + local url="$1" dest="$2" + info "Downloading ${url}" + if command -v curl &>/dev/null; then + curl -fsSL -o "$dest" "$url" + elif command -v wget &>/dev/null; then + wget -qO "$dest" "$url" + fi +} + +# Build Go weed binary asset name +go_asset_name() { + local os="$1" arch="$2" + local suffix="${os}_${arch}" + if [ "$LARGE_DISK" = true ]; then + suffix="${suffix}_large_disk" + fi + echo "${suffix}.tar.gz" +} + +# Build Rust volume server asset name +rust_asset_name() { + local os="$1" arch="$2" + local prefix="seaweed-volume" + if [ "$LARGE_DISK" = true ]; then + prefix="seaweed-volume_large_disk" + else + prefix="seaweed-volume" + fi + local suffix="${os}_${arch}" + if [ "$os" = "windows" ]; then + echo "${prefix}_${suffix}.zip" + else + echo "${prefix}_${suffix}.tar.gz" + fi +} + +# Install a single component +install_component() { + local component="$1" os="$2" arch="$3" + local asset_name download_url tmpdir + + tmpdir="$(mktemp -d)" + trap "rm -rf '$tmpdir'" EXIT + + case "$component" in + weed) + asset_name="$(go_asset_name "$os" "$arch")" + download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" + download "$download_url" "${tmpdir}/${asset_name}" + + info "Extracting ${asset_name}..." + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" + + # The Go release action puts the binary inside a directory + local weed_bin + weed_bin="$(find "$tmpdir" -name 'weed' -type f | head -1)" + if [ -z "$weed_bin" ]; then + weed_bin="$(find "$tmpdir" -name 'weed.exe' -type f | head -1)" + fi + if [ -z "$weed_bin" ]; then + error "Could not find weed binary in archive" + fi + + chmod +x "$weed_bin" + install_binary "$weed_bin" "weed" + ok "Installed weed to ${INSTALL_DIR}/weed" + ;; + + volume-rust) + # Check platform support for Rust volume server + case "$os" in + linux|darwin|windows) ;; + *) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;; + esac + case "$arch" in + amd64|arm64) ;; + *) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;; + esac + + asset_name="$(rust_asset_name "$os" "$arch")" + download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}" + download "$download_url" "${tmpdir}/${asset_name}" + + info "Extracting ${asset_name}..." + if [ "$os" = "windows" ]; then + unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir" + else + tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" + fi + + local rust_bin + if [ "$LARGE_DISK" = true ]; then + rust_bin="$(find "$tmpdir" -name 'seaweed-volume-large-disk*' -type f | head -1)" + else + rust_bin="$(find "$tmpdir" -name 'seaweed-volume-normal*' -type f | head -1)" + fi + if [ -z "$rust_bin" ]; then + rust_bin="$(find "$tmpdir" -name 'seaweed-volume*' -type f | head -1)" + fi + if [ -z "$rust_bin" ]; then + error "Could not find seaweed-volume binary in archive" + fi + + chmod +x "$rust_bin" + local dest_name="seaweed-volume" + if [ "$os" = "windows" ]; then + dest_name="seaweed-volume.exe" + fi + install_binary "$rust_bin" "$dest_name" + ok "Installed seaweed-volume to ${INSTALL_DIR}/${dest_name}" + ;; + + *) + error "Unknown component: ${component}. Use: weed, volume-rust, all" + ;; + esac +} + +# Copy binary to install dir, using sudo if needed +install_binary() { + local src="$1" name="$2" + local dest="${INSTALL_DIR}/${name}" + + mkdir -p "$INSTALL_DIR" 2>/dev/null || true + + if [ -w "$INSTALL_DIR" ]; then + cp "$src" "$dest" + else + info "Need elevated permissions to write to ${INSTALL_DIR}" + sudo cp "$src" "$dest" + fi + chmod +x "$dest" 2>/dev/null || sudo chmod +x "$dest" +} + +main() { + info "SeaweedFS Installer" + + read -r os arch <<< "$(detect_platform)" + info "Detected platform: ${os}/${arch}" + + if [ -z "$VERSION" ]; then + info "Resolving latest release..." + VERSION="$(get_latest_version)" + if [ -z "$VERSION" ]; then + error "Could not determine latest version. Specify with --version" + fi + fi + info "Version: ${VERSION}" + + if [ "$LARGE_DISK" = true ]; then + info "Variant: large disk (8TB max volume)" + else + info "Variant: normal (32GB max volume)" + fi + + case "$COMPONENT" in + all) + install_component "weed" "$os" "$arch" + install_component "volume-rust" "$os" "$arch" + ;; + *) + install_component "$COMPONENT" "$os" "$arch" + ;; + esac + + echo "" + ok "Installation complete!" + if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then + info " weed: ${INSTALL_DIR}/weed" + fi + if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then + info " seaweed-volume: ${INSTALL_DIR}/seaweed-volume" + fi + echo "" + info "Quick start:" + info " weed master # Start master server" + info " weed volume -mserver=localhost:9333 # Start Go volume server" + info " seaweed-volume -mserver localhost:9333 # Start Rust volume server" +} + +main