seaweedfs/test/volume_server/grpc/tail_test.go

package volume_server_grpc_test

import (
	"bytes"
	"context"
	"io"
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)

func TestVolumeTailSenderMissingVolume(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode")
	}

	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
	defer conn.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{VolumeId: 77777, SinceNs: 0, IdleTimeoutSeconds: 1})
	if err == nil {
		_, err = stream.Recv()
	}
	if err == nil || !strings.Contains(err.Error(), "not found volume") {
		t.Fatalf("VolumeTailSender missing-volume error mismatch: %v", err)
	}
}

func TestVolumeTailSenderHeartbeatThenEOF(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode")
	}

	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
	defer conn.Close()

	const volumeID = uint32(71)
	framework.AllocateVolume(t, grpcClient, volumeID, "")

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
		VolumeId:           volumeID,
		SinceNs:            0,
		IdleTimeoutSeconds: 1,
	})
	if err != nil {
		t.Fatalf("VolumeTailSender start failed: %v", err)
	}

	msg, err := stream.Recv()
	if err != nil {
		t.Fatalf("VolumeTailSender first recv failed: %v", err)
	}
	if !msg.GetIsLastChunk() {
		t.Fatalf("expected first tail message to be heartbeat IsLastChunk=true")
	}

	_, err = stream.Recv()
	if err != io.EOF {
		t.Fatalf("expected EOF after idle timeout drain, got: %v", err)
	}
}

func TestVolumeTailReceiverMissingVolume(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode")
	}

	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
	defer conn.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{VolumeId: 88888, SourceVolumeServer: clusterHarness.VolumeServerAddress(), SinceNs: 0, IdleTimeoutSeconds: 1})
	if err == nil || !strings.Contains(err.Error(), "receiver not found volume") {
		t.Fatalf("VolumeTailReceiver missing-volume error mismatch: %v", err)
	}
}

func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode")
	}

	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
	defer sourceConn.Close()
	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
	defer destConn.Close()

	const volumeID = uint32(72)
	framework.AllocateVolume(t, sourceClient, volumeID, "")
	framework.AllocateVolume(t, destClient, volumeID, "")

	httpClient := framework.NewHTTPClient()
	fid := framework.NewFileID(volumeID, 880003, 0x3456789A)
	payload := []byte("tail-receiver-replicates-source-updates")

	sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload)
	_ = framework.ReadAllAndClose(t, sourceUploadResp)
	if sourceUploadResp.StatusCode != http.StatusCreated {
		t.Fatalf("source upload expected 201, got %d", sourceUploadResp.StatusCode)
	}

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	_, err := destClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
		VolumeId:           volumeID,
		SourceVolumeServer: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
		SinceNs:            0,
		IdleTimeoutSeconds: 1,
	})
	if err != nil {
		t.Fatalf("VolumeTailReceiver success path failed: %v", err)
	}

	destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
	destReadBody := framework.ReadAllAndClose(t, destReadResp)
	if destReadResp.StatusCode != http.StatusOK {
		t.Fatalf("destination read after tail receive expected 200, got %d", destReadResp.StatusCode)
	}
	if string(destReadBody) != string(payload) {
		t.Fatalf("destination tail-received payload mismatch: got %q want %q", string(destReadBody), string(payload))
	}
}

func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode")
	}

	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
	defer conn.Close()

	const volumeID = uint32(73)
	framework.AllocateVolume(t, grpcClient, volumeID, "")

	httpClient := framework.NewHTTPClient()
	fid := framework.NewFileID(volumeID, 880004, 0x456789AB)
	largePayload := bytes.Repeat([]byte("L"), 2*1024*1024+128*1024)
	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, largePayload)
	_ = framework.ReadAllAndClose(t, uploadResp)
	if uploadResp.StatusCode != http.StatusCreated {
		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
	}

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
		VolumeId:           volumeID,
		SinceNs:            0,
		IdleTimeoutSeconds: 1,
	})
	if err != nil {
		t.Fatalf("VolumeTailSender start failed: %v", err)
	}

	dataChunkCount := 0
	sawNonLastDataChunk := false
	sawLastDataChunk := false
	for {
		msg, recvErr := stream.Recv()
		if recvErr == io.EOF {
			break
		}
		if recvErr != nil {
			t.Fatalf("VolumeTailSender recv failed: %v", recvErr)
		}
		if len(msg.GetNeedleBody()) == 0 {
			continue
		}
		dataChunkCount++
		if msg.GetIsLastChunk() {
			sawLastDataChunk = true
		} else {
			sawNonLastDataChunk = true
		}
	}

	if dataChunkCount < 2 {
		t.Fatalf("VolumeTailSender expected multiple chunks for large needle, got %d", dataChunkCount)
	}
	if !sawNonLastDataChunk {
		t.Fatalf("VolumeTailSender expected at least one non-last data chunk")
	}
	if !sawLastDataChunk {
		t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true")
	}
}