seaweedfs/test/volume_server/http/chunk_manifest_test.go

package volume_server_http_test

import (
	"bytes"
	"encoding/json"
	"net/http"
	"testing"

	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
	"github.com/seaweedfs/seaweedfs/weed/operation"
)

func TestChunkManifestExpansionAndBypass(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode")
	}

	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
	defer conn.Close()

	const volumeID = uint32(102)
	framework.AllocateVolume(t, grpcClient, volumeID, "")

	client := framework.NewHTTPClient()

	chunkFID := framework.NewFileID(volumeID, 772005, 0x5E6F7081)
	chunkPayload := []byte("chunk-manifest-expanded-content")
	chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload)
	_ = framework.ReadAllAndClose(t, chunkUploadResp)
	if chunkUploadResp.StatusCode != http.StatusCreated {
		t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode)
	}

	manifest := &operation.ChunkManifest{
		Name: "manifest.bin",
		Mime: "application/octet-stream",
		Size: int64(len(chunkPayload)),
		Chunks: []*operation.ChunkInfo{
			{
				Fid:    chunkFID,
				Offset: 0,
				Size:   int64(len(chunkPayload)),
			},
		},
	}
	manifestBytes, err := json.Marshal(manifest)
	if err != nil {
		t.Fatalf("marshal chunk manifest: %v", err)
	}

	manifestFID := framework.NewFileID(volumeID, 772006, 0x6F708192)
	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
	if err != nil {
		t.Fatalf("create manifest upload request: %v", err)
	}
	manifestUploadReq.Header.Set("Content-Type", "application/json")
	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
	_ = framework.ReadAllAndClose(t, manifestUploadResp)
	if manifestUploadResp.StatusCode != http.StatusCreated {
		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
	}

	expandedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID)
	expandedReadBody := framework.ReadAllAndClose(t, expandedReadResp)
	if expandedReadResp.StatusCode != http.StatusOK {
		t.Fatalf("manifest expanded read expected 200, got %d", expandedReadResp.StatusCode)
	}
	if string(expandedReadBody) != string(chunkPayload) {
		t.Fatalf("manifest expanded read mismatch: got %q want %q", string(expandedReadBody), string(chunkPayload))
	}
	if expandedReadResp.Header.Get("X-File-Store") != "chunked" {
		t.Fatalf("manifest expanded read expected X-File-Store=chunked, got %q", expandedReadResp.Header.Get("X-File-Store"))
	}

	bypassReadResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false"))
	bypassReadBody := framework.ReadAllAndClose(t, bypassReadResp)
	if bypassReadResp.StatusCode != http.StatusOK {
		t.Fatalf("manifest bypass read expected 200, got %d", bypassReadResp.StatusCode)
	}
	if bypassReadResp.Header.Get("X-File-Store") != "" {
		t.Fatalf("manifest bypass read expected empty X-File-Store header, got %q", bypassReadResp.Header.Get("X-File-Store"))
	}

	var gotManifest operation.ChunkManifest
	if err = json.Unmarshal(bypassReadBody, &gotManifest); err != nil {
		t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(bypassReadBody))
	}
	if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != chunkFID {
		t.Fatalf("manifest bypass read payload mismatch: %+v", gotManifest)
	}
}

func TestChunkManifestDeleteRemovesChildChunks(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode")
	}

	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
	defer conn.Close()

	const volumeID = uint32(104)
	framework.AllocateVolume(t, grpcClient, volumeID, "")

	client := framework.NewHTTPClient()

	chunkFID := framework.NewFileID(volumeID, 772008, 0x8192A3B4)
	chunkPayload := []byte("chunk-manifest-delete-content")
	chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload)
	_ = framework.ReadAllAndClose(t, chunkUploadResp)
	if chunkUploadResp.StatusCode != http.StatusCreated {
		t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode)
	}

	manifest := &operation.ChunkManifest{
		Name: "manifest-delete.bin",
		Mime: "application/octet-stream",
		Size: int64(len(chunkPayload)),
		Chunks: []*operation.ChunkInfo{
			{
				Fid:    chunkFID,
				Offset: 0,
				Size:   int64(len(chunkPayload)),
			},
		},
	}
	manifestBytes, err := json.Marshal(manifest)
	if err != nil {
		t.Fatalf("marshal chunk manifest: %v", err)
	}

	manifestFID := framework.NewFileID(volumeID, 772009, 0x92A3B4C5)
	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
	if err != nil {
		t.Fatalf("create manifest upload request: %v", err)
	}
	manifestUploadReq.Header.Set("Content-Type", "application/json")
	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
	_ = framework.ReadAllAndClose(t, manifestUploadResp)
	if manifestUploadResp.StatusCode != http.StatusCreated {
		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
	}

	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID))
	deleteBody := framework.ReadAllAndClose(t, deleteResp)
	if deleteResp.StatusCode != http.StatusAccepted {
		t.Fatalf("manifest delete expected 202, got %d", deleteResp.StatusCode)
	}
	var deleteResult map[string]int64
	if err = json.Unmarshal(deleteBody, &deleteResult); err != nil {
		t.Fatalf("decode manifest delete response: %v body=%q", err, string(deleteBody))
	}
	if deleteResult["size"] != int64(len(chunkPayload)) {
		t.Fatalf("manifest delete expected size=%d, got %d", len(chunkPayload), deleteResult["size"])
	}

	manifestReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID)
	_ = framework.ReadAllAndClose(t, manifestReadAfterDelete)
	if manifestReadAfterDelete.StatusCode != http.StatusNotFound {
		t.Fatalf("manifest read after delete expected 404, got %d", manifestReadAfterDelete.StatusCode)
	}

	chunkReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID)
	_ = framework.ReadAllAndClose(t, chunkReadAfterDelete)
	if chunkReadAfterDelete.StatusCode != http.StatusNotFound {
		t.Fatalf("chunk read after manifest delete expected 404, got %d", chunkReadAfterDelete.StatusCode)
	}
}

func TestChunkManifestDeleteFailsWhenChildDeletionFails(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode")
	}

	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
	defer conn.Close()

	const volumeID = uint32(105)
	framework.AllocateVolume(t, grpcClient, volumeID, "")

	client := framework.NewHTTPClient()
	manifest := &operation.ChunkManifest{
		Name: "manifest-delete-failure.bin",
		Mime: "application/octet-stream",
		Size: 1,
		Chunks: []*operation.ChunkInfo{
			{
				Fid:    "not-a-valid-fid",
				Offset: 0,
				Size:   1,
			},
		},
	}
	manifestBytes, err := json.Marshal(manifest)
	if err != nil {
		t.Fatalf("marshal chunk manifest: %v", err)
	}

	manifestFID := framework.NewFileID(volumeID, 772010, 0xA3B4C5D6)
	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
	if err != nil {
		t.Fatalf("create manifest upload request: %v", err)
	}
	manifestUploadReq.Header.Set("Content-Type", "application/json")
	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
	_ = framework.ReadAllAndClose(t, manifestUploadResp)
	if manifestUploadResp.StatusCode != http.StatusCreated {
		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
	}

	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID))
	deleteBody := framework.ReadAllAndClose(t, deleteResp)
	if deleteResp.StatusCode != http.StatusInternalServerError {
		t.Fatalf("manifest delete with invalid child fid expected 500, got %d body=%q", deleteResp.StatusCode, string(deleteBody))
	}

	manifestBypassRead := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false"))
	manifestBypassBody := framework.ReadAllAndClose(t, manifestBypassRead)
	if manifestBypassRead.StatusCode != http.StatusOK {
		t.Fatalf("manifest bypass read after failed delete expected 200, got %d", manifestBypassRead.StatusCode)
	}
	var gotManifest operation.ChunkManifest
	if err = json.Unmarshal(manifestBypassBody, &gotManifest); err != nil {
		t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(manifestBypassBody))
	}
	if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != "not-a-valid-fid" {
		t.Fatalf("manifest payload mismatch after failed delete: %+v", gotManifest)
	}
}