Browse Source

fix(shell): avoid marking skipped or unplaced volumes as fixed (#8866)

* fix(s3api): fix AWS Signature V2 format and validation

* fix(s3api): Skip space after "AWS" prefix (+1 offset)

* test(s3api): add unit tests for Signature V2 authentication fix

* fix(s3api): simply comparing signatures

* validation for the colon extraction in expectedAuth

* fix(shell): avoid marking skipped or unplaced volumes as fixed

---------

Co-authored-by: chrislu <chris.lu@gmail.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
pull/8871/head
qzh 1 day ago
committed by GitHub
parent
commit
4c72512ea2
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 17
      weed/shell/command_volume_fix_replication.go

17
weed/shell/command_volume_fix_replication.go

@ -337,8 +337,9 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm
} }
for _, vid := range volumeIds { for _, vid := range volumeIds {
for i := 0; i < retryCount+1; i++ { for i := 0; i < retryCount+1; i++ {
if err = c.fixOneUnderReplicatedVolume(commandEnv, writer, applyChanges, volumeReplicas, vid, allLocations); err == nil {
if applyChanges {
var copied bool
if copied, err = c.fixOneUnderReplicatedVolume(commandEnv, writer, applyChanges, volumeReplicas, vid, allLocations); err == nil {
if applyChanges && copied {
fixedVolumes[strconv.FormatUint(uint64(vid), 10)] = len(volumeReplicas[vid]) fixedVolumes[strconv.FormatUint(uint64(vid), 10)] = len(volumeReplicas[vid])
} }
break break
@ -350,7 +351,7 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm
return fixedVolumes, nil return fixedVolumes, nil
} }
func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *CommandEnv, writer io.Writer, applyChanges bool, volumeReplicas map[uint32][]*VolumeReplica, vid uint32, allLocations []location) error {
func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *CommandEnv, writer io.Writer, applyChanges bool, volumeReplicas map[uint32][]*VolumeReplica, vid uint32, allLocations []location) (bool, error) {
replicas := volumeReplicas[vid] replicas := volumeReplicas[vid]
replica := pickOneReplicaToCopyFrom(replicas) replica := pickOneReplicaToCopyFrom(replicas)
replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement)) replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement))
@ -370,7 +371,7 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co
var err error var err error
matched, err = filepath.Match(*c.collectionPattern, replica.info.Collection) matched, err = filepath.Match(*c.collectionPattern, replica.info.Collection)
if err != nil { if err != nil {
return fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err)
return false, fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err)
} }
} }
if !matched { if !matched {
@ -386,7 +387,7 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co
if !applyChanges { if !applyChanges {
// adjust volume count // adjust volume count
addVolumeCount(dst.dataNode.DiskInfos[replica.info.DiskType], 1) addVolumeCount(dst.dataNode.DiskInfos[replica.info.DiskType], 1)
break
return true, nil
} }
err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dst.dataNode), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dst.dataNode), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
@ -415,19 +416,19 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co
}) })
if err != nil { if err != nil {
return err
return false, err
} }
// adjust volume count // adjust volume count
addVolumeCount(dst.dataNode.DiskInfos[replica.info.DiskType], 1) addVolumeCount(dst.dataNode.DiskInfos[replica.info.DiskType], 1)
break
return true, nil
} }
} }
if !foundNewLocation && !hasSkippedCollection { if !foundNewLocation && !hasSkippedCollection {
fmt.Fprintf(writer, "failed to place volume %d replica as %s, existing:%+v\n", replica.info.Id, replicaPlacement, len(replicas)) fmt.Fprintf(writer, "failed to place volume %d replica as %s, existing:%+v\n", replica.info.Id, replicaPlacement, len(replicas))
} }
return nil
return false, nil
} }
func addVolumeCount(info *master_pb.DiskInfo, count int) { func addVolumeCount(info *master_pb.DiskInfo, count int) {

Loading…
Cancel
Save