You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

208 lines
5.8 KiB

10 years ago
5 years ago
5 years ago
5 years ago
3 years ago
4 years ago
3 years ago
3 years ago
10 years ago
3 years ago
3 years ago
volume: proxy writes to remote volume server, with replication or not the panic is triggered by uploading a file to a volume server not holding the designated replica. 2020-03-15 10:20:14.365488 I | http: panic serving 127.0.0.1:57124: runtime error: invalid memory address or nil pointer dereference goroutine 119 [running]: net/http.(*conn).serve.func1(0xc0001a8000) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1772 +0x139 panic(0x2316fe0, 0x3662900) /home/travis/.gimme/versions/go1.14.linux.amd64/src/runtime/panic.go:973 +0x396 github.com/chrislusf/seaweedfs/weed/topology.getWritableRemoteReplications(0xc00009c000, 0x2, 0x7ffeefbffbd2, 0xe, 0x0, 0xa, 0x0, 0x0, 0xbb4bf1f7) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:157 +0x53 github.com/chrislusf/seaweedfs/weed/topology.ReplicatedWrite(0x7ffeefbffbd2, 0xe, 0xc00009c000, 0xc000000002, 0xc000472750, 0xc0001b2200, 0x0, 0x1, 0x0) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:29 +0xc7 github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).PostHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers_write.go:52 +0x56f github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).privateStoreHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers.go:37 +0x21f net/http.HandlerFunc.ServeHTTP(0xc0004420e0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2012 +0x44 net/http.(*ServeMux).ServeHTTP(0xc0001fc800, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2387 +0x1a5 net/http.serverHandler.ServeHTTP(0xc0001781c0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2807 +0xa3 net/http.(*conn).serve(0xc0001a8000, 0x2934420, 0xc000212400) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1895 +0x86c created by net/http.(*Server).Serve /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2933 +0x35c Eg: server A (datacenter 1) and server B (datacenter 2) hold replica (100) for volume 1. If you upload a file with a key 1,xxxxx to server C (datacenter 3) will trigger the panic on server C. The server C should either proxy upload file to the correct volume server or should return an HTTP error code and not panic.
5 years ago
5 years ago
volume: proxy writes to remote volume server, with replication or not the panic is triggered by uploading a file to a volume server not holding the designated replica. 2020-03-15 10:20:14.365488 I | http: panic serving 127.0.0.1:57124: runtime error: invalid memory address or nil pointer dereference goroutine 119 [running]: net/http.(*conn).serve.func1(0xc0001a8000) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1772 +0x139 panic(0x2316fe0, 0x3662900) /home/travis/.gimme/versions/go1.14.linux.amd64/src/runtime/panic.go:973 +0x396 github.com/chrislusf/seaweedfs/weed/topology.getWritableRemoteReplications(0xc00009c000, 0x2, 0x7ffeefbffbd2, 0xe, 0x0, 0xa, 0x0, 0x0, 0xbb4bf1f7) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:157 +0x53 github.com/chrislusf/seaweedfs/weed/topology.ReplicatedWrite(0x7ffeefbffbd2, 0xe, 0xc00009c000, 0xc000000002, 0xc000472750, 0xc0001b2200, 0x0, 0x1, 0x0) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:29 +0xc7 github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).PostHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers_write.go:52 +0x56f github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).privateStoreHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers.go:37 +0x21f net/http.HandlerFunc.ServeHTTP(0xc0004420e0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2012 +0x44 net/http.(*ServeMux).ServeHTTP(0xc0001fc800, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2387 +0x1a5 net/http.serverHandler.ServeHTTP(0xc0001781c0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2807 +0xa3 net/http.(*conn).serve(0xc0001a8000, 0x2934420, 0xc000212400) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1895 +0x86c created by net/http.(*Server).Serve /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2933 +0x35c Eg: server A (datacenter 1) and server B (datacenter 2) hold replica (100) for volume 1. If you upload a file with a key 1,xxxxx to server C (datacenter 3) will trigger the panic on server C. The server C should either proxy upload file to the correct volume server or should return an HTTP error code and not panic.
5 years ago
volume: proxy writes to remote volume server, with replication or not the panic is triggered by uploading a file to a volume server not holding the designated replica. 2020-03-15 10:20:14.365488 I | http: panic serving 127.0.0.1:57124: runtime error: invalid memory address or nil pointer dereference goroutine 119 [running]: net/http.(*conn).serve.func1(0xc0001a8000) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1772 +0x139 panic(0x2316fe0, 0x3662900) /home/travis/.gimme/versions/go1.14.linux.amd64/src/runtime/panic.go:973 +0x396 github.com/chrislusf/seaweedfs/weed/topology.getWritableRemoteReplications(0xc00009c000, 0x2, 0x7ffeefbffbd2, 0xe, 0x0, 0xa, 0x0, 0x0, 0xbb4bf1f7) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:157 +0x53 github.com/chrislusf/seaweedfs/weed/topology.ReplicatedWrite(0x7ffeefbffbd2, 0xe, 0xc00009c000, 0xc000000002, 0xc000472750, 0xc0001b2200, 0x0, 0x1, 0x0) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:29 +0xc7 github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).PostHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers_write.go:52 +0x56f github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).privateStoreHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers.go:37 +0x21f net/http.HandlerFunc.ServeHTTP(0xc0004420e0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2012 +0x44 net/http.(*ServeMux).ServeHTTP(0xc0001fc800, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2387 +0x1a5 net/http.serverHandler.ServeHTTP(0xc0001781c0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2807 +0xa3 net/http.(*conn).serve(0xc0001a8000, 0x2934420, 0xc000212400) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1895 +0x86c created by net/http.(*Server).Serve /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2933 +0x35c Eg: server A (datacenter 1) and server B (datacenter 2) hold replica (100) for volume 1. If you upload a file with a key 1,xxxxx to server C (datacenter 3) will trigger the panic on server C. The server C should either proxy upload file to the correct volume server or should return an HTTP error code and not panic.
5 years ago
volume: proxy writes to remote volume server, with replication or not the panic is triggered by uploading a file to a volume server not holding the designated replica. 2020-03-15 10:20:14.365488 I | http: panic serving 127.0.0.1:57124: runtime error: invalid memory address or nil pointer dereference goroutine 119 [running]: net/http.(*conn).serve.func1(0xc0001a8000) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1772 +0x139 panic(0x2316fe0, 0x3662900) /home/travis/.gimme/versions/go1.14.linux.amd64/src/runtime/panic.go:973 +0x396 github.com/chrislusf/seaweedfs/weed/topology.getWritableRemoteReplications(0xc00009c000, 0x2, 0x7ffeefbffbd2, 0xe, 0x0, 0xa, 0x0, 0x0, 0xbb4bf1f7) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:157 +0x53 github.com/chrislusf/seaweedfs/weed/topology.ReplicatedWrite(0x7ffeefbffbd2, 0xe, 0xc00009c000, 0xc000000002, 0xc000472750, 0xc0001b2200, 0x0, 0x1, 0x0) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:29 +0xc7 github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).PostHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers_write.go:52 +0x56f github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).privateStoreHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers.go:37 +0x21f net/http.HandlerFunc.ServeHTTP(0xc0004420e0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2012 +0x44 net/http.(*ServeMux).ServeHTTP(0xc0001fc800, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2387 +0x1a5 net/http.serverHandler.ServeHTTP(0xc0001781c0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2807 +0xa3 net/http.(*conn).serve(0xc0001a8000, 0x2934420, 0xc000212400) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1895 +0x86c created by net/http.(*Server).Serve /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2933 +0x35c Eg: server A (datacenter 1) and server B (datacenter 2) hold replica (100) for volume 1. If you upload a file with a key 1,xxxxx to server C (datacenter 3) will trigger the panic on server C. The server C should either proxy upload file to the correct volume server or should return an HTTP error code and not panic.
5 years ago
volume: proxy writes to remote volume server, with replication or not the panic is triggered by uploading a file to a volume server not holding the designated replica. 2020-03-15 10:20:14.365488 I | http: panic serving 127.0.0.1:57124: runtime error: invalid memory address or nil pointer dereference goroutine 119 [running]: net/http.(*conn).serve.func1(0xc0001a8000) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1772 +0x139 panic(0x2316fe0, 0x3662900) /home/travis/.gimme/versions/go1.14.linux.amd64/src/runtime/panic.go:973 +0x396 github.com/chrislusf/seaweedfs/weed/topology.getWritableRemoteReplications(0xc00009c000, 0x2, 0x7ffeefbffbd2, 0xe, 0x0, 0xa, 0x0, 0x0, 0xbb4bf1f7) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:157 +0x53 github.com/chrislusf/seaweedfs/weed/topology.ReplicatedWrite(0x7ffeefbffbd2, 0xe, 0xc00009c000, 0xc000000002, 0xc000472750, 0xc0001b2200, 0x0, 0x1, 0x0) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:29 +0xc7 github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).PostHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers_write.go:52 +0x56f github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).privateStoreHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers.go:37 +0x21f net/http.HandlerFunc.ServeHTTP(0xc0004420e0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2012 +0x44 net/http.(*ServeMux).ServeHTTP(0xc0001fc800, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2387 +0x1a5 net/http.serverHandler.ServeHTTP(0xc0001781c0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2807 +0xa3 net/http.(*conn).serve(0xc0001a8000, 0x2934420, 0xc000212400) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1895 +0x86c created by net/http.(*Server).Serve /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2933 +0x35c Eg: server A (datacenter 1) and server B (datacenter 2) hold replica (100) for volume 1. If you upload a file with a key 1,xxxxx to server C (datacenter 3) will trigger the panic on server C. The server C should either proxy upload file to the correct volume server or should return an HTTP error code and not panic.
5 years ago
volume: proxy writes to remote volume server, with replication or not the panic is triggered by uploading a file to a volume server not holding the designated replica. 2020-03-15 10:20:14.365488 I | http: panic serving 127.0.0.1:57124: runtime error: invalid memory address or nil pointer dereference goroutine 119 [running]: net/http.(*conn).serve.func1(0xc0001a8000) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1772 +0x139 panic(0x2316fe0, 0x3662900) /home/travis/.gimme/versions/go1.14.linux.amd64/src/runtime/panic.go:973 +0x396 github.com/chrislusf/seaweedfs/weed/topology.getWritableRemoteReplications(0xc00009c000, 0x2, 0x7ffeefbffbd2, 0xe, 0x0, 0xa, 0x0, 0x0, 0xbb4bf1f7) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:157 +0x53 github.com/chrislusf/seaweedfs/weed/topology.ReplicatedWrite(0x7ffeefbffbd2, 0xe, 0xc00009c000, 0xc000000002, 0xc000472750, 0xc0001b2200, 0x0, 0x1, 0x0) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/topology/store_replicate.go:29 +0xc7 github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).PostHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers_write.go:52 +0x56f github.com/chrislusf/seaweedfs/weed/server.(*VolumeServer).privateStoreHandler(0xc0001513f0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/gopath/src/github.com/chrislusf/seaweedfs/weed/server/volume_server_handlers.go:37 +0x21f net/http.HandlerFunc.ServeHTTP(0xc0004420e0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2012 +0x44 net/http.(*ServeMux).ServeHTTP(0xc0001fc800, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2387 +0x1a5 net/http.serverHandler.ServeHTTP(0xc0001781c0, 0x292bde0, 0xc0001fe2a0, 0xc0001b2200) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2807 +0xa3 net/http.(*conn).serve(0xc0001a8000, 0x2934420, 0xc000212400) /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:1895 +0x86c created by net/http.(*Server).Serve /home/travis/.gimme/versions/go1.14.linux.amd64/src/net/http/server.go:2933 +0x35c Eg: server A (datacenter 1) and server B (datacenter 2) hold replica (100) for volume 1. If you upload a file with a key 1,xxxxx to server C (datacenter 3) will trigger the panic on server C. The server C should either proxy upload file to the correct volume server or should return an HTTP error code and not panic.
5 years ago
5 years ago
  1. package topology
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "fmt"
  6. "google.golang.org/grpc"
  7. "net/http"
  8. "net/url"
  9. "strconv"
  10. "strings"
  11. "github.com/seaweedfs/seaweedfs/weed/glog"
  12. "github.com/seaweedfs/seaweedfs/weed/operation"
  13. "github.com/seaweedfs/seaweedfs/weed/security"
  14. "github.com/seaweedfs/seaweedfs/weed/stats"
  15. "github.com/seaweedfs/seaweedfs/weed/storage"
  16. "github.com/seaweedfs/seaweedfs/weed/storage/needle"
  17. "github.com/seaweedfs/seaweedfs/weed/storage/types"
  18. "github.com/seaweedfs/seaweedfs/weed/util"
  19. )
  20. func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption, s *storage.Store, volumeId needle.VolumeId, n *needle.Needle, r *http.Request) (isUnchanged bool, err error) {
  21. //check JWT
  22. jwt := security.GetJwt(r)
  23. // check whether this is a replicated write request
  24. var remoteLocations []operation.Location
  25. if r.FormValue("type") != "replicate" {
  26. // this is the initial request
  27. remoteLocations, err = GetWritableRemoteReplications(s, grpcDialOption, volumeId, masterFn)
  28. if err != nil {
  29. glog.V(0).Infoln(err)
  30. return
  31. }
  32. }
  33. // read fsync value
  34. fsync := false
  35. if r.FormValue("fsync") == "true" {
  36. fsync = true
  37. }
  38. if s.GetVolume(volumeId) != nil {
  39. isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync)
  40. if err != nil {
  41. stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc()
  42. err = fmt.Errorf("failed to write to local disk: %v", err)
  43. glog.V(0).Infoln(err)
  44. return
  45. }
  46. }
  47. if len(remoteLocations) > 0 { //send to other replica locations
  48. if err = DistributedOperation(remoteLocations, func(location operation.Location) error {
  49. u := url.URL{
  50. Scheme: "http",
  51. Host: location.Url,
  52. Path: r.URL.Path,
  53. }
  54. q := url.Values{
  55. "type": {"replicate"},
  56. "ttl": {n.Ttl.String()},
  57. }
  58. if n.LastModified > 0 {
  59. q.Set("ts", strconv.FormatUint(n.LastModified, 10))
  60. }
  61. if n.IsChunkedManifest() {
  62. q.Set("cm", "true")
  63. }
  64. u.RawQuery = q.Encode()
  65. pairMap := make(map[string]string)
  66. if n.HasPairs() {
  67. tmpMap := make(map[string]string)
  68. err := json.Unmarshal(n.Pairs, &tmpMap)
  69. if err != nil {
  70. stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc()
  71. glog.V(0).Infoln("Unmarshal pairs error:", err)
  72. }
  73. for k, v := range tmpMap {
  74. pairMap[needle.PairNamePrefix+k] = v
  75. }
  76. }
  77. // volume server do not know about encryption
  78. // TODO optimize here to compress data only once
  79. uploadOption := &operation.UploadOption{
  80. UploadUrl: u.String(),
  81. Filename: string(n.Name),
  82. Cipher: false,
  83. IsInputCompressed: n.IsCompressed(),
  84. MimeType: string(n.Mime),
  85. PairMap: pairMap,
  86. Jwt: jwt,
  87. }
  88. _, err := operation.UploadData(n.Data, uploadOption)
  89. return err
  90. }); err != nil {
  91. stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc()
  92. err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)
  93. glog.V(0).Infoln(err)
  94. return false, err
  95. }
  96. }
  97. return
  98. }
  99. func ReplicatedDelete(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption, store *storage.Store, volumeId needle.VolumeId, n *needle.Needle, r *http.Request) (size types.Size, err error) {
  100. //check JWT
  101. jwt := security.GetJwt(r)
  102. var remoteLocations []operation.Location
  103. if r.FormValue("type") != "replicate" {
  104. remoteLocations, err = GetWritableRemoteReplications(store, grpcDialOption, volumeId, masterFn)
  105. if err != nil {
  106. glog.V(0).Infoln(err)
  107. return
  108. }
  109. }
  110. size, err = store.DeleteVolumeNeedle(volumeId, n)
  111. if err != nil {
  112. glog.V(0).Infoln("delete error:", err)
  113. return
  114. }
  115. if len(remoteLocations) > 0 { //send to other replica locations
  116. if err = DistributedOperation(remoteLocations, func(location operation.Location) error {
  117. return util.Delete("http://"+location.Url+r.URL.Path+"?type=replicate", string(jwt))
  118. }); err != nil {
  119. size = 0
  120. }
  121. }
  122. return
  123. }
  124. type DistributedOperationResult map[string]error
  125. func (dr DistributedOperationResult) Error() error {
  126. var errs []string
  127. for k, v := range dr {
  128. if v != nil {
  129. errs = append(errs, fmt.Sprintf("[%s]: %v", k, v))
  130. }
  131. }
  132. if len(errs) == 0 {
  133. return nil
  134. }
  135. return errors.New(strings.Join(errs, "\n"))
  136. }
  137. type RemoteResult struct {
  138. Host string
  139. Error error
  140. }
  141. func DistributedOperation(locations []operation.Location, op func(location operation.Location) error) error {
  142. length := len(locations)
  143. results := make(chan RemoteResult)
  144. for _, location := range locations {
  145. go func(location operation.Location, results chan RemoteResult) {
  146. results <- RemoteResult{location.Url, op(location)}
  147. }(location, results)
  148. }
  149. ret := DistributedOperationResult(make(map[string]error))
  150. for i := 0; i < length; i++ {
  151. result := <-results
  152. ret[result.Host] = result.Error
  153. }
  154. return ret.Error()
  155. }
  156. func GetWritableRemoteReplications(s *storage.Store, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, masterFn operation.GetMasterFn) (remoteLocations []operation.Location, err error) {
  157. v := s.GetVolume(volumeId)
  158. if v != nil && v.ReplicaPlacement.GetCopyCount() == 1 {
  159. return
  160. }
  161. // not on local store, or has replications
  162. lookupResult, lookupErr := operation.LookupVolumeId(masterFn, grpcDialOption, volumeId.String())
  163. if lookupErr == nil {
  164. selfUrl := util.JoinHostPort(s.Ip, s.Port)
  165. for _, location := range lookupResult.Locations {
  166. if location.Url != selfUrl {
  167. remoteLocations = append(remoteLocations, location)
  168. }
  169. }
  170. } else {
  171. err = fmt.Errorf("failed to lookup for %d: %v", volumeId, lookupErr)
  172. return
  173. }
  174. if v != nil {
  175. // has one local and has remote replications
  176. copyCount := v.ReplicaPlacement.GetCopyCount()
  177. if len(lookupResult.Locations) < copyCount {
  178. err = fmt.Errorf("replicating opetations [%d] is less than volume %d replication copy count [%d]",
  179. len(lookupResult.Locations), volumeId, copyCount)
  180. }
  181. }
  182. return
  183. }