hilimd
5 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 982 additions and 463 deletions
-
12README.md
-
1go.mod
-
4go.sum
-
2k8s/seaweedfs/Chart.yaml
-
2k8s/seaweedfs/values.yaml
-
2other/java/client/pom.xml
-
2other/java/client/pom.xml.deploy
-
2other/java/client/pom_debug.xml
-
3other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java
-
176other/java/hdfs2/dependency-reduced-pom.xml
-
4other/java/hdfs2/pom.xml
-
2other/java/hdfs3/dependency-reduced-pom.xml
-
4other/java/hdfs3/pom.xml
-
73unmaintained/s3/presigned_put/presigned_put.go
-
3weed/filer2/stream.go
-
14weed/filesys/dir.go
-
1weed/filesys/dir_rename.go
-
1weed/filesys/file.go
-
9weed/filesys/filehandle.go
-
207weed/filesys/fscache.go
-
96weed/filesys/fscache_test.go
-
4weed/filesys/wfs.go
-
2weed/operation/needle_parse_test.go
-
12weed/operation/upload_content.go
-
14weed/s3api/auth_credentials.go
-
4weed/s3api/s3api_object_handlers.go
-
4weed/s3api/s3api_object_multipart_handlers.go
-
205weed/s3api/s3api_objects_list_handlers.go
-
211weed/server/filer_server_handlers_write.go
-
187weed/server/filer_server_handlers_write_autochunk.go
-
1weed/server/filer_server_handlers_write_cipher.go
-
21weed/server/master_grpc_server.go
-
24weed/server/volume_grpc_client_to_master.go
-
3weed/server/volume_server_handlers_write.go
-
3weed/storage/needle/needle.go
-
9weed/storage/needle/needle_parse_upload.go
-
2weed/storage/needle/needle_read_write.go
-
25weed/util/bytes.go
-
2weed/util/constants.go
@ -1,4 +1,4 @@ |
|||||
apiVersion: v1 |
apiVersion: v1 |
||||
description: SeaweedFS |
description: SeaweedFS |
||||
name: seaweedfs |
name: seaweedfs |
||||
version: 1.87 |
|
||||
|
version: 1.88 |
||||
@ -0,0 +1,73 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"github.com/aws/aws-sdk-go/aws" |
||||
|
"github.com/aws/aws-sdk-go/aws/session" |
||||
|
"github.com/aws/aws-sdk-go/service/s3" |
||||
|
"encoding/base64" |
||||
|
"fmt" |
||||
|
"crypto/md5" |
||||
|
"strings" |
||||
|
"time" |
||||
|
"net/http" |
||||
|
) |
||||
|
|
||||
|
// Downloads an item from an S3 Bucket in the region configured in the shared config
|
||||
|
// or AWS_REGION environment variable.
|
||||
|
//
|
||||
|
// Usage:
|
||||
|
// go run presigned_put.go
|
||||
|
// For this exampl to work, the domainName is needd
|
||||
|
// weed s3 -domainName=localhost
|
||||
|
func main() { |
||||
|
h := md5.New() |
||||
|
content := strings.NewReader(stringContent) |
||||
|
content.WriteTo(h) |
||||
|
|
||||
|
// Initialize a session in us-west-2 that the SDK will use to load
|
||||
|
// credentials from the shared credentials file ~/.aws/credentials.
|
||||
|
sess, err := session.NewSession(&aws.Config{ |
||||
|
Region: aws.String("us-east-1"), |
||||
|
Endpoint: aws.String("http://localhost:8333"), |
||||
|
}) |
||||
|
|
||||
|
// Create S3 service client
|
||||
|
svc := s3.New(sess) |
||||
|
|
||||
|
putRequest, output := svc.PutObjectRequest(&s3.PutObjectInput{ |
||||
|
Bucket: aws.String("dev"), |
||||
|
Key: aws.String("testKey"), |
||||
|
}) |
||||
|
fmt.Printf("output: %+v\n", output) |
||||
|
|
||||
|
md5s := base64.StdEncoding.EncodeToString(h.Sum(nil)) |
||||
|
putRequest.HTTPRequest.Header.Set("Content-MD5", md5s) |
||||
|
|
||||
|
url, err := putRequest.Presign(15 * time.Minute) |
||||
|
if err != nil { |
||||
|
fmt.Println("error presigning request", err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
fmt.Println(url) |
||||
|
|
||||
|
req, err := http.NewRequest("PUT", url, strings.NewReader(stringContent)) |
||||
|
req.Header.Set("Content-MD5", md5s) |
||||
|
if err != nil { |
||||
|
fmt.Println("error creating request", url) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
resp, err := http.DefaultClient.Do(req) |
||||
|
if err != nil { |
||||
|
fmt.Printf("error put request: %v\n", err) |
||||
|
return |
||||
|
} |
||||
|
fmt.Printf("response: %+v\n", resp) |
||||
|
} |
||||
|
|
||||
|
var stringContent = `Generate a Pre-Signed URL for an Amazon S3 PUT Operation with a Specific Payload |
||||
|
You can generate a pre-signed URL for a PUT operation that checks whether users upload the correct content. When the SDK pre-signs a request, it computes the checksum of the request body and generates an MD5 checksum that is included in the pre-signed URL. Users must upload the same content that produces the same MD5 checksum generated by the SDK; otherwise, the operation fails. This is not the Content-MD5, but the signature. To enforce Content-MD5, simply add the header to the request. |
||||
|
|
||||
|
The following example adds a Body field to generate a pre-signed PUT operation that requires a specific payload to be uploaded by users. |
||||
|
` |
||||
@ -0,0 +1,207 @@ |
|||||
|
package filesys |
||||
|
|
||||
|
import ( |
||||
|
"sync" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
"github.com/seaweedfs/fuse/fs" |
||||
|
) |
||||
|
|
||||
|
type FsCache struct { |
||||
|
root *FsNode |
||||
|
sync.RWMutex |
||||
|
} |
||||
|
type FsNode struct { |
||||
|
parent *FsNode |
||||
|
node fs.Node |
||||
|
name string |
||||
|
childrenLock sync.RWMutex |
||||
|
children map[string]*FsNode |
||||
|
} |
||||
|
|
||||
|
func newFsCache(root fs.Node) *FsCache { |
||||
|
return &FsCache{ |
||||
|
root: &FsNode{ |
||||
|
node: root, |
||||
|
}, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (c *FsCache) GetFsNode(path util.FullPath) fs.Node { |
||||
|
|
||||
|
c.RLock() |
||||
|
defer c.RUnlock() |
||||
|
|
||||
|
return c.doGetFsNode(path) |
||||
|
} |
||||
|
|
||||
|
func (c *FsCache) doGetFsNode(path util.FullPath) fs.Node { |
||||
|
t := c.root |
||||
|
for _, p := range path.Split() { |
||||
|
t = t.findChild(p) |
||||
|
if t == nil { |
||||
|
return nil |
||||
|
} |
||||
|
} |
||||
|
return t.node |
||||
|
} |
||||
|
|
||||
|
func (c *FsCache) SetFsNode(path util.FullPath, node fs.Node) { |
||||
|
|
||||
|
c.Lock() |
||||
|
defer c.Unlock() |
||||
|
|
||||
|
c.doSetFsNode(path, node) |
||||
|
} |
||||
|
|
||||
|
func (c *FsCache) doSetFsNode(path util.FullPath, node fs.Node) { |
||||
|
t := c.root |
||||
|
for _, p := range path.Split() { |
||||
|
t = t.ensureChild(p) |
||||
|
} |
||||
|
t.node = node |
||||
|
} |
||||
|
|
||||
|
func (c *FsCache) EnsureFsNode(path util.FullPath, genNodeFn func() fs.Node) fs.Node { |
||||
|
|
||||
|
c.Lock() |
||||
|
defer c.Unlock() |
||||
|
|
||||
|
t := c.doGetFsNode(path) |
||||
|
if t != nil { |
||||
|
return t |
||||
|
} |
||||
|
t = genNodeFn() |
||||
|
c.doSetFsNode(path, t) |
||||
|
return t |
||||
|
} |
||||
|
|
||||
|
func (c *FsCache) DeleteFsNode(path util.FullPath) { |
||||
|
|
||||
|
c.Lock() |
||||
|
defer c.Unlock() |
||||
|
|
||||
|
t := c.root |
||||
|
for _, p := range path.Split() { |
||||
|
t = t.findChild(p) |
||||
|
if t == nil { |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
if t.parent != nil { |
||||
|
t.parent.disconnectChild(t) |
||||
|
} |
||||
|
t.deleteSelf() |
||||
|
} |
||||
|
|
||||
|
// oldPath and newPath are full path including the new name
|
||||
|
func (c *FsCache) Move(oldPath util.FullPath, newPath util.FullPath) *FsNode { |
||||
|
|
||||
|
c.Lock() |
||||
|
defer c.Unlock() |
||||
|
|
||||
|
// find old node
|
||||
|
src := c.root |
||||
|
for _, p := range oldPath.Split() { |
||||
|
src = src.findChild(p) |
||||
|
if src == nil { |
||||
|
return src |
||||
|
} |
||||
|
} |
||||
|
if src.parent != nil { |
||||
|
src.parent.disconnectChild(src) |
||||
|
} |
||||
|
|
||||
|
// find new node
|
||||
|
target := c.root |
||||
|
for _, p := range newPath.Split() { |
||||
|
target = target.ensureChild(p) |
||||
|
} |
||||
|
parent := target.parent |
||||
|
src.name = target.name |
||||
|
if dir, ok := src.node.(*Dir); ok { |
||||
|
dir.name = target.name // target is not Dir, but a shortcut
|
||||
|
} |
||||
|
if f, ok := src.node.(*File); ok { |
||||
|
f.Name = target.name |
||||
|
if f.entry != nil { |
||||
|
f.entry.Name = f.Name |
||||
|
} |
||||
|
} |
||||
|
parent.disconnectChild(target) |
||||
|
|
||||
|
target.deleteSelf() |
||||
|
|
||||
|
src.connectToParent(parent) |
||||
|
|
||||
|
return src |
||||
|
} |
||||
|
|
||||
|
func (n *FsNode) connectToParent(parent *FsNode) { |
||||
|
n.parent = parent |
||||
|
oldNode := parent.findChild(n.name) |
||||
|
if oldNode != nil { |
||||
|
oldNode.deleteSelf() |
||||
|
} |
||||
|
if dir, ok := n.node.(*Dir); ok { |
||||
|
dir.parent = parent.node.(*Dir) |
||||
|
} |
||||
|
if f, ok := n.node.(*File); ok { |
||||
|
f.dir = parent.node.(*Dir) |
||||
|
} |
||||
|
n.childrenLock.Lock() |
||||
|
parent.children[n.name] = n |
||||
|
n.childrenLock.Unlock() |
||||
|
} |
||||
|
|
||||
|
func (n *FsNode) findChild(name string) *FsNode { |
||||
|
n.childrenLock.RLock() |
||||
|
defer n.childrenLock.RUnlock() |
||||
|
|
||||
|
child, found := n.children[name] |
||||
|
if found { |
||||
|
return child |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (n *FsNode) ensureChild(name string) *FsNode { |
||||
|
n.childrenLock.Lock() |
||||
|
defer n.childrenLock.Unlock() |
||||
|
|
||||
|
if n.children == nil { |
||||
|
n.children = make(map[string]*FsNode) |
||||
|
} |
||||
|
child, found := n.children[name] |
||||
|
if found { |
||||
|
return child |
||||
|
} |
||||
|
t := &FsNode{ |
||||
|
parent: n, |
||||
|
node: nil, |
||||
|
name: name, |
||||
|
children: nil, |
||||
|
} |
||||
|
n.children[name] = t |
||||
|
return t |
||||
|
} |
||||
|
|
||||
|
func (n *FsNode) disconnectChild(child *FsNode) { |
||||
|
n.childrenLock.Lock() |
||||
|
delete(n.children, child.name) |
||||
|
n.childrenLock.Unlock() |
||||
|
child.parent = nil |
||||
|
} |
||||
|
|
||||
|
func (n *FsNode) deleteSelf() { |
||||
|
n.childrenLock.Lock() |
||||
|
for _, child := range n.children { |
||||
|
child.deleteSelf() |
||||
|
} |
||||
|
n.children = nil |
||||
|
n.childrenLock.Unlock() |
||||
|
|
||||
|
n.node = nil |
||||
|
n.parent = nil |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,96 @@ |
|||||
|
package filesys |
||||
|
|
||||
|
import ( |
||||
|
"testing" |
||||
|
|
||||
|
"github.com/chrislusf/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
func TestPathSplit(t *testing.T) { |
||||
|
parts := util.FullPath("/").Split() |
||||
|
if len(parts) != 0 { |
||||
|
t.Errorf("expecting an empty list, but getting %d", len(parts)) |
||||
|
} |
||||
|
|
||||
|
parts = util.FullPath("/readme.md").Split() |
||||
|
if len(parts) != 1 { |
||||
|
t.Errorf("expecting an empty list, but getting %d", len(parts)) |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func TestFsCache(t *testing.T) { |
||||
|
|
||||
|
cache := newFsCache(nil) |
||||
|
|
||||
|
x := cache.GetFsNode(util.FullPath("/y/x")) |
||||
|
if x != nil { |
||||
|
t.Errorf("wrong node!") |
||||
|
} |
||||
|
|
||||
|
p := util.FullPath("/a/b/c") |
||||
|
cache.SetFsNode(p, &File{Name: "cc"}) |
||||
|
tNode := cache.GetFsNode(p) |
||||
|
tFile := tNode.(*File) |
||||
|
if tFile.Name != "cc" { |
||||
|
t.Errorf("expecting a FsNode") |
||||
|
} |
||||
|
|
||||
|
cache.SetFsNode(util.FullPath("/a/b/d"), &File{Name: "dd"}) |
||||
|
cache.SetFsNode(util.FullPath("/a/b/e"), &File{Name: "ee"}) |
||||
|
cache.SetFsNode(util.FullPath("/a/b/f"), &File{Name: "ff"}) |
||||
|
cache.SetFsNode(util.FullPath("/z"), &File{Name: "zz"}) |
||||
|
cache.SetFsNode(util.FullPath("/a"), &File{Name: "aa"}) |
||||
|
|
||||
|
b := cache.GetFsNode(util.FullPath("/a/b")) |
||||
|
if b != nil { |
||||
|
t.Errorf("unexpected node!") |
||||
|
} |
||||
|
|
||||
|
a := cache.GetFsNode(util.FullPath("/a")) |
||||
|
if a == nil { |
||||
|
t.Errorf("missing node!") |
||||
|
} |
||||
|
|
||||
|
cache.DeleteFsNode(util.FullPath("/a")) |
||||
|
if b != nil { |
||||
|
t.Errorf("unexpected node!") |
||||
|
} |
||||
|
|
||||
|
a = cache.GetFsNode(util.FullPath("/a")) |
||||
|
if a != nil { |
||||
|
t.Errorf("wrong DeleteFsNode!") |
||||
|
} |
||||
|
|
||||
|
z := cache.GetFsNode(util.FullPath("/z")) |
||||
|
if z == nil { |
||||
|
t.Errorf("missing node!") |
||||
|
} |
||||
|
|
||||
|
y := cache.GetFsNode(util.FullPath("/x/y")) |
||||
|
if y != nil { |
||||
|
t.Errorf("wrong node!") |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
func TestFsCacheMove(t *testing.T) { |
||||
|
|
||||
|
cache := newFsCache(nil) |
||||
|
|
||||
|
cache.SetFsNode(util.FullPath("/a/b/d"), &File{Name: "dd"}) |
||||
|
cache.SetFsNode(util.FullPath("/a/b/e"), &File{Name: "ee"}) |
||||
|
cache.SetFsNode(util.FullPath("/z"), &File{Name: "zz"}) |
||||
|
cache.SetFsNode(util.FullPath("/a"), &File{Name: "aa"}) |
||||
|
|
||||
|
cache.Move(util.FullPath("/a/b"), util.FullPath("/z/x")) |
||||
|
|
||||
|
d := cache.GetFsNode(util.FullPath("/z/x/d")) |
||||
|
if d == nil { |
||||
|
t.Errorf("unexpected nil node!") |
||||
|
} |
||||
|
if d.(*File).Name != "dd" { |
||||
|
t.Errorf("unexpected non dd node!") |
||||
|
} |
||||
|
|
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue