Merge pull request #6 from chrislusf/master

sync

Makefile

@@ -40,7 +40,7 @@ linux: deps
 	mkdir -p linux
 	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o linux/$(BINARY) $(SOURCE_DIR)
 
-release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
+release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_arm64_build 5_byte_darwin_build 5_byte_windows_build
 
 ##### LINUX BUILDS #####
 5_byte_linux_build:
@@ -55,6 +55,14 @@ release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_buil
 	$(call build_large,windows,amd64,.exe)
 	$(call zip_large,windows,amd64,.exe)
 
+5_byte_arm_build: $(sources)
+	$(call build_large,linux,arm,)
+	$(call tar_large,linux,arm)
+
+5_byte_arm64_build: $(sources)
+	$(call build_large,linux,arm64,)
+	$(call tar_large,linux,arm64)
+
 linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
 
 build/linux_386.tar.gz: $(sources)

other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java

@@ -6,7 +6,6 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 public class FileChunkManifest {
 
@@ -51,13 +50,17 @@ public class FileChunkManifest {
 
     private static byte[] fetchChunk(final FilerGrpcClient filerGrpcClient, FilerProto.FileChunk chunk) throws IOException {
 
-        FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
         String vid = "" + chunk.getFid().getVolumeId();
-        lookupRequest.addVolumeIds(vid);
-        FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
-                .getBlockingStub().lookupVolume(lookupRequest.build());
-        Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
-        FilerProto.Locations locations = vid2Locations.get(vid);
+        FilerProto.Locations locations = filerGrpcClient.vidLocations.get(vid);
+        if (locations == null) {
+            FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
+            lookupRequest.addVolumeIds(vid);
+            FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
+                    .getBlockingStub().lookupVolume(lookupRequest.build());
+            locations = lookupResponse.getLocationsMapMap().get(vid);
+            filerGrpcClient.vidLocations.put(vid, locations);
+            LOG.debug("fetchChunk vid:{} locations:{}", vid, locations);
+        }
 
         SeaweedRead.ChunkView chunkView = new SeaweedRead.ChunkView(
                 FilerClient.toFileId(chunk.getFid()), // avoid deprecated chunk.getFileId()

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -9,6 +9,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLException;
+import java.util.Map;
+import java.util.HashMap;
 import java.util.concurrent.TimeUnit;
 
 public class FilerGrpcClient {
@@ -24,6 +26,7 @@ public class FilerGrpcClient {
         }
     }
 
+    public final Map<String, FilerProto.Locations> vidLocations = new HashMap<>();
     private final ManagedChannel channel;
     private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
     private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -15,7 +15,7 @@ public class SeaweedRead {
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
 
-    static ChunkCache chunkCache = new ChunkCache(0);
+    static ChunkCache chunkCache = new ChunkCache(4);
 
     // returns bytesRead
     public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
@@ -62,6 +62,7 @@ public class SeaweedRead {
 
         if (chunkData == null) {
             chunkData = doFetchFullChunkData(chunkView, locations);
+            chunkCache.setChunk(chunkView.fileId, chunkData);
         }
 
         int len = (int) chunkView.size;
@@ -69,8 +70,6 @@ public class SeaweedRead {
                 chunkView.fileId, chunkData.length, chunkView.offset, buffer.length, startOffset, len);
         System.arraycopy(chunkData, (int) chunkView.offset, buffer, startOffset, len);
 
-        chunkCache.setChunk(chunkView.fileId, chunkData);
-
         return len;
     }
 
@@ -109,6 +108,8 @@ public class SeaweedRead {
             }
         }
 
+        LOG.debug("doFetchFullChunkData fid:{} chunkData.length:{}", chunkView.fileId, data.length);
+
         return data;
 
     }

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -61,9 +61,6 @@ public class SeaweedWrite {
 
         String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength, cipherKey);
 
-        // cache fileId ~ bytes
-        SeaweedRead.chunkCache.setChunk(fileId, bytes);
-
         LOG.debug("write file chunk {} size {}", targetUrl, bytesLength);
 
         return FilerProto.FileChunk.newBuilder()

weed/command/mount.go

@@ -9,7 +9,6 @@ type MountOptions struct {
 	filerMountRootPath          *string
 	dir                         *string
 	dirAutoCreate               *bool
-	dirListCacheLimit           *int64
 	collection                  *string
 	replication                 *string
 	ttlSec                      *int
@@ -35,7 +34,6 @@ func init() {
 	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
 	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
 	mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to")
-	mountOptions.dirListCacheLimit = cmdMount.Flag.Int64("dirListCacheLimit", 1000000, "limit cache size to speed up directory long format listing")
 	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
 	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")

weed/command/mount_std.go

@@ -165,7 +165,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		CacheDir:                    *option.cacheDir,
 		CacheSizeMB:                 *option.cacheSizeMB,
 		DataCenter:                  *option.dataCenter,
-		DirListCacheLimit:           *option.dirListCacheLimit,
 		EntryCacheTtl:               3 * time.Second,
 		MountUid:                    uid,
 		MountGid:                    gid,

weed/command/s3.go

@@ -151,6 +151,7 @@ func (s3opt *S3Options) startS3Server() bool {
 
 	_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
 		Filer:            *s3opt.filer,
+		Port:             *s3opt.port,
 		FilerGrpcAddress: filerGrpcAddress,
 		Config:           *s3opt.config,
 		DomainName:       *s3opt.domainName,

weed/filesys/dir.go

@@ -3,6 +3,7 @@ package filesys
 import (
 	"bytes"
 	"context"
+	"math"
 	"os"
 	"strings"
 	"time"
@@ -277,7 +278,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 	dirPath := util.FullPath(dir.FullPath())
 	meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath)
-	listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(dir.wfs.option.DirListCacheLimit))
+	listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(math.MaxInt32))
 	if listErr != nil {
 		glog.Errorf("list meta cache: %v", listErr)
 		return nil, fuse.EIO

weed/filesys/wfs.go

@@ -34,7 +34,6 @@ type Option struct {
 	CacheDir           string
 	CacheSizeMB        int64
 	DataCenter         string
-	DirListCacheLimit  int64
 	EntryCacheTtl      time.Duration
 	Umask              os.FileMode
 

weed/s3api/s3api_server.go

@@ -1,6 +1,7 @@
 package s3api
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -9,6 +10,7 @@ import (
 
 type S3ApiServerOption struct {
 	Filer            string
+	Port             int
 	FilerGrpcAddress string
 	Config           string
 	DomainName       string
@@ -37,7 +39,10 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 	apiRouter := router.PathPrefix("/").Subrouter()
 	var routers []*mux.Router
 	if s3a.option.DomainName != "" {
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+s3a.option.DomainName).Subrouter())
+		routers = append(routers, apiRouter.Host(
+			fmt.Sprintf("%s.%s:%d", "{bucket:.+}", s3a.option.DomainName, s3a.option.Port)).Subrouter())
+		routers = append(routers, apiRouter.Host(
+			fmt.Sprintf("%s.%s", "{bucket:.+}", s3a.option.DomainName)).Subrouter())
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())