Browse Source

add volume id cache

logrus
Chris Lu 4 years ago
parent
commit
a2962604ad
  1. 1
      other/java/client/src/main/java/seaweedfs/client/ChunkCache.java
  2. 11
      other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
  3. 36
      other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java

1
other/java/client/src/main/java/seaweedfs/client/ChunkCache.java

@ -15,7 +15,6 @@ public class ChunkCache {
}
this.cache = CacheBuilder.newBuilder()
.maximumSize(maxEntries)
.weakValues()
.expireAfterAccess(1, TimeUnit.HOURS)
.build();
}

11
other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@ -19,6 +19,7 @@ public class SeaweedRead {
private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
static ChunkCache chunkCache = new ChunkCache(4);
static VolumeIdCache volumeIdCache = new VolumeIdCache(4 * 1024);
// returns bytesRead
public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
@ -30,13 +31,19 @@ public class SeaweedRead {
FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
for (ChunkView chunkView : chunkViews) {
String vid = parseVolumeId(chunkView.fileId);
if (volumeIdCache.getLocations(vid)==null){
lookupRequest.addVolumeIds(vid);
}
}
if (lookupRequest.getVolumeIdsCount()>0){
FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
.getBlockingStub().lookupVolume(lookupRequest.build());
Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
for (Map.Entry<String,FilerProto.Locations> entry : vid2Locations.entrySet()) {
volumeIdCache.setLocations(entry.getKey(), entry.getValue());
}
}
//TODO parallel this
long readCount = 0;
@ -50,7 +57,7 @@ public class SeaweedRead {
startOffset += gap;
}
FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId));
FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId));
if (locations == null || locations.getLocationsCount() == 0) {
LOG.error("failed to locate {}", chunkView.fileId);
// log here!

36
other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java

@ -0,0 +1,36 @@
package seaweedfs.client;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import java.util.concurrent.TimeUnit;
public class VolumeIdCache {
private Cache<String, FilerProto.Locations> cache = null;
public VolumeIdCache(int maxEntries) {
if (maxEntries == 0) {
return;
}
this.cache = CacheBuilder.newBuilder()
.maximumSize(maxEntries)
.expireAfterAccess(1, TimeUnit.HOURS)
.build();
}
public FilerProto.Locations getLocations(String volumeId) {
if (this.cache == null) {
return null;
}
return this.cache.getIfPresent(volumeId);
}
public void setLocations(String volumeId, FilerProto.Locations locations) {
if (this.cache == null) {
return;
}
this.cache.put(volumeId, locations);
}
}
Loading…
Cancel
Save