gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

add volume id cache

Browse source
This commit is contained in:
Chris Lu 2020-11-12 23:25:11 -08:00
parent 559dfacdac
commit a2962604ad
3 changed files with 49 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
other/java/client/src/main/java/seaweedfs/client/ChunkCache.java
View file

@ -15,7 +15,6 @@ public class ChunkCache {
} }
this.cache = CacheBuilder.newBuilder() this.cache = CacheBuilder.newBuilder()
.maximumSize(maxEntries) .maximumSize(maxEntries)
.weakValues()
.expireAfterAccess(1, TimeUnit.HOURS) .expireAfterAccess(1, TimeUnit.HOURS)
.build(); .build();
} }

19
other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java
View file

@ -19,6 +19,7 @@ public class SeaweedRead {
private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class); private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
static ChunkCache chunkCache = new ChunkCache(4); static ChunkCache chunkCache = new ChunkCache(4);
static VolumeIdCache volumeIdCache = new VolumeIdCache(4 * 1024);
// returns bytesRead // returns bytesRead
public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals, public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
@ -30,13 +31,19 @@ public class SeaweedRead {
FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
for (ChunkView chunkView : chunkViews) { for (ChunkView chunkView : chunkViews) {
String vid = parseVolumeId(chunkView.fileId); String vid = parseVolumeId(chunkView.fileId);
lookupRequest.addVolumeIds(vid); if (volumeIdCache.getLocations(vid)==null){
lookupRequest.addVolumeIds(vid);
}
} }
FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient if (lookupRequest.getVolumeIdsCount()>0){
.getBlockingStub().lookupVolume(lookupRequest.build()); FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
.getBlockingStub().lookupVolume(lookupRequest.build());
Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap(); Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
for (Map.Entry<String,FilerProto.Locations> entry : vid2Locations.entrySet()) {
volumeIdCache.setLocations(entry.getKey(), entry.getValue());
}
}
//TODO parallel this //TODO parallel this
long readCount = 0; long readCount = 0;
@ -50,7 +57,7 @@ public class SeaweedRead {
startOffset += gap; startOffset += gap;
} }
FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId)); FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId));
if (locations == null || locations.getLocationsCount() == 0) { if (locations == null || locations.getLocationsCount() == 0) {
LOG.error("failed to locate {}", chunkView.fileId); LOG.error("failed to locate {}", chunkView.fileId);
// log here! // log here!

36
other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java Normal file
View file

@ -0,0 +1,36 @@
package seaweedfs.client;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import java.util.concurrent.TimeUnit;
public class VolumeIdCache {
private Cache<String, FilerProto.Locations> cache = null;
public VolumeIdCache(int maxEntries) {
if (maxEntries == 0) {
return;
}
this.cache = CacheBuilder.newBuilder()
.maximumSize(maxEntries)
.expireAfterAccess(1, TimeUnit.HOURS)
.build();
}
public FilerProto.Locations getLocations(String volumeId) {
if (this.cache == null) {
return null;
}
return this.cache.getIfPresent(volumeId);
}
public void setLocations(String volumeId, FilerProto.Locations locations) {
if (this.cache == null) {
return;
}
this.cache.put(volumeId, locations);
}
}