HCFS: streaming list files

This commit is contained in:
Chris Lu 2019-12-13 10:04:53 -08:00
parent bbb6ebc3c0
commit 987108a2b1

View file

@ -7,6 +7,7 @@ import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Iterator;
import java.util.List; import java.util.List;
public class FilerClient { public class FilerClient {
@ -173,17 +174,18 @@ public class FilerClient {
} }
public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) { public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
List<FilerProto.Entry> entries = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder() Iterator<FilerProto.ListEntriesResponse> iter = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
.setDirectory(path) .setDirectory(path)
.setPrefix(entryPrefix) .setPrefix(entryPrefix)
.setStartFromFileName(lastEntryName) .setStartFromFileName(lastEntryName)
.setLimit(limit) .setLimit(limit)
.build()).getEntriesList(); .build());
List<FilerProto.Entry> fixedEntries = new ArrayList<>(entries.size()); List<FilerProto.Entry> entries = new ArrayList<>();
for (FilerProto.Entry entry : entries) { while (iter.hasNext()){
fixedEntries.add(fixEntryAfterReading(entry)); FilerProto.ListEntriesResponse resp = iter.next();
entries.add(fixEntryAfterReading(resp.getEntry()));
} }
return fixedEntries; return entries;
} }
public FilerProto.Entry lookupEntry(String directory, String entryName) { public FilerProto.Entry lookupEntry(String directory, String entryName) {