From d11cfdb356d93e1d0dfba739dee9458a43b67e19 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 17 Dec 2018 01:25:02 -0800 Subject: [PATCH] paginate through large folders --- other/java/client/pom.xml | 2 +- .../java/seaweedfs/client/FilerClient.java | 19 ++++++++++++++++++- other/java/hdfs/pom.xml | 13 +++++++------ .../java/seaweed/hdfs/SeaweedFileSystem.java | 7 ------- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index a3c56856e..808abb98d 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -4,7 +4,7 @@ com.github.chrislusf seaweedfs-client - 1.0.2 + 1.0.3 org.sonatype.oss diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 8414ee303..ff84b053f 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -137,7 +138,23 @@ public class FilerClient { } public List listEntries(String path) { - return listEntries(path, "", "", 100000); + List results = new ArrayList(); + String lastFileName = ""; + for (int limit = Integer.MAX_VALUE; limit > 0; ) { + List t = listEntries(path, "", lastFileName, 1024); + if (t == null) { + break; + } + int nSize = t.size(); + if (nSize > 0) { + limit -= nSize; + lastFileName = t.get(nSize - 1).getName(); + } + if (t.size() < 1024) { + break; + } + } + return results; } public List listEntries(String path, String entryPrefix, String lastEntryName, int limit) { diff --git a/other/java/hdfs/pom.xml b/other/java/hdfs/pom.xml index 53d6746fa..1600bea66 100644 --- a/other/java/hdfs/pom.xml +++ b/other/java/hdfs/pom.xml @@ -4,9 +4,14 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + 1.0.3 + 3.1.1 + + com.github.chrislusf seaweedfs-hadoop-client - 1.0.2 + ${seaweedfs.client.version} org.sonatype.oss @@ -133,10 +138,6 @@ - - 3.1.1 - - org.apache.hadoop @@ -146,7 +147,7 @@ com.github.chrislusf seaweedfs-client - 1.0.2 + ${seaweedfs.client.version} org.apache.hadoop diff --git a/other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 7d992a7a1..2a0ef78af 100644 --- a/other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -310,9 +310,6 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem { getClass().getSimpleName() + " FileSystem implementation"); } - /** - * See {@link FileContext#createSymlink(Path, Path, boolean)}. - */ @Override public void createSymlink(final Path target, final Path link, final boolean createParent) throws AccessControlException, @@ -324,10 +321,6 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem { "Filesystem does not support symlinks!"); } - /** - * See {@link AbstractFileSystem#supportsSymlinks()}. - */ - @Override public boolean supportsSymlinks() { return false; }