From a566bfc6e1b25c262c3eb20cd9a8a2ef7a5f5392 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 28 Jul 2020 08:47:24 -0700 Subject: [PATCH 1/8] s3: use bucket in the domain fix https://github.com/chrislusf/seaweedfs/issues/1405 --- weed/command/s3.go | 1 + weed/s3api/s3api_server.go | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/weed/command/s3.go b/weed/command/s3.go index 7ebd4fab0..92f13673c 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -151,6 +151,7 @@ func (s3opt *S3Options) startS3Server() bool { _, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{ Filer: *s3opt.filer, + Port: *s3opt.port, FilerGrpcAddress: filerGrpcAddress, Config: *s3opt.config, DomainName: *s3opt.domainName, diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 773094a5f..010958245 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1,6 +1,7 @@ package s3api import ( + "fmt" "net/http" "github.com/gorilla/mux" @@ -9,6 +10,7 @@ import ( type S3ApiServerOption struct { Filer string + Port int FilerGrpcAddress string Config string DomainName string @@ -37,7 +39,10 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { apiRouter := router.PathPrefix("/").Subrouter() var routers []*mux.Router if s3a.option.DomainName != "" { - routers = append(routers, apiRouter.Host("{bucket:.+}."+s3a.option.DomainName).Subrouter()) + routers = append(routers, apiRouter.Host( + fmt.Sprintf("%s.%s:%d", "{bucket:.+}", s3a.option.DomainName, s3a.option.Port)).Subrouter()) + routers = append(routers, apiRouter.Host( + fmt.Sprintf("%s.%s", "{bucket:.+}", s3a.option.DomainName)).Subrouter()) } routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) From eed525b71740f9aa4bffc8cab63f75fcaa8ae6b6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 28 Jul 2020 09:24:39 -0700 Subject: [PATCH 2/8] FUSE mount: remove DirListCacheLimit outdated parameter --- weed/command/mount.go | 2 -- weed/command/mount_std.go | 1 - weed/filesys/dir.go | 3 ++- weed/filesys/wfs.go | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/weed/command/mount.go b/weed/command/mount.go index 440aca8c6..a0e573423 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -9,7 +9,6 @@ type MountOptions struct { filerMountRootPath *string dir *string dirAutoCreate *bool - dirListCacheLimit *int64 collection *string replication *string ttlSec *int @@ -35,7 +34,6 @@ func init() { mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server") mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to") - mountOptions.dirListCacheLimit = cmdMount.Flag.Int64("dirListCacheLimit", 1000000, "limit cache size to speed up directory long format listing") mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.") mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds") diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 56df740c4..3975575e9 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -165,7 +165,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { CacheDir: *option.cacheDir, CacheSizeMB: *option.cacheSizeMB, DataCenter: *option.dataCenter, - DirListCacheLimit: *option.dirListCacheLimit, EntryCacheTtl: 3 * time.Second, MountUid: uid, MountGid: gid, diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 77d01d463..2214b1ac7 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -3,6 +3,7 @@ package filesys import ( "bytes" "context" + "math" "os" "strings" "time" @@ -277,7 +278,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { dirPath := util.FullPath(dir.FullPath()) meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath) - listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(dir.wfs.option.DirListCacheLimit)) + listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(math.MaxInt32)) if listErr != nil { glog.Errorf("list meta cache: %v", listErr) return nil, fuse.EIO diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index e41693048..68ad987be 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -34,7 +34,6 @@ type Option struct { CacheDir string CacheSizeMB int64 DataCenter string - DirListCacheLimit int64 EntryCacheTtl time.Duration Umask os.FileMode From 9f6ecfbd42d4f01efd4385e3d8f126b7ae81d2cf Mon Sep 17 00:00:00 2001 From: mwpeterson Date: Tue, 28 Jul 2020 15:57:49 -0400 Subject: [PATCH 3/8] add 5-byte large disk builds for arm and arm64 --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 59a3c46a2..b165f42b9 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,14 @@ release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_buil $(call build_large,windows,amd64,.exe) $(call zip_large,windows,amd64,.exe) +5_byte_arm_build: $(sources) + $(call build_large,linux,arm,) + $(call tar_large,linux,arm) + +5_byte_arm64_build: $(sources) + $(call build_large,linux,arm64,) + $(call tar_large,linux,arm64) + linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz build/linux_386.tar.gz: $(sources) From 80fb139a108b0a0cd6a477ef566b44d3e4ea8040 Mon Sep 17 00:00:00 2001 From: mwpeterson Date: Wed, 29 Jul 2020 14:07:04 -0400 Subject: [PATCH 4/8] add 5_byte_arm63 to release --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b165f42b9..830e3ca76 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ linux: deps mkdir -p linux GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o linux/$(BINARY) $(SOURCE_DIR) -release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build +release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_arm64_build 5_byte_darwin_build 5_byte_windows_build ##### LINUX BUILDS ##### 5_byte_linux_build: From 735912e0f5de0d1d287fa6683504624a52e842e9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 17:39:18 -0700 Subject: [PATCH 5/8] enable read cache --- .../client/src/main/java/seaweedfs/client/SeaweedRead.java | 2 +- .../client/src/main/java/seaweedfs/client/SeaweedWrite.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index f0490540d..1d8ded426 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -15,7 +15,7 @@ public class SeaweedRead { private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class); - static ChunkCache chunkCache = new ChunkCache(0); + static ChunkCache chunkCache = new ChunkCache(4); // returns bytesRead public static long read(FilerGrpcClient filerGrpcClient, List visibleIntervals, diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index 5f4d888bd..d3cecab75 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -61,9 +61,6 @@ public class SeaweedWrite { String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength, cipherKey); - // cache fileId ~ bytes - SeaweedRead.chunkCache.setChunk(fileId, bytes); - LOG.debug("write file chunk {} size {}", targetUrl, bytesLength); return FilerProto.FileChunk.newBuilder() From b684c312d232f4b8ffa3bb276033fd69c469d354 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 18:25:35 -0700 Subject: [PATCH 6/8] minor --- .../client/src/main/java/seaweedfs/client/SeaweedRead.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index 1d8ded426..af3aac0f5 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -62,6 +62,7 @@ public class SeaweedRead { if (chunkData == null) { chunkData = doFetchFullChunkData(chunkView, locations); + chunkCache.setChunk(chunkView.fileId, chunkData); } int len = (int) chunkView.size; @@ -69,8 +70,6 @@ public class SeaweedRead { chunkView.fileId, chunkData.length, chunkView.offset, buffer.length, startOffset, len); System.arraycopy(chunkData, (int) chunkView.offset, buffer, startOffset, len); - chunkCache.setChunk(chunkView.fileId, chunkData); - return len; } From daf8c0c8ce4c507264a175912d4693415f40f606 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 18:25:52 -0700 Subject: [PATCH 7/8] cache vid locations --- .../seaweedfs/client/FileChunkManifest.java | 17 ++++++++++------- .../java/seaweedfs/client/FilerGrpcClient.java | 3 +++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java index d8d29ede8..a15671f46 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java +++ b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java @@ -6,7 +6,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; public class FileChunkManifest { @@ -51,13 +50,17 @@ public class FileChunkManifest { private static byte[] fetchChunk(final FilerGrpcClient filerGrpcClient, FilerProto.FileChunk chunk) throws IOException { - FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); String vid = "" + chunk.getFid().getVolumeId(); - lookupRequest.addVolumeIds(vid); - FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient - .getBlockingStub().lookupVolume(lookupRequest.build()); - Map vid2Locations = lookupResponse.getLocationsMapMap(); - FilerProto.Locations locations = vid2Locations.get(vid); + FilerProto.Locations locations = filerGrpcClient.vidLocations.get(vid); + if (locations == null) { + FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); + lookupRequest.addVolumeIds(vid); + FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient + .getBlockingStub().lookupVolume(lookupRequest.build()); + locations = lookupResponse.getLocationsMapMap().get(vid); + filerGrpcClient.vidLocations.put(vid, locations); + LOG.warn("fetchChunk vid:{} locations:{}", vid, locations); + } SeaweedRead.ChunkView chunkView = new SeaweedRead.ChunkView( FilerClient.toFileId(chunk.getFid()), // avoid deprecated chunk.getFileId() diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 57b67f6b0..1a719f3c0 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -9,6 +9,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.SSLException; +import java.util.Map; +import java.util.HashMap; import java.util.concurrent.TimeUnit; public class FilerGrpcClient { @@ -24,6 +26,7 @@ public class FilerGrpcClient { } } + public final Map vidLocations = new HashMap<>(); private final ManagedChannel channel; private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub; private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub; From 5080bc1d6964cc71044333edc7ee36c1c1f06adb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 18:26:26 -0700 Subject: [PATCH 8/8] debug --- .../src/main/java/seaweedfs/client/FileChunkManifest.java | 2 +- .../java/client/src/main/java/seaweedfs/client/SeaweedRead.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java index a15671f46..28c2f47fc 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java +++ b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java @@ -59,7 +59,7 @@ public class FileChunkManifest { .getBlockingStub().lookupVolume(lookupRequest.build()); locations = lookupResponse.getLocationsMapMap().get(vid); filerGrpcClient.vidLocations.put(vid, locations); - LOG.warn("fetchChunk vid:{} locations:{}", vid, locations); + LOG.debug("fetchChunk vid:{} locations:{}", vid, locations); } SeaweedRead.ChunkView chunkView = new SeaweedRead.ChunkView( diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index af3aac0f5..05457ed48 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -108,6 +108,8 @@ public class SeaweedRead { } } + LOG.debug("doFetchFullChunkData fid:{} chunkData.length:{}", chunkView.fileId, data.length); + return data; }