diff --git a/Makefile b/Makefile index 59a3c46a2..830e3ca76 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ linux: deps mkdir -p linux GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o linux/$(BINARY) $(SOURCE_DIR) -release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build +release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_arm64_build 5_byte_darwin_build 5_byte_windows_build ##### LINUX BUILDS ##### 5_byte_linux_build: @@ -55,6 +55,14 @@ release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_buil $(call build_large,windows,amd64,.exe) $(call zip_large,windows,amd64,.exe) +5_byte_arm_build: $(sources) + $(call build_large,linux,arm,) + $(call tar_large,linux,arm) + +5_byte_arm64_build: $(sources) + $(call build_large,linux,arm64,) + $(call tar_large,linux,arm64) + linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz build/linux_386.tar.gz: $(sources) diff --git a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java index d8d29ede8..28c2f47fc 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java +++ b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java @@ -6,7 +6,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; public class FileChunkManifest { @@ -51,13 +50,17 @@ public class FileChunkManifest { private static byte[] fetchChunk(final FilerGrpcClient filerGrpcClient, FilerProto.FileChunk chunk) throws IOException { - FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); String vid = "" + chunk.getFid().getVolumeId(); - lookupRequest.addVolumeIds(vid); - FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient - .getBlockingStub().lookupVolume(lookupRequest.build()); - Map vid2Locations = lookupResponse.getLocationsMapMap(); - FilerProto.Locations locations = vid2Locations.get(vid); + FilerProto.Locations locations = filerGrpcClient.vidLocations.get(vid); + if (locations == null) { + FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); + lookupRequest.addVolumeIds(vid); + FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient + .getBlockingStub().lookupVolume(lookupRequest.build()); + locations = lookupResponse.getLocationsMapMap().get(vid); + filerGrpcClient.vidLocations.put(vid, locations); + LOG.debug("fetchChunk vid:{} locations:{}", vid, locations); + } SeaweedRead.ChunkView chunkView = new SeaweedRead.ChunkView( FilerClient.toFileId(chunk.getFid()), // avoid deprecated chunk.getFileId() diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 57b67f6b0..1a719f3c0 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -9,6 +9,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.SSLException; +import java.util.Map; +import java.util.HashMap; import java.util.concurrent.TimeUnit; public class FilerGrpcClient { @@ -24,6 +26,7 @@ public class FilerGrpcClient { } } + public final Map vidLocations = new HashMap<>(); private final ManagedChannel channel; private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub; private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub; diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index f0490540d..05457ed48 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -15,7 +15,7 @@ public class SeaweedRead { private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class); - static ChunkCache chunkCache = new ChunkCache(0); + static ChunkCache chunkCache = new ChunkCache(4); // returns bytesRead public static long read(FilerGrpcClient filerGrpcClient, List visibleIntervals, @@ -62,6 +62,7 @@ public class SeaweedRead { if (chunkData == null) { chunkData = doFetchFullChunkData(chunkView, locations); + chunkCache.setChunk(chunkView.fileId, chunkData); } int len = (int) chunkView.size; @@ -69,8 +70,6 @@ public class SeaweedRead { chunkView.fileId, chunkData.length, chunkView.offset, buffer.length, startOffset, len); System.arraycopy(chunkData, (int) chunkView.offset, buffer, startOffset, len); - chunkCache.setChunk(chunkView.fileId, chunkData); - return len; } @@ -109,6 +108,8 @@ public class SeaweedRead { } } + LOG.debug("doFetchFullChunkData fid:{} chunkData.length:{}", chunkView.fileId, data.length); + return data; } diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index 5f4d888bd..d3cecab75 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -61,9 +61,6 @@ public class SeaweedWrite { String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength, cipherKey); - // cache fileId ~ bytes - SeaweedRead.chunkCache.setChunk(fileId, bytes); - LOG.debug("write file chunk {} size {}", targetUrl, bytesLength); return FilerProto.FileChunk.newBuilder() diff --git a/weed/command/mount.go b/weed/command/mount.go index 440aca8c6..a0e573423 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -9,7 +9,6 @@ type MountOptions struct { filerMountRootPath *string dir *string dirAutoCreate *bool - dirListCacheLimit *int64 collection *string replication *string ttlSec *int @@ -35,7 +34,6 @@ func init() { mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server") mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") mountOptions.dirAutoCreate = cmdMount.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to") - mountOptions.dirListCacheLimit = cmdMount.Flag.Int64("dirListCacheLimit", 1000000, "limit cache size to speed up directory long format listing") mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.") mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds") diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 56df740c4..3975575e9 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -165,7 +165,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { CacheDir: *option.cacheDir, CacheSizeMB: *option.cacheSizeMB, DataCenter: *option.dataCenter, - DirListCacheLimit: *option.dirListCacheLimit, EntryCacheTtl: 3 * time.Second, MountUid: uid, MountGid: gid, diff --git a/weed/command/s3.go b/weed/command/s3.go index 7ebd4fab0..92f13673c 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -151,6 +151,7 @@ func (s3opt *S3Options) startS3Server() bool { _, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{ Filer: *s3opt.filer, + Port: *s3opt.port, FilerGrpcAddress: filerGrpcAddress, Config: *s3opt.config, DomainName: *s3opt.domainName, diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 77d01d463..2214b1ac7 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -3,6 +3,7 @@ package filesys import ( "bytes" "context" + "math" "os" "strings" "time" @@ -277,7 +278,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { dirPath := util.FullPath(dir.FullPath()) meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath) - listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(dir.wfs.option.DirListCacheLimit)) + listedEntries, listErr := dir.wfs.metaCache.ListDirectoryEntries(context.Background(), util.FullPath(dir.FullPath()), "", false, int(math.MaxInt32)) if listErr != nil { glog.Errorf("list meta cache: %v", listErr) return nil, fuse.EIO diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index e41693048..68ad987be 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -34,7 +34,6 @@ type Option struct { CacheDir string CacheSizeMB int64 DataCenter string - DirListCacheLimit int64 EntryCacheTtl time.Duration Umask os.FileMode diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 773094a5f..010958245 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -1,6 +1,7 @@ package s3api import ( + "fmt" "net/http" "github.com/gorilla/mux" @@ -9,6 +10,7 @@ import ( type S3ApiServerOption struct { Filer string + Port int FilerGrpcAddress string Config string DomainName string @@ -37,7 +39,10 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { apiRouter := router.PathPrefix("/").Subrouter() var routers []*mux.Router if s3a.option.DomainName != "" { - routers = append(routers, apiRouter.Host("{bucket:.+}."+s3a.option.DomainName).Subrouter()) + routers = append(routers, apiRouter.Host( + fmt.Sprintf("%s.%s:%d", "{bucket:.+}", s3a.option.DomainName, s3a.option.Port)).Subrouter()) + routers = append(routers, apiRouter.Host( + fmt.Sprintf("%s.%s", "{bucket:.+}", s3a.option.DomainName)).Subrouter()) } routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())