From 816ef0ed302c263939ce900a01624f80dbdf17b9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 18 Feb 2021 10:34:46 -0800 Subject: [PATCH 01/16] add more help message fix https://github.com/chrislusf/seaweedfs/issues/1816 --- weed/shell/shell_liner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go index 2d5166acf..00adb1abc 100644 --- a/weed/shell/shell_liner.go +++ b/weed/shell/shell_liner.go @@ -92,7 +92,7 @@ func processEachCmd(reg *regexp.Regexp, cmd string, commandEnv *CommandEnv) bool func printGenericHelp() { msg := - `Type: "help " for help on + `Type: "help " for help on . Most commands support " -h" also for options. ` fmt.Print(msg) From c2ad6f104757b2d58c9a354400d4ec8bca268c3e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 18 Feb 2021 12:15:09 -0800 Subject: [PATCH 02/16] webdav add replication setting fix https://github.com/chrislusf/seaweedfs/issues/1817 --- weed/command/filer.go | 3 ++- weed/command/server.go | 1 + weed/command/webdav.go | 3 +++ weed/server/webdav_server.go | 5 +++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 7f665cec8..f8fa57584 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -52,7 +52,7 @@ type FilerOptions struct { func init() { cmdFiler.Run = runFiler // break init cycle f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers") - f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection") + f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection") f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address") f.bindIp = cmdFiler.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") @@ -83,6 +83,7 @@ func init() { filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway") filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port") filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files") + filerWebDavOptions.replication = cmdFiler.Flag.String("webdav.replication", "", "replication to create the files") filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file") filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") diff --git a/weed/command/server.go b/weed/command/server.go index 61cac2cc7..f5a07e981 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -121,6 +121,7 @@ func init() { webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port") webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files") + webdavOptions.replication = cmdServer.Flag.String("webdav.replication", "", "replication to create the files") webdavOptions.tlsPrivateKey = cmdServer.Flag.String("webdav.key.file", "", "path to the TLS private key file") webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") diff --git a/weed/command/webdav.go b/weed/command/webdav.go index dc84b1fd0..158ba292f 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -25,6 +25,7 @@ type WebDavOption struct { filer *string port *int collection *string + replication *string tlsPrivateKey *string tlsCertificate *string cacheDir *string @@ -36,6 +37,7 @@ func init() { webDavStandaloneOptions.filer = cmdWebDav.Flag.String("filer", "localhost:8888", "filer server address") webDavStandaloneOptions.port = cmdWebDav.Flag.Int("port", 7333, "webdav server http listen port") webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files") + webDavStandaloneOptions.replication = cmdWebDav.Flag.String("replication", "", "replication to create the files") webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file") webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file") webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks") @@ -107,6 +109,7 @@ func (wo *WebDavOption) startWebDav() bool { FilerGrpcAddress: filerGrpcAddress, GrpcDialOption: grpcDialOption, Collection: *wo.collection, + Replication: *wo.replication, Uid: uid, Gid: gid, Cipher: cipher, diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 4b57c7afe..c3f68fdee 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -33,6 +33,7 @@ type WebDavOption struct { BucketsPath string GrpcDialOption grpc.DialOption Collection string + Replication string DiskType string Uid uint32 Gid uint32 @@ -225,7 +226,7 @@ func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, f Uid: fs.option.Uid, Gid: fs.option.Gid, Collection: fs.option.Collection, - Replication: "000", + Replication: fs.option.Replication, TtlSec: 0, }, }, @@ -381,7 +382,7 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64 request := &filer_pb.AssignVolumeRequest{ Count: 1, - Replication: "", + Replication: f.fs.option.Replication, Collection: f.fs.option.Collection, DiskType: f.fs.option.DiskType, Path: name, From 4de2d6774af43fd290ccc02eb333e661144ea741 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Feb 2021 21:23:26 +0100 Subject: [PATCH 03/16] Add build arg to choose dev or stable release --- docker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 528a52609..6309e7e2e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,7 @@ FROM alpine +ARG RELEASE=latest # 'latest' or 'dev' + RUN \ ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \ elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \ @@ -13,7 +15,7 @@ RUN \ # Install SeaweedFS and Supercronic ( for cron job mode ) apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \ apk add fuse && \ - wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz") && \ + wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz") && \ tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \ curl -fsSLO "$SUPERCRONIC_URL" && \ echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \ From e1992c83deeff50d5ddbd385dccac46e941fafcb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 18 Feb 2021 15:45:12 -0800 Subject: [PATCH 04/16] webdav adds disktype --- weed/command/filer.go | 1 + weed/command/server.go | 1 + weed/command/webdav.go | 3 +++ 3 files changed, 5 insertions(+) diff --git a/weed/command/filer.go b/weed/command/filer.go index f8fa57584..6660bd694 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -84,6 +84,7 @@ func init() { filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port") filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files") filerWebDavOptions.replication = cmdFiler.Flag.String("webdav.replication", "", "replication to create the files") + filerWebDavOptions.disk = cmdFiler.Flag.String("webdav.disk", "", "[hdd|ssd] hard drive or solid state drive") filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file") filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") diff --git a/weed/command/server.go b/weed/command/server.go index f5a07e981..d7c41b014 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -122,6 +122,7 @@ func init() { webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port") webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files") webdavOptions.replication = cmdServer.Flag.String("webdav.replication", "", "replication to create the files") + webdavOptions.disk = cmdServer.Flag.String("webdav.disk", "", "[hdd|ssd] hard drive or solid state drive") webdavOptions.tlsPrivateKey = cmdServer.Flag.String("webdav.key.file", "", "path to the TLS private key file") webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") diff --git a/weed/command/webdav.go b/weed/command/webdav.go index 158ba292f..3e4532d6e 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -26,6 +26,7 @@ type WebDavOption struct { port *int collection *string replication *string + disk *string tlsPrivateKey *string tlsCertificate *string cacheDir *string @@ -38,6 +39,7 @@ func init() { webDavStandaloneOptions.port = cmdWebDav.Flag.Int("port", 7333, "webdav server http listen port") webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files") webDavStandaloneOptions.replication = cmdWebDav.Flag.String("replication", "", "replication to create the files") + webDavStandaloneOptions.disk = cmdWebDav.Flag.String("disk", "", "[hdd|ssd] hard drive or solid state drive") webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file") webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file") webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks") @@ -110,6 +112,7 @@ func (wo *WebDavOption) startWebDav() bool { GrpcDialOption: grpcDialOption, Collection: *wo.collection, Replication: *wo.replication, + DiskType: *wo.disk, Uid: uid, Gid: gid, Cipher: cipher, From 776f49746930ef3eabf8361886161f18ca3e2c8c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 18 Feb 2021 17:07:02 -0800 Subject: [PATCH 05/16] filer: fs.configure should try to read from entry.content also related to https://github.com/chrislusf/seaweedfs/issues/1792 --- weed/filer/filer_conf.go | 16 +++++++++------- weed/filer/filer_on_meta_event.go | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index b4f0e5890..8e549f5ad 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -46,17 +46,19 @@ func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) { return fc.LoadFromBytes(entry.Content) } - return fc.loadFromChunks(filer, entry.Chunks) + return fc.loadFromChunks(filer, entry.Content, entry.Chunks) } -func (fc *FilerConf) loadFromChunks(filer *Filer, chunks []*filer_pb.FileChunk) (err error) { - data, err := filer.readEntry(chunks) - if err != nil { - glog.Errorf("read filer conf content: %v", err) - return +func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk) (err error) { + if len(content) == 0 { + content, err = filer.readEntry(chunks) + if err != nil { + glog.Errorf("read filer conf content: %v", err) + return + } } - return fc.LoadFromBytes(data) + return fc.LoadFromBytes(content) } func (fc *FilerConf) LoadFromBytes(data []byte) (err error) { diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index d93a7e9e3..295a5039e 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -40,7 +40,7 @@ func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) { fc := NewFilerConf() - err := fc.loadFromChunks(f, entry.Chunks) + err := fc.loadFromChunks(f, entry.Content, entry.Chunks) if err != nil { glog.Errorf("read filer conf chunks: %v", err) return From 73958e357dba827cd82e3ed8b277d865aae37a72 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 18 Feb 2021 19:10:20 -0800 Subject: [PATCH 06/16] add descriptive error if no free volumes --- weed/server/master_grpc_server_volume.go | 2 +- weed/server/master_server_handlers.go | 2 +- weed/storage/super_block/replica_placement.go | 6 ++--- weed/topology/volume_growth.go | 22 ++++++++++--------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 1699eaa36..29aff5c0b 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -77,7 +77,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest if !ms.Topo.HasWritableVolume(option) { if ms.Topo.AvailableSpaceFor(option) <= 0 { - return nil, fmt.Errorf("No free volumes left!") + return nil, fmt.Errorf("no free volumes left for "+option.String()) } ms.vgLock.Lock() if !ms.Topo.HasWritableVolume(option) { diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index 117423ec6..a9fecc5bd 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -113,7 +113,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) if !ms.Topo.HasWritableVolume(option) { if ms.Topo.AvailableSpaceFor(option) <= 0 { - writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left!"}) + writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left for " + option.String()}) return } ms.vgLock.Lock() diff --git a/weed/storage/super_block/replica_placement.go b/weed/storage/super_block/replica_placement.go index fcccbba7d..65ec53819 100644 --- a/weed/storage/super_block/replica_placement.go +++ b/weed/storage/super_block/replica_placement.go @@ -6,9 +6,9 @@ import ( ) type ReplicaPlacement struct { - SameRackCount int - DiffRackCount int - DiffDataCenterCount int + SameRackCount int `json:"node,omitempty"` + DiffRackCount int `json:"rack,omitempty"` + DiffDataCenterCount int `json:"dc,omitempty"` } func NewReplicaPlacementFromString(t string) (*ReplicaPlacement, error) { diff --git a/weed/topology/volume_growth.go b/weed/topology/volume_growth.go index 4b0a4837e..8941a049b 100644 --- a/weed/topology/volume_growth.go +++ b/weed/topology/volume_growth.go @@ -1,6 +1,7 @@ package topology import ( + "encoding/json" "fmt" "github.com/chrislusf/seaweedfs/weed/storage/types" "math/rand" @@ -25,15 +26,15 @@ This package is created to resolve these replica placement issues: */ type VolumeGrowOption struct { - Collection string - ReplicaPlacement *super_block.ReplicaPlacement - Ttl *needle.TTL - DiskType types.DiskType - Prealloacte int64 - DataCenter string - Rack string - DataNode string - MemoryMapMaxSizeMb uint32 + Collection string `json:"collection,omitempty"` + ReplicaPlacement *super_block.ReplicaPlacement `json:"replication,omitempty"` + Ttl *needle.TTL `json:"ttl,omitempty"` + DiskType types.DiskType `json:"disk,omitempty"` + Prealloacte int64 `json:"prealloacte,omitempty"` + DataCenter string `json:"dataCenter,omitempty"` + Rack string `json:"rack,omitempty"` + DataNode string `json:"dataNode,omitempty"` + MemoryMapMaxSizeMb uint32 `json:"memoryMapMaxSizeMb,omitempty"` } type VolumeGrowth struct { @@ -41,7 +42,8 @@ type VolumeGrowth struct { } func (o *VolumeGrowOption) String() string { - return fmt.Sprintf("Collection:%s, ReplicaPlacement:%v, Ttl:%v, DataCenter:%s, Rack:%s, DataNode:%s", o.Collection, o.ReplicaPlacement, o.Ttl, o.DataCenter, o.Rack, o.DataNode) + blob, _ := json.Marshal(o) + return string(blob) } func NewDefaultVolumeGrowth() *VolumeGrowth { From 29575dadc54b2e3780073f8b86f110d7cb7c1f20 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 18 Feb 2021 19:14:58 -0800 Subject: [PATCH 07/16] remove extra files --- weed/filer.toml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 weed/filer.toml diff --git a/weed/filer.toml b/weed/filer.toml deleted file mode 100644 index c0530b3f1..000000000 --- a/weed/filer.toml +++ /dev/null @@ -1,11 +0,0 @@ -[elastic7] -enabled = true -servers = [ - "http://localhost:9200", -] -username = "" -password = "" -sniff_enabled = false -healthcheck_enabled = false -# increase the value is recommend, be sure the value in Elastic is greater or equal here -index.max_result_window = 10000 From c78409a5983145784d128c86541c6715dad3a937 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 18 Feb 2021 23:22:54 -0800 Subject: [PATCH 08/16] volume: avoid file.stat(), file.seek() if possible during writes --- weed/storage/backend/disk_file.go | 23 +++++++++++++++++++++-- weed/storage/needle_map.go | 16 ++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/weed/storage/backend/disk_file.go b/weed/storage/backend/disk_file.go index 2b04c8df2..6a42ad8e7 100644 --- a/weed/storage/backend/disk_file.go +++ b/weed/storage/backend/disk_file.go @@ -12,6 +12,8 @@ var ( type DiskFile struct { File *os.File fullFilePath string + fileSize int64 + modTime time.Time } func NewDiskFile(f *os.File) *DiskFile { @@ -26,11 +28,24 @@ func (df *DiskFile) ReadAt(p []byte, off int64) (n int, err error) { } func (df *DiskFile) WriteAt(p []byte, off int64) (n int, err error) { - return df.File.WriteAt(p, off) + n, err = df.File.WriteAt(p, off) + if err == nil { + waterMark := off + int64(n) + if waterMark > df.fileSize { + df.fileSize = waterMark + df.modTime = time.Now() + } + } + return } func (df *DiskFile) Truncate(off int64) error { - return df.File.Truncate(off) + err := df.File.Truncate(off) + if err == nil { + df.fileSize = off + df.modTime = time.Now() + } + return err } func (df *DiskFile) Close() error { @@ -38,6 +53,9 @@ func (df *DiskFile) Close() error { } func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) { + if df.fileSize != 0 { + return df.fileSize, df.modTime, nil + } stat, e := df.File.Stat() if e == nil { return stat.Size(), stat.ModTime(), nil @@ -50,5 +68,6 @@ func (df *DiskFile) Name() string { } func (df *DiskFile) Sync() error { + df.fileSize = 0 return df.File.Sync() } diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go index 5b41286ea..f12198c57 100644 --- a/weed/storage/needle_map.go +++ b/weed/storage/needle_map.go @@ -41,6 +41,7 @@ type baseNeedleMapper struct { indexFile *os.File indexFileAccessLock sync.Mutex + indexFileOffset int64 } func (nm *baseNeedleMapper) IndexFileSize() uint64 { @@ -56,11 +57,18 @@ func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size nm.indexFileAccessLock.Lock() defer nm.indexFileAccessLock.Unlock() - if _, err := nm.indexFile.Seek(0, 2); err != nil { - return fmt.Errorf("cannot seek end of indexfile %s: %v", - nm.indexFile.Name(), err) + if nm.indexFileOffset == 0 { + if fileSize, err := nm.indexFile.Seek(0, 2); err != nil { + return fmt.Errorf("cannot seek end of indexfile %s: %v", + nm.indexFile.Name(), err) + } else { + nm.indexFileOffset = fileSize + } + } + written, err := nm.indexFile.WriteAt(bytes, nm.indexFileOffset) + if err == nil { + nm.indexFileOffset += int64(written) } - _, err := nm.indexFile.Write(bytes) return err } From c576ad04aca940319ec6d41583c93398b68dbe58 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 19 Feb 2021 01:38:56 -0800 Subject: [PATCH 09/16] fix volume server display for volumes --- weed/topology/data_node.go | 21 ++++++++++++++++++++- weed/topology/disk.go | 19 ------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index 1a0ebf761..d4ec94fbe 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -207,7 +207,26 @@ func (dn *DataNode) ToMap() interface{} { ret := make(map[string]interface{}) ret["Url"] = dn.Url() ret["PublicUrl"] = dn.PublicUrl - ret["Disks"] = dn.diskUsages.ToMap() + + // aggregated volume info + var volumeCount, ecShardCount, maxVolumeCount int64 + var volumeIds string + for _, diskUsage := range dn.diskUsages.usages { + volumeCount += diskUsage.volumeCount + ecShardCount += diskUsage.ecShardCount + maxVolumeCount += diskUsage.maxVolumeCount + } + + for _, disk := range dn.Children() { + d := disk.(*Disk) + volumeIds += d.GetVolumeIds() + } + + ret["Volumes"] = volumeCount + ret["EcShards"] = ecShardCount + ret["Max"] = maxVolumeCount + ret["volumeIds"] = volumeIds + return ret } diff --git a/weed/topology/disk.go b/weed/topology/disk.go index 37d5e1272..a085f8dff 100644 --- a/weed/topology/disk.go +++ b/weed/topology/disk.go @@ -58,16 +58,6 @@ func (d *DiskUsages) negative() *DiskUsages { return t } -func (d *DiskUsages) ToMap() interface{} { - d.RLock() - defer d.RUnlock() - ret := make(map[string]interface{}) - for diskType, diskUsage := range d.usages { - ret[diskType.String()] = diskUsage.ToMap() - } - return ret -} - func (d *DiskUsages) ToDiskInfo() map[string]*master_pb.DiskInfo { ret := make(map[string]*master_pb.DiskInfo) for diskType, diskUsageCounts := range d.usages { @@ -135,15 +125,6 @@ func (a *DiskUsageCounts) minus(b *DiskUsageCounts) *DiskUsageCounts { } } -func (diskUsage *DiskUsageCounts) ToMap() interface{} { - ret := make(map[string]interface{}) - ret["Volumes"] = diskUsage.volumeCount - ret["EcShards"] = diskUsage.ecShardCount - ret["Max"] = diskUsage.maxVolumeCount - ret["Free"] = diskUsage.FreeSpace() - return ret -} - func (du *DiskUsages) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts { du.Lock() defer du.Unlock() From 60d7b809c14e2f5b903a704fbc2c9ed16948248e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 19 Feb 2021 02:40:30 -0800 Subject: [PATCH 10/16] add back actions --- .github/workflows/cleanup.yml | 22 ++++++++++ .github/workflows/release.yml | 76 +++++++++++++++++------------------ 2 files changed, 58 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/cleanup.yml diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 000000000..47a677e6d --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,22 @@ +name: Cleanup + +on: + push: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + + steps: + + - name: Delete old release assets + uses: mknejp/delete-release-assets@v1 + with: + token: ${{ github.token }} + tag: dev + fail-if-no-assets: false + assets: | + weed-* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7439fb604..0681f3dac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: matrix: goos: [linux, windows, darwin, freebsd ] goarch: [amd64, arm] - exclude: + exclude: - goarch: arm goos: darwin - goarch: arm @@ -21,46 +21,42 @@ jobs: steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v2 + - name: Check out code into the Go module directory + uses: actions/checkout@v2 - - name: Delete old release assets - uses: mknejp/delete-release-assets@v1 - with: - token: ${{ github.token }} - tag: dev - fail-if-no-assets: false - assets: | - weed-* + - name: Wait for the deletion + uses: jakejarvis/wait-action@master + with: + time: '30s' - - name: Set BUILD_TIME env - run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV} + - name: Set BUILD_TIME env + run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV} - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.14 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - release_tag: dev - overwrite: true - build_flags: -tags 5BytesOffset # optional, default is - ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} - # Where to run `go build .` - project_path: weed - binary_name: weed-large-disk - asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + - name: Go Release Binaries + uses: wangyoucao577/go-release-action@v1.14 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-large-disk + asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.14 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - release_tag: dev - overwrite: true - ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} - # Where to run `go build .` - project_path: weed - binary_name: weed- - asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" + - name: Go Release Binaries + uses: wangyoucao577/go-release-action@v1.14 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + release_tag: dev + overwrite: true + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed- + asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" From f465d63b5d3577052243c5c24668d0f40930a923 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 19 Feb 2021 03:37:29 -0800 Subject: [PATCH 11/16] grpc should fail when heart beating to master fix https://github.com/chrislusf/seaweedfs/issues/1820 --- weed/pb/grpc_client_server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index ec49d177b..910114313 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -64,7 +64,6 @@ func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*gr grpc.WithDefaultCallOptions( grpc.MaxCallSendMsgSize(Max_Message_Size), grpc.MaxCallRecvMsgSize(Max_Message_Size), - grpc.WaitForReady(true), ), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 30 * time.Second, // client ping server if no activity for this long From b961cd6208104641c5d7ae41bff3498a48ee3279 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 19 Feb 2021 03:39:19 -0800 Subject: [PATCH 12/16] add WIP message --- weed/shell/command_volume_tier_move.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/shell/command_volume_tier_move.go b/weed/shell/command_volume_tier_move.go index 3c6c31eac..27c6815d4 100644 --- a/weed/shell/command_volume_tier_move.go +++ b/weed/shell/command_volume_tier_move.go @@ -20,11 +20,11 @@ type commandVolumeTierMove struct { } func (c *commandVolumeTierMove) Name() string { - return "volume.tier.upload" + return "volume.tier.move" } func (c *commandVolumeTierMove) Help() string { - return `change a volume from one disk type to another + return ` change a volume from one disk type to another volume.tier.move -source=hdd -target=ssd [-collection=""] [-fullPercent=95] [-quietFor=1h] volume.tier.move -target=hdd [-collection=""] -volumeId= From a1210d1e8d923aa2415a2f3aa91643b7e1686091 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 19 Feb 2021 03:56:27 -0800 Subject: [PATCH 13/16] Revert "volume: avoid file.stat(), file.seek() if possible during writes" This reverts commit c78409a5983145784d128c86541c6715dad3a937. --- weed/storage/backend/disk_file.go | 23 ++--------------------- weed/storage/needle_map.go | 16 ++++------------ 2 files changed, 6 insertions(+), 33 deletions(-) diff --git a/weed/storage/backend/disk_file.go b/weed/storage/backend/disk_file.go index 6a42ad8e7..2b04c8df2 100644 --- a/weed/storage/backend/disk_file.go +++ b/weed/storage/backend/disk_file.go @@ -12,8 +12,6 @@ var ( type DiskFile struct { File *os.File fullFilePath string - fileSize int64 - modTime time.Time } func NewDiskFile(f *os.File) *DiskFile { @@ -28,24 +26,11 @@ func (df *DiskFile) ReadAt(p []byte, off int64) (n int, err error) { } func (df *DiskFile) WriteAt(p []byte, off int64) (n int, err error) { - n, err = df.File.WriteAt(p, off) - if err == nil { - waterMark := off + int64(n) - if waterMark > df.fileSize { - df.fileSize = waterMark - df.modTime = time.Now() - } - } - return + return df.File.WriteAt(p, off) } func (df *DiskFile) Truncate(off int64) error { - err := df.File.Truncate(off) - if err == nil { - df.fileSize = off - df.modTime = time.Now() - } - return err + return df.File.Truncate(off) } func (df *DiskFile) Close() error { @@ -53,9 +38,6 @@ func (df *DiskFile) Close() error { } func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) { - if df.fileSize != 0 { - return df.fileSize, df.modTime, nil - } stat, e := df.File.Stat() if e == nil { return stat.Size(), stat.ModTime(), nil @@ -68,6 +50,5 @@ func (df *DiskFile) Name() string { } func (df *DiskFile) Sync() error { - df.fileSize = 0 return df.File.Sync() } diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go index f12198c57..5b41286ea 100644 --- a/weed/storage/needle_map.go +++ b/weed/storage/needle_map.go @@ -41,7 +41,6 @@ type baseNeedleMapper struct { indexFile *os.File indexFileAccessLock sync.Mutex - indexFileOffset int64 } func (nm *baseNeedleMapper) IndexFileSize() uint64 { @@ -57,18 +56,11 @@ func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size nm.indexFileAccessLock.Lock() defer nm.indexFileAccessLock.Unlock() - if nm.indexFileOffset == 0 { - if fileSize, err := nm.indexFile.Seek(0, 2); err != nil { - return fmt.Errorf("cannot seek end of indexfile %s: %v", - nm.indexFile.Name(), err) - } else { - nm.indexFileOffset = fileSize - } - } - written, err := nm.indexFile.WriteAt(bytes, nm.indexFileOffset) - if err == nil { - nm.indexFileOffset += int64(written) + if _, err := nm.indexFile.Seek(0, 2); err != nil { + return fmt.Errorf("cannot seek end of indexfile %s: %v", + nm.indexFile.Name(), err) } + _, err := nm.indexFile.Write(bytes) return err } From 6593a0f3d19659615754677c78430100c48f16ab Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 19 Feb 2021 14:08:29 -0800 Subject: [PATCH 14/16] adjust --- docker/compose/local-mount-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/compose/local-mount-compose.yml b/docker/compose/local-mount-compose.yml index 4361d6d99..b1c579cdf 100644 --- a/docker/compose/local-mount-compose.yml +++ b/docker/compose/local-mount-compose.yml @@ -30,7 +30,7 @@ services: mount_1: image: chrislusf/seaweedfs:local privileged: true - entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1" -volumeServerAccess=filerProxy' + entrypoint: '/bin/sh -c "mkdir -p t1 && mkdir -p cache/t1 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t1 -dir=./t1 -filer.path=/c1 -volumeServerAccess=filerProxy"' depends_on: - master - volume @@ -38,7 +38,7 @@ services: mount_2: image: chrislusf/seaweedfs:local privileged: true - entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1" -volumeServerAcess=publicUrl' + entrypoint: '/bin/sh -c "mkdir -p t2 && mkdir -p cache/t2 && weed -v=4 mount -filer=filer:8888 -cacheDir=./cache/t2 -dir=./t2 -filer.path=/c1 -volumeServerAcess=publicUrl"' depends_on: - master - volume From a37473ae6096d73d6e4fbaaf7dc92422dd6f786c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 19 Feb 2021 14:22:12 -0800 Subject: [PATCH 15/16] add back volume ids address https://github.com/chrislusf/seaweedfs/issues/1792#issuecomment-782339576 --- weed/topology/data_node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index d4ec94fbe..c249a9842 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -225,7 +225,7 @@ func (dn *DataNode) ToMap() interface{} { ret["Volumes"] = volumeCount ret["EcShards"] = ecShardCount ret["Max"] = maxVolumeCount - ret["volumeIds"] = volumeIds + ret["VolumeIds"] = volumeIds return ret } From 565f7a6e724cf2dca401bd699a65f5814606ebfe Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 19 Feb 2021 14:22:36 -0800 Subject: [PATCH 16/16] Update data_node.go --- weed/topology/data_node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index c249a9842..69f739dd5 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -219,7 +219,7 @@ func (dn *DataNode) ToMap() interface{} { for _, disk := range dn.Children() { d := disk.(*Disk) - volumeIds += d.GetVolumeIds() + volumeIds += " " + d.GetVolumeIds() } ret["Volumes"] = volumeCount