From 19026ae55d08ef971be1cfb6b66b75069763c6ae Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Tue, 27 Oct 2020 16:49:31 +0800 Subject: [PATCH 01/84] return x-amz-tag-count header when GET object. --- weed/s3api/filer_util_tags.go | 2 +- weed/server/filer_server_handlers_read.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/weed/s3api/filer_util_tags.go b/weed/s3api/filer_util_tags.go index 3d4da7825..fde8e243e 100644 --- a/weed/s3api/filer_util_tags.go +++ b/weed/s3api/filer_util_tags.go @@ -7,7 +7,7 @@ import ( ) const ( - S3TAG_PREFIX = "s3-" + S3TAG_PREFIX = "s3-tag-" ) func (s3a *S3ApiServer) getTags(parentDirectoryPath string, entryName string) (tags map[string]string, err error) { diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index fbd45d6b9..ca584f9f6 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -93,6 +93,19 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } } + //set tag count + if r.Method == "GET" { + tagCount := 0 + for k, _ := range entry.Extended { + if strings.HasPrefix(k, "s3-tag-") { + tagCount++ + } + } + if tagCount > 0 { + w.Header().Set("x-amz-tag-count", strconv.Itoa(tagCount)) + } + } + // set etag etag := filer.ETagEntry(entry) if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" { From 8766ca1b955a0b142d30c0df55fe18b8d8024d7d Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Tue, 27 Oct 2020 17:33:24 +0800 Subject: [PATCH 02/84] rename s3 tag prefix. --- weed/s3api/filer_util_tags.go | 2 +- weed/server/filer_server_handlers_read.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/s3api/filer_util_tags.go b/weed/s3api/filer_util_tags.go index fde8e243e..bdc8f31da 100644 --- a/weed/s3api/filer_util_tags.go +++ b/weed/s3api/filer_util_tags.go @@ -7,7 +7,7 @@ import ( ) const ( - S3TAG_PREFIX = "s3-tag-" + S3TAG_PREFIX = "x-amz-tagging-" ) func (s3a *S3ApiServer) getTags(parentDirectoryPath string, entryName string) (tags map[string]string, err error) { diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index ca584f9f6..1fb24369b 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -97,7 +97,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, if r.Method == "GET" { tagCount := 0 for k, _ := range entry.Extended { - if strings.HasPrefix(k, "s3-tag-") { + if strings.HasPrefix(k, "x-amz-tagging") { tagCount++ } } From 72f6b77deae239cc324ce30a31bc672fcd419ca2 Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Tue, 27 Oct 2020 18:01:37 +0800 Subject: [PATCH 03/84] fix tag prefix. --- weed/server/filer_server_handlers_read.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 1fb24369b..731bd3545 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -97,7 +97,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, if r.Method == "GET" { tagCount := 0 for k, _ := range entry.Extended { - if strings.HasPrefix(k, "x-amz-tagging") { + if strings.HasPrefix(k, "x-amz-tagging-") { tagCount++ } } From 06c15ab35cd67cad133c08ed47d7838d745a3899 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 27 Oct 2020 13:11:56 -0700 Subject: [PATCH 04/84] volume: add special handling for .dat larger than 32GB --- weed/storage/needle/needle_read_write.go | 7 +++++++ weed/storage/volume_read_write.go | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/weed/storage/needle/needle_read_write.go b/weed/storage/needle/needle_read_write.go index 89fc85b0d..e758a6fee 100644 --- a/weed/storage/needle/needle_read_write.go +++ b/weed/storage/needle/needle_read_write.go @@ -24,6 +24,8 @@ const ( TtlBytesLength = 2 ) +var ErrorSizeMismatch = errors.New("size mismatch") + func (n *Needle) DiskSize(version Version) int64 { return GetActualSize(n.Size, version) } @@ -168,6 +170,11 @@ func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, versi func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Version) (err error) { n.ParseNeedleHeader(bytes) if n.Size != size { + // cookie is not always passed in for this API. Use size to do preliminary checking. + if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) { + glog.Errorf("entry not found1: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) + return ErrorSizeMismatch + } return fmt.Errorf("entry not found: offset %d found id %x size %d, expected size %d", offset, n.Id, n.Size, size) } switch version { diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go index 94c1d0ea1..b4ffc8319 100644 --- a/weed/storage/volume_read_write.go +++ b/weed/storage/volume_read_write.go @@ -17,6 +17,7 @@ import ( var ErrorNotFound = errors.New("not found") var ErrorDeleted = errors.New("already deleted") +var ErrorSizeMismatch = errors.New("size mismatch") // isFileUnchanged checks whether this needle to write is same as last one. // It requires serialized access in the same volume. @@ -274,6 +275,9 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption) (int, erro return 0, nil } err := n.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset(), readSize, v.Version()) + if err == needle.ErrorSizeMismatch && OffsetSize == 4 { + err = n.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) + } if err != nil { return 0, err } From 24bf142596f0303474b48378dbf72ae193ea1f4d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 27 Oct 2020 15:31:13 -0700 Subject: [PATCH 05/84] copy large file first --- weed/server/volume_grpc_copy.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/weed/server/volume_grpc_copy.go b/weed/server/volume_grpc_copy.go index cd2b53c8a..115f57422 100644 --- a/weed/server/volume_grpc_copy.go +++ b/weed/server/volume_grpc_copy.go @@ -61,12 +61,11 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo volumeFileName = storage.VolumeFileName(location.Directory, volFileInfoResp.Collection, int(req.VolumeId)) // println("source:", volFileInfoResp.String()) - // copy ecx file - if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, volumeFileName, ".idx", false, false); err != nil { + if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".dat", false, true); err != nil { return err } - if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".dat", false, true); err != nil { + if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, volumeFileName, ".idx", false, false); err != nil { return err } From 6da87720eb24ecebf4c483ab57c5c7f9811321af Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 27 Oct 2020 15:33:26 -0700 Subject: [PATCH 06/84] shorter tag prefix --- weed/s3api/filer_util_tags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/s3api/filer_util_tags.go b/weed/s3api/filer_util_tags.go index bdc8f31da..3d4da7825 100644 --- a/weed/s3api/filer_util_tags.go +++ b/weed/s3api/filer_util_tags.go @@ -7,7 +7,7 @@ import ( ) const ( - S3TAG_PREFIX = "x-amz-tagging-" + S3TAG_PREFIX = "s3-" ) func (s3a *S3ApiServer) getTags(parentDirectoryPath string, entryName string) (tags map[string]string, err error) { From 53c3aad87528d57343afc5fdb3fb5107544af0fc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 27 Oct 2020 15:56:49 -0700 Subject: [PATCH 07/84] volume: add a note file to avoid incomplete volume files fix https://github.com/chrislusf/seaweedfs/issues/1567 --- weed/server/volume_grpc_copy.go | 5 +++++ weed/storage/disk_location.go | 8 ++++++++ weed/storage/volume_read_write.go | 19 ++++++++++++------- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/weed/server/volume_grpc_copy.go b/weed/server/volume_grpc_copy.go index 115f57422..17372eef4 100644 --- a/weed/server/volume_grpc_copy.go +++ b/weed/server/volume_grpc_copy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "io/ioutil" "math" "os" "time" @@ -60,6 +61,8 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo volumeFileName = storage.VolumeFileName(location.Directory, volFileInfoResp.Collection, int(req.VolumeId)) + ioutil.WriteFile(volumeFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755) + // println("source:", volFileInfoResp.String()) if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".dat", false, true); err != nil { return err @@ -73,6 +76,8 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo return err } + os.Remove(volumeFileName+".note") + return nil }) diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index ed57aa54b..5dec21c32 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -13,6 +13,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/util" ) type DiskLocation struct { @@ -60,6 +61,13 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool { name := fileInfo.Name() if !fileInfo.IsDir() && strings.HasSuffix(name, ".idx") { + noteFile := l.Directory + "/" + name + ".note" + if util.FileExists(noteFile) { + note, _ := ioutil.ReadFile(noteFile) + glog.Warningf("volume %s was not completed: %s", name, string(note)) + removeVolumeFiles(l.Directory + "/" + name) + return false + } vid, collection, err := l.volumeIdFromPath(fileInfo) if err != nil { glog.Warningf("get volume id failed, %s, err : %s", name, err) diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go index b4ffc8319..9abc2aed4 100644 --- a/weed/storage/volume_read_write.go +++ b/weed/storage/volume_read_write.go @@ -56,16 +56,21 @@ func (v *Volume) Destroy() (err error) { } } v.Close() - os.Remove(v.FileName() + ".dat") - os.Remove(v.FileName() + ".idx") - os.Remove(v.FileName() + ".vif") - os.Remove(v.FileName() + ".sdx") - os.Remove(v.FileName() + ".cpd") - os.Remove(v.FileName() + ".cpx") - os.RemoveAll(v.FileName() + ".ldb") + removeVolumeFiles(v.FileName()) return } +func removeVolumeFiles(filename string) { + os.Remove(filename+ ".dat") + os.Remove(filename + ".idx") + os.Remove(filename + ".vif") + os.Remove(filename + ".sdx") + os.Remove(filename + ".cpd") + os.Remove(filename + ".cpx") + os.RemoveAll(filename + ".ldb") + os.Remove(filename + ".note") +} + func (v *Volume) asyncRequestAppend(request *needle.AsyncRequest) { v.asyncRequestsChan <- request } From a67eb1afa1dd78f62effcd2cba4d9f8c17895ad8 Mon Sep 17 00:00:00 2001 From: shibinbin Date: Wed, 28 Oct 2020 12:03:06 +0800 Subject: [PATCH 08/84] fix: restart volumeserver sometime occurred out of memory error --- weed/storage/volume_checking.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index e42fb238b..39fd2e3c1 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -61,6 +61,18 @@ func readIndexEntryAtOffset(indexFile *os.File, offset int64) (bytes []byte, err func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, offset int64, key NeedleId, size Size) (lastAppendAtNs uint64, err error) { n := new(needle.Needle) + // case: node total memory 8g, set volumeLimitSize=2048 , save 10 files, every file size 2.2g or more , when we restart the volume server , while see out of memory error + // fix: When the size of the last file exceeds 10M, consider directly returning the last modify time + if size > 10 * 1024 * 1024 { + bytes , err := needle.ReadNeedleBlob(datFile, offset+int64(size), 0, v); + if err == nil { + if v == needle.Version3 { + tsOffset := NeedleHeaderSize + 0 + needle.NeedleChecksumSize + n.AppendAtNs = util.BytesToUint64(bytes[tsOffset : tsOffset+TimestampSize]) + } + } + return n.AppendAtNs, err + } if err = n.ReadData(datFile, offset, size, v); err != nil { return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", offset, offset+int64(size), err) } From 3e925faddd66c1287da843fb1878e7f7f4d31a71 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 28 Oct 2020 01:14:05 -0700 Subject: [PATCH 09/84] trim out name extension --- weed/storage/disk_location.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 5dec21c32..1f58c9977 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -61,6 +61,7 @@ func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeI func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool { name := fileInfo.Name() if !fileInfo.IsDir() && strings.HasSuffix(name, ".idx") { + name := name[:len(name)-len(".idx")] noteFile := l.Directory + "/" + name + ".note" if util.FileExists(noteFile) { note, _ := ioutil.ReadFile(noteFile) @@ -93,7 +94,7 @@ func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind Ne size, _, _ := v.FileStat() glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s", - l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), size, v.Ttl.String()) + l.Directory+"/"+name+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String()) return true } return false From 744c5594da46e5f01b75545eaa5f6de105989643 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 28 Oct 2020 01:14:39 -0700 Subject: [PATCH 10/84] volume: automatically trim out unreachable entries --- weed/storage/volume_checking.go | 73 +++++++++++++++++++++++---------- weed/storage/volume_loading.go | 2 +- 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 39fd2e3c1..78c693f5c 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/storage/idx" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -11,17 +12,28 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) -func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uint64, e error) { +func CheckAndFixVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uint64, err error) { var indexSize int64 - if indexSize, e = verifyIndexFileIntegrity(indexFile); e != nil { - return 0, fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", indexFile.Name(), e) + if indexSize, err = verifyIndexFileIntegrity(indexFile); err != nil { + return 0, fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", indexFile.Name(), err) } if indexSize == 0 { return 0, nil } + for i := 1; i <= 10; i++ { + // check and fix last 10 entries + lastAppendAtNs, err = doCheckAndFixVolumeData(v, indexFile, indexSize-int64(i)*NeedleMapEntrySize) + if err != ErrorSizeMismatch { + break + } + } + return +} + +func doCheckAndFixVolumeData(v *Volume, indexFile *os.File, indexOffset int64) (lastAppendAtNs uint64, err error) { var lastIdxEntry []byte - if lastIdxEntry, e = readIndexEntryAtOffset(indexFile, indexSize-NeedleMapEntrySize); e != nil { - return 0, fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e) + if lastIdxEntry, err = readIndexEntryAtOffset(indexFile, indexOffset); err != nil { + return 0, fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), err) } key, offset, size := idx.IdxFileEntry(lastIdxEntry) if offset.IsZero() { @@ -29,15 +41,15 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uin } if size < 0 { // read the deletion entry - if lastAppendAtNs, e = verifyDeletedNeedleIntegrity(v.DataBackend, v.Version(), key); e != nil { - return lastAppendAtNs, fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), e) + if lastAppendAtNs, err = verifyDeletedNeedleIntegrity(v.DataBackend, v.Version(), key); err != nil { + return lastAppendAtNs, fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), err) } } else { - if lastAppendAtNs, e = verifyNeedleIntegrity(v.DataBackend, v.Version(), offset.ToAcutalOffset(), key, size); e != nil { - return lastAppendAtNs, fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), e) + if lastAppendAtNs, err = verifyNeedleIntegrity(v.DataBackend, v.Version(), offset.ToAcutalOffset(), key, size); err != nil { + return lastAppendAtNs, err } } - return + return lastAppendAtNs, nil } func verifyIndexFileIntegrity(indexFile *os.File) (indexSize int64, err error) { @@ -60,18 +72,37 @@ func readIndexEntryAtOffset(indexFile *os.File, offset int64) (bytes []byte, err } func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, offset int64, key NeedleId, size Size) (lastAppendAtNs uint64, err error) { - n := new(needle.Needle) - // case: node total memory 8g, set volumeLimitSize=2048 , save 10 files, every file size 2.2g or more , when we restart the volume server , while see out of memory error - // fix: When the size of the last file exceeds 10M, consider directly returning the last modify time - if size > 10 * 1024 * 1024 { - bytes , err := needle.ReadNeedleBlob(datFile, offset+int64(size), 0, v); - if err == nil { - if v == needle.Version3 { - tsOffset := NeedleHeaderSize + 0 + needle.NeedleChecksumSize - n.AppendAtNs = util.BytesToUint64(bytes[tsOffset : tsOffset+TimestampSize]) - } + n, _, _, err := needle.ReadNeedleHeader(datFile, v, offset) + if err != nil { + return 0, fmt.Errorf("read %s at %d", datFile.Name(), offset) + } + if n.Size != size { + return 0, ErrorSizeMismatch + } + if v == needle.Version3 { + bytes := make([]byte, TimestampSize) + _, err = datFile.ReadAt(bytes, offset+NeedleHeaderSize+int64(size)+needle.NeedleChecksumSize) + if err != nil { + return 0, fmt.Errorf("check %s entry offset %d size %d: %v", datFile.Name(), offset, size, err) } - return n.AppendAtNs, err + n.AppendAtNs = util.BytesToUint64(bytes) + fileTailOffset := offset + needle.GetActualSize(size, v) + fileSize, _, err := datFile.GetStat() + if err != nil { + return 0, fmt.Errorf("stat file %s: %v", datFile.Name(), err) + } + if fileSize == fileTailOffset { + return n.AppendAtNs, nil + } + if fileSize > fileTailOffset { + glog.Warningf("Truncate %s from %d bytes to %d bytes!", datFile.Name(), fileSize, fileTailOffset) + err = datFile.Truncate(fileTailOffset) + if err == nil { + return n.AppendAtNs, nil + } + return n.AppendAtNs, fmt.Errorf("truncate file %s: %v", datFile.Name(), err) + } + glog.Warningf("data file %s has %d bytes, less than expected %d bytes!", datFile.Name(), fileSize, fileTailOffset) } if err = n.ReadData(datFile, offset, size, v); err != nil { return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", offset, offset+int64(size), err) diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go index 73e2de02b..05684cbdb 100644 --- a/weed/storage/volume_loading.go +++ b/weed/storage/volume_loading.go @@ -89,7 +89,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, err) } } - if v.lastAppendAtNs, err = CheckVolumeDataIntegrity(v, indexFile); err != nil { + if v.lastAppendAtNs, err = CheckAndFixVolumeDataIntegrity(v, indexFile); err != nil { v.noWriteOrDelete = true glog.V(0).Infof("volumeDataIntegrityChecking failed %v", err) } From 9ff0490dab409781d7085514fe0d8970f1d881e1 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 28 Oct 2020 01:29:06 -0700 Subject: [PATCH 11/84] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d5a8a80d..f718b04a6 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ On top of the object store, optional [Filer] can support directories and POSIX a * Automatic compaction to reclaim disk space after deletion or update. * [Automatic entry TTL expiration][VolumeServerTTL]. * Any server with some disk spaces can add to the total storage space. -* Adding/Removing servers does **not** cause any data re-balancing. +* Adding/Removing servers does **not** cause any data re-balancing unless trigged by admin commands. * Optional picture resizing. * Support ETag, Accept-Range, Last-Modified, etc. * Support in-memory/leveldb/readonly mode tuning for memory/performance balance. From 5db46bb0dde30b19c166454d89b43d1bc934d89f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 28 Oct 2020 01:45:55 -0700 Subject: [PATCH 12/84] typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f718b04a6..ddf41b396 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ On top of the object store, optional [Filer] can support directories and POSIX a * Automatic compaction to reclaim disk space after deletion or update. * [Automatic entry TTL expiration][VolumeServerTTL]. * Any server with some disk spaces can add to the total storage space. -* Adding/Removing servers does **not** cause any data re-balancing unless trigged by admin commands. +* Adding/Removing servers does **not** cause any data re-balancing unless triggered by admin commands. * Optional picture resizing. * Support ETag, Accept-Range, Last-Modified, etc. * Support in-memory/leveldb/readonly mode tuning for memory/performance balance. From 2dcc178d0ae20a013a901e4b4472dc9d8843ba59 Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Wed, 28 Oct 2020 18:16:05 +0800 Subject: [PATCH 13/84] save s3 metadata to filer. --- weed/server/filer_server_handlers_read.go | 2 +- .../filer_server_handlers_write_autochunk.go | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 731bd3545..16edb6167 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -97,7 +97,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, if r.Method == "GET" { tagCount := 0 for k, _ := range entry.Extended { - if strings.HasPrefix(k, "x-amz-tagging-") { + if strings.HasPrefix(k, "X-Amz-Tagging-") { tagCount++ } } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index d86d49b2a..d996c6208 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -176,6 +176,8 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa Size: chunkOffset, } + fs.saveAmzMetaData(r, entry) + if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil { fs.filer.DeleteChunks(entry.Chunks) replyerr = dbErr @@ -308,3 +310,36 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http } return filerResult, replyerr } + +func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) { + var ( + storageClass = "X-Amz-Storage-Class" + objectTagging = "X-Amz-Tagging" + userMetaPrefix = "X-Amz-Meta-" + ) + + if entry.Extended == nil { + entry.Extended = make(map[string][]byte) + } + + if sc := r.Header.Get(storageClass); sc != "" { + entry.Extended[storageClass] = []byte(sc) + } + + if tags := r.Header.Get(objectTagging); tags != "" { + for _, v := range strings.Split(tags, "&") { + tag := strings.Split(v, "=") + if len(tag) == 2 { + entry.Extended[objectTagging+"-"+tag[0]] = []byte(tag[1]) + } + } + } + + for header, values := range r.Header { + if strings.HasPrefix(header, userMetaPrefix) { + for _, value := range values { + entry.Extended[header] = []byte(value) + } + } + } +} From 1ad1b8c4f63fd36912dd9844d45d8ff5dc0170b5 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 28 Oct 2020 17:38:26 +0500 Subject: [PATCH 14/84] collection Volume ReadOnly Count with detailed status --- weed/storage/store.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/weed/storage/store.go b/weed/storage/store.go index b9fcfcba9..3a8c07e6e 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -200,7 +200,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { maxVolumeCount := 0 var maxFileKey NeedleId collectionVolumeSize := make(map[string]uint64) - collectionVolumeReadOnlyCount := make(map[string]uint8) + collectionVolumeReadOnlyCount := make(map[string]map[string]uint8) for _, location := range s.Locations { var deleteVids []needle.VolumeId maxVolumeCount = maxVolumeCount + location.MaxVolumeCount @@ -220,11 +220,25 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { } } collectionVolumeSize[v.Collection] += volumeMessage.Size + if _, exist := collectionVolumeReadOnlyCount[v.Collection]; !exist { + collectionVolumeReadOnlyCount[v.Collection] = map[string]uint8{ + "IsReadOnly": 0, + "noWriteOrDelete": 0, + "noWriteCanDelete": 0, + "isDiskSpaceLow": 0, + } + } if v.IsReadOnly() { - collectionVolumeReadOnlyCount[v.Collection] += 1 - } else { - if _, exist := collectionVolumeReadOnlyCount[v.Collection]; !exist { - collectionVolumeReadOnlyCount[v.Collection] = 0 + collectionVolumeReadOnlyCount[v.Collection] = make(map[string]uint8) + collectionVolumeReadOnlyCount[v.Collection]["IsReadOnly"] += 1 + if v.noWriteOrDelete { + collectionVolumeReadOnlyCount[v.Collection]["noWriteOrDelete"] += 1 + } + if v.noWriteCanDelete { + collectionVolumeReadOnlyCount[v.Collection]["noWriteCanDelete"] += 1 + } + if v.location.isDiskSpaceLow { + collectionVolumeReadOnlyCount[v.Collection]["isDiskSpaceLow"] += 1 } } } @@ -251,8 +265,10 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size)) } - for col, count := range collectionVolumeReadOnlyCount { - stats.VolumeServerReadOnlyVolumeGauge.WithLabelValues(col, "normal").Set(float64(count)) + for col, types := range collectionVolumeReadOnlyCount { + for t, count := range types { + stats.VolumeServerReadOnlyVolumeGauge.WithLabelValues(col, t).Set(float64(count)) + } } return &master_pb.Heartbeat{ From 884db215a14258d73e5744ae32fdb97e6f3cb09b Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 28 Oct 2020 22:47:09 +0500 Subject: [PATCH 15/84] add shell command volume mark writable --- weed/shell/command_volume_mark.go | 47 +++++++++++++++++++++++++++++++ weed/shell/command_volume_move.go | 15 ++++++++++ 2 files changed, 62 insertions(+) create mode 100644 weed/shell/command_volume_mark.go diff --git a/weed/shell/command_volume_mark.go b/weed/shell/command_volume_mark.go new file mode 100644 index 000000000..cb3e5c0bc --- /dev/null +++ b/weed/shell/command_volume_mark.go @@ -0,0 +1,47 @@ +package shell + +import ( + "flag" + "io" + + "github.com/chrislusf/seaweedfs/weed/storage/needle" +) + +func init() { + Commands = append(Commands, &commandVolumeMark{}) +} + +type commandVolumeMark struct { +} + +func (c *commandVolumeMark) Name() string { + return "volume.mark" +} + +func (c *commandVolumeMark) Help() string { + return `Mark volume readonly from one volume server + + volume.mark -node -volumeId -readonly true +` +} + +func (c *commandVolumeMark) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + if err = commandEnv.confirmIsLocked(); err != nil { + return + } + + volMarkCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + volumeIdInt := volMarkCommand.Int("volumeId", 0, "the volume id") + nodeStr := volMarkCommand.String("node", "", "the volume server :") + writable := volMarkCommand.Bool("writable", true, "volume mark writable/readonly") + if err = volMarkCommand.Parse(args); err != nil { + return nil + } + + sourceVolumeServer := *nodeStr + + volumeId := needle.VolumeId(*volumeIdInt) + + return markVolumeWritable(commandEnv.option.GrpcDialOption, volumeId, sourceVolumeServer, *writable) +} diff --git a/weed/shell/command_volume_move.go b/weed/shell/command_volume_move.go index b136604e5..2bc8dfad8 100644 --- a/weed/shell/command_volume_move.go +++ b/weed/shell/command_volume_move.go @@ -166,3 +166,18 @@ func deleteVolume(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sour return deleteErr }) } + +func markVolumeWritable(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer string, writable bool) (err error) { + return operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + if writable { + _, err = volumeServerClient.VolumeMarkWritable(context.Background(), &volume_server_pb.VolumeMarkWritableRequest{ + VolumeId: uint32(volumeId), + }) + } else { + _, err = volumeServerClient.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{ + VolumeId: uint32(volumeId), + }) + } + return err + }) +} From fab01f9d8d7f191c8830423611544eebfeb5213c Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 28 Oct 2020 23:17:03 +0500 Subject: [PATCH 16/84] add readonly --- weed/shell/command_volume_mark.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/weed/shell/command_volume_mark.go b/weed/shell/command_volume_mark.go index cb3e5c0bc..19b614310 100644 --- a/weed/shell/command_volume_mark.go +++ b/weed/shell/command_volume_mark.go @@ -2,6 +2,7 @@ package shell import ( "flag" + "fmt" "io" "github.com/chrislusf/seaweedfs/weed/storage/needle" @@ -19,9 +20,9 @@ func (c *commandVolumeMark) Name() string { } func (c *commandVolumeMark) Help() string { - return `Mark volume readonly from one volume server + return `Mark volume writable or readonly from one volume server - volume.mark -node -volumeId -readonly true + volume.mark -node -volumeId -writable or -readonly ` } @@ -34,14 +35,21 @@ func (c *commandVolumeMark) Do(args []string, commandEnv *CommandEnv, writer io. volMarkCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeIdInt := volMarkCommand.Int("volumeId", 0, "the volume id") nodeStr := volMarkCommand.String("node", "", "the volume server :") - writable := volMarkCommand.Bool("writable", true, "volume mark writable/readonly") + writable := volMarkCommand.Bool("writable", false, "volume mark writable") + readonly := volMarkCommand.Bool("readonly", false, "volume mark readonly") if err = volMarkCommand.Parse(args); err != nil { return nil } + markWritable := false + if (*writable && *readonly) || (!*writable && !*readonly) { + return fmt.Errorf("use -readonly or -writable") + } else if *writable { + markWritable = true + } sourceVolumeServer := *nodeStr volumeId := needle.VolumeId(*volumeIdInt) - return markVolumeWritable(commandEnv.option.GrpcDialOption, volumeId, sourceVolumeServer, *writable) + return markVolumeWritable(commandEnv.option.GrpcDialOption, volumeId, sourceVolumeServer, markWritable) } From 918546bbdb2054fbe8c0016c757fc73d03758fcd Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 29 Oct 2020 08:29:04 +0500 Subject: [PATCH 17/84] add WaitForReady to grpc DialOption --- weed/pb/grpc_client_server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index ce706e282..f19af43b2 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -62,6 +62,7 @@ func GrpcDial(ctx context.Context, address string, opts ...grpc.DialOption) (*gr grpc.WithDefaultCallOptions( grpc.MaxCallSendMsgSize(Max_Message_Size), grpc.MaxCallRecvMsgSize(Max_Message_Size), + grpc.WaitForReady(true), ), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 30 * time.Second, // client ping server if no activity for this long From b917be79559ec171f351ac842f9d4c20bb335e41 Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Thu, 29 Oct 2020 14:57:19 +0800 Subject: [PATCH 18/84] S3 bucket list, response with uploaded storageclass. --- weed/s3api/s3api_objects_list_handlers.go | 11 +- weed/server/filer_server_handlers_read.go | 4 +- .../filer_server_handlers_write_autochunk.go | 15 +- weed/util/http_header.go | 141 ++++++++++++++++++ 4 files changed, 157 insertions(+), 14 deletions(-) create mode 100644 weed/util/http_header.go diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 23406d6df..d291b588c 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -4,7 +4,6 @@ import ( "context" "encoding/xml" "fmt" - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" "io" "net/http" "net/url" @@ -13,6 +12,10 @@ import ( "strings" "time" + "github.com/chrislusf/seaweedfs/weed/util" + + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) @@ -137,6 +140,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m }) } } else { + storageClass := "STANDARD" + if v, ok := entry.Extended[util.AmzStorageClass]; ok { + storageClass = string(v) + } contents = append(contents, ListEntry{ Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), @@ -146,7 +153,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m ID: fmt.Sprintf("%x", entry.Attributes.Uid), DisplayName: entry.Attributes.UserName, }, - StorageClass: "STANDARD", + StorageClass: StorageClass(storageClass), }) } }) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 16edb6167..42d540df8 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -97,12 +97,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, if r.Method == "GET" { tagCount := 0 for k, _ := range entry.Extended { - if strings.HasPrefix(k, "X-Amz-Tagging-") { + if strings.HasPrefix(k, util.AmzObjectTagging+"-") { tagCount++ } } if tagCount > 0 { - w.Header().Set("x-amz-tag-count", strconv.Itoa(tagCount)) + w.Header().Set(util.AmzTagCount, strconv.Itoa(tagCount)) } } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index d996c6208..5396cc5ca 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -312,31 +312,26 @@ func (fs *FilerServer) mkdir(ctx context.Context, w http.ResponseWriter, r *http } func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) { - var ( - storageClass = "X-Amz-Storage-Class" - objectTagging = "X-Amz-Tagging" - userMetaPrefix = "X-Amz-Meta-" - ) if entry.Extended == nil { entry.Extended = make(map[string][]byte) } - if sc := r.Header.Get(storageClass); sc != "" { - entry.Extended[storageClass] = []byte(sc) + if sc := r.Header.Get(util.AmzStorageClass); sc != "" { + entry.Extended[util.AmzStorageClass] = []byte(sc) } - if tags := r.Header.Get(objectTagging); tags != "" { + if tags := r.Header.Get(util.AmzObjectTagging); tags != "" { for _, v := range strings.Split(tags, "&") { tag := strings.Split(v, "=") if len(tag) == 2 { - entry.Extended[objectTagging+"-"+tag[0]] = []byte(tag[1]) + entry.Extended[util.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) } } } for header, values := range r.Header { - if strings.HasPrefix(header, userMetaPrefix) { + if strings.HasPrefix(header, util.AmzUserMetaPrefix) { for _, value := range values { entry.Extended[header] = []byte(value) } diff --git a/weed/util/http_header.go b/weed/util/http_header.go new file mode 100644 index 000000000..f7633da83 --- /dev/null +++ b/weed/util/http_header.go @@ -0,0 +1,141 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +// Standard S3 HTTP response constants +const ( + LastModified = "Last-Modified" + Date = "Date" + ETag = "ETag" + ContentType = "Content-Type" + ContentMD5 = "Content-Md5" + ContentEncoding = "Content-Encoding" + Expires = "Expires" + ContentLength = "Content-Length" + ContentLanguage = "Content-Language" + ContentRange = "Content-Range" + Connection = "Connection" + AcceptRanges = "Accept-Ranges" + AmzBucketRegion = "X-Amz-Bucket-Region" + ServerInfo = "Server" + RetryAfter = "Retry-After" + Location = "Location" + CacheControl = "Cache-Control" + ContentDisposition = "Content-Disposition" + Authorization = "Authorization" + Action = "Action" + Range = "Range" +) + +// Non standard S3 HTTP response constants +const ( + XCache = "X-Cache" + XCacheLookup = "X-Cache-Lookup" +) + +// Standard S3 HTTP request constants +const ( + IfModifiedSince = "If-Modified-Since" + IfUnmodifiedSince = "If-Unmodified-Since" + IfMatch = "If-Match" + IfNoneMatch = "If-None-Match" + + // S3 storage class + AmzStorageClass = "x-amz-storage-class" + + // S3 user-defined metadata + AmzUserMetaPrefix = "X-Amz-Meta-" + + // S3 object version ID + AmzVersionID = "x-amz-version-id" + AmzDeleteMarker = "x-amz-delete-marker" + + // S3 object tagging + AmzObjectTagging = "X-Amz-Tagging" + AmzTagCount = "x-amz-tagging-count" + AmzTagDirective = "X-Amz-Tagging-Directive" + + // S3 extensions + AmzCopySourceIfModifiedSince = "x-amz-copy-source-if-modified-since" + AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since" + + AmzCopySourceIfNoneMatch = "x-amz-copy-source-if-none-match" + AmzCopySourceIfMatch = "x-amz-copy-source-if-match" + + AmzCopySource = "X-Amz-Copy-Source" + AmzCopySourceVersionID = "X-Amz-Copy-Source-Version-Id" + AmzCopySourceRange = "X-Amz-Copy-Source-Range" + AmzMetadataDirective = "X-Amz-Metadata-Directive" + AmzObjectLockMode = "X-Amz-Object-Lock-Mode" + AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" + AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold" + AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention" + AmzBucketReplicationStatus = "X-Amz-Replication-Status" + // Multipart parts count + AmzMpPartsCount = "x-amz-mp-parts-count" + + // Object date/time of expiration + AmzExpiration = "x-amz-expiration" + + // Dummy putBucketACL + AmzACL = "x-amz-acl" + + // Signature V4 related contants. + AmzContentSha256 = "X-Amz-Content-Sha256" + AmzDate = "X-Amz-Date" + AmzAlgorithm = "X-Amz-Algorithm" + AmzExpires = "X-Amz-Expires" + AmzSignedHeaders = "X-Amz-SignedHeaders" + AmzSignature = "X-Amz-Signature" + AmzCredential = "X-Amz-Credential" + AmzSecurityToken = "X-Amz-Security-Token" + AmzDecodedContentLength = "X-Amz-Decoded-Content-Length" + + AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length" + AmzMetaUnencryptedContentMD5 = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5" + + // Signature v2 related constants + AmzSignatureV2 = "Signature" + AmzAccessKeyID = "AWSAccessKeyId" + + // Response request id. + AmzRequestID = "x-amz-request-id" + + // Deployment id. + MinioDeploymentID = "x-minio-deployment-id" + + // Server-Status + MinIOServerStatus = "x-minio-server-status" + + // Delete special flag to force delete a bucket + MinIOForceDelete = "x-minio-force-delete" + + // Header indicates if the mtime should be preserved by client + MinIOSourceMTime = "x-minio-source-mtime" + + // Header indicates if the etag should be preserved by client + MinIOSourceETag = "x-minio-source-etag" +) + +// Common http query params S3 API +const ( + VersionID = "versionId" + + PartNumber = "partNumber" + + UploadID = "uploadId" +) From c6d9974fe76be3004161d9ccf4328cebc5a25287 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 00:24:16 -0700 Subject: [PATCH 19/84] server: Add option to enable/disable volume server fix https://github.com/chrislusf/seaweedfs/issues/1572 --- weed/command/server.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/weed/command/server.go b/weed/command/server.go index 6a78fb3f4..d06b1d40a 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -2,13 +2,14 @@ package command import ( "fmt" - stats_collect "github.com/chrislusf/seaweedfs/weed/stats" "os" "runtime" "runtime/pprof" "strings" "time" + stats_collect "github.com/chrislusf/seaweedfs/weed/stats" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -60,9 +61,10 @@ var ( serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") // pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") - isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") - isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") - isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker") + isStartingVolumeServer = cmdServer.Flag.Bool("volume", true, "whether to start volume server") + isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") + isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway") + isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker") serverWhiteList []string @@ -214,7 +216,7 @@ func runServer(cmd *Command, args []string) bool { } // start volume server - { + if *isStartingVolumeServer { go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent) } From 22a9ea05129d8c4776187d667f9e31b03ddbb2e0 Mon Sep 17 00:00:00 2001 From: "ruitao.liu" Date: Thu, 29 Oct 2020 16:05:40 +0800 Subject: [PATCH 20/84] adjust s3 header file. --- weed/s3api/filer_util_tags.go | 3 +- weed/s3api/http/header.go | 30 ++++ weed/s3api/s3api_objects_list_handlers.go | 8 +- weed/server/filer_server_handlers_read.go | 5 +- .../filer_server_handlers_write_autochunk.go | 11 +- weed/util/http_header.go | 141 ------------------ 6 files changed, 44 insertions(+), 154 deletions(-) create mode 100644 weed/s3api/http/header.go delete mode 100644 weed/util/http_header.go diff --git a/weed/s3api/filer_util_tags.go b/weed/s3api/filer_util_tags.go index 3d4da7825..75d3b37d0 100644 --- a/weed/s3api/filer_util_tags.go +++ b/weed/s3api/filer_util_tags.go @@ -4,10 +4,11 @@ import ( "strings" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" ) const ( - S3TAG_PREFIX = "s3-" + S3TAG_PREFIX = xhttp.AmzObjectTagging + "-" ) func (s3a *S3ApiServer) getTags(parentDirectoryPath string, entryName string) (tags map[string]string, err error) { diff --git a/weed/s3api/http/header.go b/weed/s3api/http/header.go new file mode 100644 index 000000000..2802b560f --- /dev/null +++ b/weed/s3api/http/header.go @@ -0,0 +1,30 @@ +/* + * MinIO Cloud Storage, (C) 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package http + +// Standard S3 HTTP request constants +const ( + // S3 storage class + AmzStorageClass = "x-amz-storage-class" + + // S3 user-defined metadata + AmzUserMetaPrefix = "X-Amz-Meta-" + + // S3 object tagging + AmzObjectTagging = "X-Amz-Tagging" + AmzTagCount = "x-amz-tagging-count" +) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index d291b588c..5d63f1039 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -12,12 +12,10 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/util" - - "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" ) type ListBucketResultV2 struct { @@ -141,7 +139,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } } else { storageClass := "STANDARD" - if v, ok := entry.Extended[util.AmzStorageClass]; ok { + if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { storageClass = string(v) } contents = append(contents, ListEntry{ diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 42d540df8..7b08e1686 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -15,6 +15,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -97,12 +98,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, if r.Method == "GET" { tagCount := 0 for k, _ := range entry.Extended { - if strings.HasPrefix(k, util.AmzObjectTagging+"-") { + if strings.HasPrefix(k, xhttp.AmzObjectTagging+"-") { tagCount++ } } if tagCount > 0 { - w.Header().Set(util.AmzTagCount, strconv.Itoa(tagCount)) + w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount)) } } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 5396cc5ca..d308dafa2 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -18,6 +18,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" @@ -317,21 +318,21 @@ func (fs *FilerServer) saveAmzMetaData(r *http.Request, entry *filer.Entry) { entry.Extended = make(map[string][]byte) } - if sc := r.Header.Get(util.AmzStorageClass); sc != "" { - entry.Extended[util.AmzStorageClass] = []byte(sc) + if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" { + entry.Extended[xhttp.AmzStorageClass] = []byte(sc) } - if tags := r.Header.Get(util.AmzObjectTagging); tags != "" { + if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" { for _, v := range strings.Split(tags, "&") { tag := strings.Split(v, "=") if len(tag) == 2 { - entry.Extended[util.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) + entry.Extended[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1]) } } } for header, values := range r.Header { - if strings.HasPrefix(header, util.AmzUserMetaPrefix) { + if strings.HasPrefix(header, xhttp.AmzUserMetaPrefix) { for _, value := range values { entry.Extended[header] = []byte(value) } diff --git a/weed/util/http_header.go b/weed/util/http_header.go deleted file mode 100644 index f7633da83..000000000 --- a/weed/util/http_header.go +++ /dev/null @@ -1,141 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package util - -// Standard S3 HTTP response constants -const ( - LastModified = "Last-Modified" - Date = "Date" - ETag = "ETag" - ContentType = "Content-Type" - ContentMD5 = "Content-Md5" - ContentEncoding = "Content-Encoding" - Expires = "Expires" - ContentLength = "Content-Length" - ContentLanguage = "Content-Language" - ContentRange = "Content-Range" - Connection = "Connection" - AcceptRanges = "Accept-Ranges" - AmzBucketRegion = "X-Amz-Bucket-Region" - ServerInfo = "Server" - RetryAfter = "Retry-After" - Location = "Location" - CacheControl = "Cache-Control" - ContentDisposition = "Content-Disposition" - Authorization = "Authorization" - Action = "Action" - Range = "Range" -) - -// Non standard S3 HTTP response constants -const ( - XCache = "X-Cache" - XCacheLookup = "X-Cache-Lookup" -) - -// Standard S3 HTTP request constants -const ( - IfModifiedSince = "If-Modified-Since" - IfUnmodifiedSince = "If-Unmodified-Since" - IfMatch = "If-Match" - IfNoneMatch = "If-None-Match" - - // S3 storage class - AmzStorageClass = "x-amz-storage-class" - - // S3 user-defined metadata - AmzUserMetaPrefix = "X-Amz-Meta-" - - // S3 object version ID - AmzVersionID = "x-amz-version-id" - AmzDeleteMarker = "x-amz-delete-marker" - - // S3 object tagging - AmzObjectTagging = "X-Amz-Tagging" - AmzTagCount = "x-amz-tagging-count" - AmzTagDirective = "X-Amz-Tagging-Directive" - - // S3 extensions - AmzCopySourceIfModifiedSince = "x-amz-copy-source-if-modified-since" - AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since" - - AmzCopySourceIfNoneMatch = "x-amz-copy-source-if-none-match" - AmzCopySourceIfMatch = "x-amz-copy-source-if-match" - - AmzCopySource = "X-Amz-Copy-Source" - AmzCopySourceVersionID = "X-Amz-Copy-Source-Version-Id" - AmzCopySourceRange = "X-Amz-Copy-Source-Range" - AmzMetadataDirective = "X-Amz-Metadata-Directive" - AmzObjectLockMode = "X-Amz-Object-Lock-Mode" - AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" - AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold" - AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention" - AmzBucketReplicationStatus = "X-Amz-Replication-Status" - // Multipart parts count - AmzMpPartsCount = "x-amz-mp-parts-count" - - // Object date/time of expiration - AmzExpiration = "x-amz-expiration" - - // Dummy putBucketACL - AmzACL = "x-amz-acl" - - // Signature V4 related contants. - AmzContentSha256 = "X-Amz-Content-Sha256" - AmzDate = "X-Amz-Date" - AmzAlgorithm = "X-Amz-Algorithm" - AmzExpires = "X-Amz-Expires" - AmzSignedHeaders = "X-Amz-SignedHeaders" - AmzSignature = "X-Amz-Signature" - AmzCredential = "X-Amz-Credential" - AmzSecurityToken = "X-Amz-Security-Token" - AmzDecodedContentLength = "X-Amz-Decoded-Content-Length" - - AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length" - AmzMetaUnencryptedContentMD5 = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5" - - // Signature v2 related constants - AmzSignatureV2 = "Signature" - AmzAccessKeyID = "AWSAccessKeyId" - - // Response request id. - AmzRequestID = "x-amz-request-id" - - // Deployment id. - MinioDeploymentID = "x-minio-deployment-id" - - // Server-Status - MinIOServerStatus = "x-minio-server-status" - - // Delete special flag to force delete a bucket - MinIOForceDelete = "x-minio-force-delete" - - // Header indicates if the mtime should be preserved by client - MinIOSourceMTime = "x-minio-source-mtime" - - // Header indicates if the etag should be preserved by client - MinIOSourceETag = "x-minio-source-etag" -) - -// Common http query params S3 API -const ( - VersionID = "versionId" - - PartNumber = "partNumber" - - UploadID = "uploadId" -) From 47047516d61614fddce7163c1bca2e6e5e83f01f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 15:46:26 -0700 Subject: [PATCH 21/84] Revert "increase default volume file size limit to 1024" This reverts commit 0983060a I am confused why this was changed to 1024 by myself. 1GB should be too large for most cases. --- weed/command/server.go | 2 +- weed/command/volume.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/command/server.go b/weed/command/server.go index d06b1d40a..4a9c2411a 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -101,7 +101,7 @@ func init() { serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.") serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.") serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second") - serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 1024, "limit file size to avoid out of memory") + serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory") serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server") serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") diff --git a/weed/command/volume.go b/weed/command/volume.go index d73c24ed1..ce5992665 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -80,7 +80,7 @@ func init() { v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file") v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file") v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second") - v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 1024, "limit file size to avoid out of memory") + v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory") v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") } From 0b68b68ec4a3f967f56a10e3d156e18139e771c4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 16:32:45 -0700 Subject: [PATCH 22/84] reload entry only when it is a hard link fix https://github.com/chrislusf/seaweedfs/issues/1581 --- weed/filesys/file.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 7aa1016d7..fb7b83cbe 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -253,9 +253,15 @@ func (file *File) Forget() { } func (file *File) maybeLoadEntry(ctx context.Context) error { - if (file.entry != nil && len(file.entry.HardLinkId) != 0) || file.isOpen > 0 { + if file.isOpen > 0 { return nil } + if file.entry != nil { + if len(file.entry.HardLinkId) == 0 { + // only always reload hard link + return nil + } + } entry, err := file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name) if err != nil { glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err) @@ -263,6 +269,8 @@ func (file *File) maybeLoadEntry(ctx context.Context) error { } if entry != nil { file.setEntry(entry) + } else { + glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err) } return nil } From ba6439055590af827e7a5d75232ebc8c8ecaa78f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 20:54:05 -0700 Subject: [PATCH 23/84] Create build_binary.yml --- .github/workflows/build_binary.yml | 55 ++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/build_binary.yml diff --git a/.github/workflows/build_binary.yml b/.github/workflows/build_binary.yml new file mode 100644 index 000000000..265369bcd --- /dev/null +++ b/.github/workflows/build_binary.yml @@ -0,0 +1,55 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Runs a single command using the runners shell + - name: Run a one-line script + run: echo Hello, world! + + # Runs a set of commands using the runners shell + - name: Run a multi-line script + run: | + echo Add other actions to build, + echo test, and deploy your project. + - name: Go Release Binaries + uses: wangyoucao577/go-release-action@v1.8 + with: + # GITHUB_TOKEN for uploading releases to Github asserts. + github_token: ${{ secrets.GITHUB_TOKEN }} + # GOOS is the running programs operating system target: one of darwin, freebsd, linux, and so on. + goos: linux # default is + # GOARCH is the running programs architecture target: one of 386, amd64, arm, s390x, and so on. + goarch: amd64 # default is + # The `Go` compiler version. + goversion: 1.14 # optional, default is + # Additional arguments to pass the go build command. + build_flags: 5BytesOffset # optional, default is + # Values to provide to the -ldflags argument + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + # Specify another binary name if do not want to use repository basename + binary_name: weed_large_disk + # Extra command that will be executed before `go build`, may for solving dependency + + From 4c231054fbc55107b1ca433498350d55617310fb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 21:01:25 -0700 Subject: [PATCH 24/84] Update build_binary.yml --- .github/workflows/build_binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_binary.yml b/.github/workflows/build_binary.yml index 265369bcd..eae2d57f5 100644 --- a/.github/workflows/build_binary.yml +++ b/.github/workflows/build_binary.yml @@ -43,7 +43,7 @@ jobs: # The `Go` compiler version. goversion: 1.14 # optional, default is # Additional arguments to pass the go build command. - build_flags: 5BytesOffset # optional, default is + build_flags: -tags 5BytesOffset # optional, default is # Values to provide to the -ldflags argument ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} # Where to run `go build .` From cf0cd1294353cc319fd1f13988f04dbcfe8a16e9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 21:16:27 -0700 Subject: [PATCH 25/84] Update build_binary.yml --- .github/workflows/build_binary.yml | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/.github/workflows/build_binary.yml b/.github/workflows/build_binary.yml index eae2d57f5..77b27a48a 100644 --- a/.github/workflows/build_binary.yml +++ b/.github/workflows/build_binary.yml @@ -2,54 +2,28 @@ name: CI -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: push: branches: [ master ] pull_request: branches: [ master ] -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" build: - # The type of runner that the job will run on runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - - # Runs a single command using the runners shell - - name: Run a one-line script - run: echo Hello, world! - - # Runs a set of commands using the runners shell - - name: Run a multi-line script - run: | - echo Add other actions to build, - echo test, and deploy your project. - name: Go Release Binaries uses: wangyoucao577/go-release-action@v1.8 with: - # GITHUB_TOKEN for uploading releases to Github asserts. github_token: ${{ secrets.GITHUB_TOKEN }} - # GOOS is the running programs operating system target: one of darwin, freebsd, linux, and so on. goos: linux # default is - # GOARCH is the running programs architecture target: one of 386, amd64, arm, s390x, and so on. goarch: amd64 # default is - # The `Go` compiler version. goversion: 1.14 # optional, default is - # Additional arguments to pass the go build command. build_flags: -tags 5BytesOffset # optional, default is - # Values to provide to the -ldflags argument ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} # Where to run `go build .` project_path: weed - # Specify another binary name if do not want to use repository basename - binary_name: weed_large_disk - # Extra command that will be executed before `go build`, may for solving dependency + binary_name: weed-large-disk From 6560ac6466bfb335f0505a1e6cd9f8881972d878 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 22:25:09 -0700 Subject: [PATCH 26/84] volume loading: trim out unreachable idx file content fix https://github.com/chrislusf/seaweedfs/issues/1583 --- weed/storage/volume_checking.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 78c693f5c..048944199 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "io" "os" "github.com/chrislusf/seaweedfs/weed/glog" @@ -20,13 +21,25 @@ func CheckAndFixVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAt if indexSize == 0 { return 0, nil } + healthyIndexSize := indexSize for i := 1; i <= 10; i++ { // check and fix last 10 entries lastAppendAtNs, err = doCheckAndFixVolumeData(v, indexFile, indexSize-int64(i)*NeedleMapEntrySize) + if err == io.EOF { + healthyIndexSize = indexSize - int64(i)*NeedleMapEntrySize + continue + } if err != ErrorSizeMismatch { break } } + if healthyIndexSize < indexSize { + glog.Warningf("CheckAndFixVolumeDataIntegrity truncate idx file %s from %d to %d", indexFile.Name(), indexSize, healthyIndexSize) + err = indexFile.Truncate(healthyIndexSize) + if err != nil { + glog.Warningf("CheckAndFixVolumeDataIntegrity truncate idx file %s from %d to %d: %v", indexFile.Name(), indexSize, healthyIndexSize, err) + } + } return } @@ -73,6 +86,9 @@ func readIndexEntryAtOffset(indexFile *os.File, offset int64) (bytes []byte, err func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, offset int64, key NeedleId, size Size) (lastAppendAtNs uint64, err error) { n, _, _, err := needle.ReadNeedleHeader(datFile, v, offset) + if err == io.EOF { + return 0, err + } if err != nil { return 0, fmt.Errorf("read %s at %d", datFile.Name(), offset) } @@ -82,8 +98,11 @@ func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, if v == needle.Version3 { bytes := make([]byte, TimestampSize) _, err = datFile.ReadAt(bytes, offset+NeedleHeaderSize+int64(size)+needle.NeedleChecksumSize) + if err == io.EOF { + return 0, err + } if err != nil { - return 0, fmt.Errorf("check %s entry offset %d size %d: %v", datFile.Name(), offset, size, err) + return 0, fmt.Errorf("verifyNeedleIntegrity check %s entry offset %d size %d: %v", datFile.Name(), offset, size, err) } n.AppendAtNs = util.BytesToUint64(bytes) fileTailOffset := offset + needle.GetActualSize(size, v) From 23a0551c2612b8a1a05201481b5ea7eea1189dbc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 22:49:29 -0700 Subject: [PATCH 27/84] Update go.yml --- .github/workflows/go.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b2948a0b7..3d9bcf4a2 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -35,3 +35,16 @@ jobs: - name: Test run: cd weed; go test -v ./... + + - name: Go Release Binaries + uses: wangyoucao577/go-release-action@v1.8 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: linux # default is + goarch: amd64 # default is + goversion: 1.14 # optional, default is + build_flags: -tags 5BytesOffset # optional, default is + ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} + # Where to run `go build .` + project_path: weed + binary_name: weed-large-disk From 08cbc714f0c10fa927794dde6a588cd8b6f528f3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 22:49:39 -0700 Subject: [PATCH 28/84] Delete build_binary.yml --- .github/workflows/build_binary.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/build_binary.yml diff --git a/.github/workflows/build_binary.yml b/.github/workflows/build_binary.yml deleted file mode 100644 index 77b27a48a..000000000 --- a/.github/workflows/build_binary.yml +++ /dev/null @@ -1,29 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.8 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: linux # default is - goarch: amd64 # default is - goversion: 1.14 # optional, default is - build_flags: -tags 5BytesOffset # optional, default is - ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} - # Where to run `go build .` - project_path: weed - binary_name: weed-large-disk - - From 11697ca9fdc24a69c37a8a7014f4cfc0200c0bbf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:01:41 -0700 Subject: [PATCH 29/84] Update go.yml --- .github/workflows/go.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 3d9bcf4a2..f5fc53848 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -42,7 +42,6 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} goos: linux # default is goarch: amd64 # default is - goversion: 1.14 # optional, default is build_flags: -tags 5BytesOffset # optional, default is ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} # Where to run `go build .` From e379fc36583a1d49c5ccc2ea5a822010552ada38 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:09:28 -0700 Subject: [PATCH 30/84] Update go.yml --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f5fc53848..2beb63c50 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v2 with: - go-version: ^1.13 + go-version: ^1.15 id: go - name: Check out code into the Go module directory From 202964a09b86014e165899d125ab97385b39e2a2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:15:28 -0700 Subject: [PATCH 31/84] Update go.yml --- .github/workflows/go.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2beb63c50..458e4b8a8 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -18,6 +18,8 @@ jobs: with: go-version: ^1.15 id: go + - name: create symlink + run: sudo ln -f -s $GOROOT/bin/* /usr/bin/ - name: Check out code into the Go module directory uses: actions/checkout@v2 From abff1f67a60e4ffbf42082d34b99187fa8e47f6f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:26:40 -0700 Subject: [PATCH 32/84] Update go.yml --- .github/workflows/go.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 458e4b8a8..c9a3c2ff4 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -13,31 +13,9 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.x - uses: actions/setup-go@v2 - with: - go-version: ^1.15 - id: go - - name: create symlink - run: sudo ln -f -s $GOROOT/bin/* /usr/bin/ - - name: Check out code into the Go module directory uses: actions/checkout@v2 - - name: Get dependencies - run: | - cd weed; go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - - - name: Build - run: cd weed; go build -v . - - - name: Test - run: cd weed; go test -v ./... - - name: Go Release Binaries uses: wangyoucao577/go-release-action@v1.8 with: From 02dc51b1edc0fd4f06857b409fca52aaea1664ff Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:31:04 -0700 Subject: [PATCH 33/84] properly clear out the file.entry --- weed/filesys/wfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 759e21b15..293b09dcf 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -96,7 +96,7 @@ func NewSeaweedFileSystem(option *Option) *WFS { fsNode := wfs.fsNodeCache.GetFsNode(filePath) if fsNode != nil { if file, ok := fsNode.(*File); ok { - file.entry = nil + file.setEntry(nil) } } }) From 737d6ed65000eb1145c5c9f6138d5a65f26efaab Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:41:10 -0700 Subject: [PATCH 34/84] Update and rename go.yml to release.yml --- .github/workflows/go.yml | 29 ----------------------------- .github/workflows/release.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 29 deletions(-) delete mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index c9a3c2ff4..000000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Go - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - - build: - name: Build - runs-on: ubuntu-latest - steps: - - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - - - name: Go Release Binaries - uses: wangyoucao577/go-release-action@v1.8 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: linux # default is - goarch: amd64 # default is - build_flags: -tags 5BytesOffset # optional, default is - ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} - # Where to run `go build .` - project_path: weed - binary_name: weed-large-disk diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..b29f3be2d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + +# - name: Go Release Binaries +# uses: wangyoucao577/go-release-action@v1.8 +# with: +# github_token: ${{ secrets.GITHUB_TOKEN }} +# goos: linux # default is +# goarch: amd64 # default is +# build_flags: -tags 5BytesOffset # optional, default is +# ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}} +# # Where to run `go build .` +# project_path: weed +# binary_name: weed-large-disk From 0ab72a0904597dbd2175f55a64faa538278ee6fd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:41:36 -0700 Subject: [PATCH 35/84] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b29f3be2d..346837c7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Go +name: Release on: push: From 09a7cb83535d5c2722ea5f7cd9bd2d0ba11c221a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:41:54 -0700 Subject: [PATCH 36/84] Create go.yml --- .github/workflows/go.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 000000000..1c5b4860e --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,36 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + go get -v -t -d ./... + if [ -f Gopkg.toml ]; then + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + dep ensure + fi + + - name: Build + run: go build -v . + + - name: Test + run: go test -v . From 0cdf1726948b05d89e0b8140aa765fd6ceac05dc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Oct 2020 23:46:54 -0700 Subject: [PATCH 37/84] Update go.yml --- .github/workflows/go.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1c5b4860e..b2948a0b7 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -17,20 +17,21 @@ jobs: uses: actions/setup-go@v2 with: go-version: ^1.13 + id: go - name: Check out code into the Go module directory uses: actions/checkout@v2 - name: Get dependencies run: | - go get -v -t -d ./... + cd weed; go get -v -t -d ./... if [ -f Gopkg.toml ]; then curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh dep ensure fi - name: Build - run: go build -v . + run: cd weed; go build -v . - name: Test - run: go test -v . + run: cd weed; go test -v ./... From 6135cbaa02114d712241d17243a2d759b8f70733 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 30 Oct 2020 00:02:16 -0700 Subject: [PATCH 38/84] clear file.entry --- weed/filesys/file.go | 6 ++++++ weed/filesys/wfs.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index fb7b83cbe..d2ae34654 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -301,6 +301,12 @@ func (file *File) setEntry(entry *filer_pb.Entry) { file.reader = nil } +func (file *File) clearEntry() { + file.entry = nil + file.entryViewCache = nil + file.reader = nil +} + func (file *File) saveEntry() error { return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 293b09dcf..d31579fce 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -96,7 +96,7 @@ func NewSeaweedFileSystem(option *Option) *WFS { fsNode := wfs.fsNodeCache.GetFsNode(filePath) if fsNode != nil { if file, ok := fsNode.(*File); ok { - file.setEntry(nil) + file.clearEntry() } } }) From 19098f2c2d3c5d795cde7d0136371fd6fb2ae118 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 30 Oct 2020 01:23:23 -0700 Subject: [PATCH 39/84] make a copy of the filer.entry for most read operations --- weed/filesys/dir_link.go | 13 ++++--- weed/filesys/file.go | 81 ++++++++++++++++++++++------------------ 2 files changed, 51 insertions(+), 43 deletions(-) diff --git a/weed/filesys/dir_link.go b/weed/filesys/dir_link.go index f6bc41b56..ba3280f03 100644 --- a/weed/filesys/dir_link.go +++ b/weed/filesys/dir_link.go @@ -31,7 +31,7 @@ func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (f glog.V(4).Infof("Link: %v/%v -> %v/%v", oldFile.dir.FullPath(), oldFile.Name, dir.FullPath(), req.NewName) - if err := oldFile.maybeLoadEntry(ctx); err != nil { + if _, err := oldFile.maybeLoadEntry(ctx); err != nil { return nil, err } @@ -86,7 +86,7 @@ func (dir *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (f // create new file node newNode := dir.newFile(req.NewName, request.Entry) newFile := newNode.(*File) - if err := newFile.maybeLoadEntry(ctx); err != nil { + if _, err := newFile.maybeLoadEntry(ctx); err != nil { return nil, err } @@ -138,16 +138,17 @@ func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, func (file *File) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { - if err := file.maybeLoadEntry(ctx); err != nil { + entry, err := file.maybeLoadEntry(ctx) + if err != nil { return "", err } - if os.FileMode(file.entry.Attributes.FileMode)&os.ModeSymlink == 0 { + if os.FileMode(entry.Attributes.FileMode)&os.ModeSymlink == 0 { return "", fuse.Errno(syscall.EINVAL) } - glog.V(4).Infof("Readlink: %v/%v => %v", file.dir.FullPath(), file.Name, file.entry.Attributes.SymlinkTarget) + glog.V(4).Infof("Readlink: %v/%v => %v", file.dir.FullPath(), file.Name, entry.Attributes.SymlinkTarget) - return file.entry.Attributes.SymlinkTarget, nil + return entry.Attributes.SymlinkTarget, nil } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index d2ae34654..4662c23db 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -43,32 +43,33 @@ func (file *File) fullpath() util.FullPath { return util.NewFullPath(file.dir.FullPath(), file.Name) } -func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error { +func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) { glog.V(4).Infof("file Attr %s, open:%v, existing attr: %+v", file.fullpath(), file.isOpen, attr) - if file.isOpen <= 0 { - if err := file.maybeLoadEntry(ctx); err != nil { + entry := file.entry + if file.isOpen <= 0 || entry == nil { + if entry, err = file.maybeLoadEntry(ctx); err != nil { return err } } attr.Inode = file.fullpath().AsInode() attr.Valid = time.Second - attr.Mode = os.FileMode(file.entry.Attributes.FileMode) - attr.Size = filer.FileSize(file.entry) + attr.Mode = os.FileMode(entry.Attributes.FileMode) + attr.Size = filer.FileSize(entry) if file.isOpen > 0 { - attr.Size = file.entry.Attributes.FileSize + attr.Size = entry.Attributes.FileSize glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size) } - attr.Crtime = time.Unix(file.entry.Attributes.Crtime, 0) - attr.Mtime = time.Unix(file.entry.Attributes.Mtime, 0) - attr.Gid = file.entry.Attributes.Gid - attr.Uid = file.entry.Attributes.Uid + attr.Crtime = time.Unix(entry.Attributes.Crtime, 0) + attr.Mtime = time.Unix(entry.Attributes.Mtime, 0) + attr.Gid = entry.Attributes.Gid + attr.Uid = entry.Attributes.Uid attr.Blocks = attr.Size/blockSize + 1 attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit) - if file.entry.HardLinkCounter > 0 { - attr.Nlink = uint32(file.entry.HardLinkCounter) + if entry.HardLinkCounter > 0 { + attr.Nlink = uint32(entry.HardLinkCounter) } return nil @@ -79,11 +80,12 @@ func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp glog.V(4).Infof("file Getxattr %s", file.fullpath()) - if err := file.maybeLoadEntry(ctx); err != nil { + entry, err := file.maybeLoadEntry(ctx) + if err != nil { return err } - return getxattr(file.entry, req, resp) + return getxattr(entry, req, resp) } func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { @@ -104,7 +106,8 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f glog.V(4).Infof("%v file setattr %+v", file.fullpath(), req) - if err := file.maybeLoadEntry(ctx); err != nil { + _, err := file.maybeLoadEntry(ctx) + if err != nil { return err } if file.isOpen > 0 { @@ -186,7 +189,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f return nil } - return file.saveEntry() + return file.saveEntry(file.entry) } @@ -194,15 +197,16 @@ func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name) - if err := file.maybeLoadEntry(ctx); err != nil { + entry, err := file.maybeLoadEntry(ctx) + if err != nil { return err } - if err := setxattr(file.entry, req); err != nil { + if err := setxattr(entry, req); err != nil { return err } - return file.saveEntry() + return file.saveEntry(entry) } @@ -210,15 +214,16 @@ func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name) - if err := file.maybeLoadEntry(ctx); err != nil { + entry, err := file.maybeLoadEntry(ctx) + if err != nil { return err } - if err := removexattr(file.entry, req); err != nil { + if err := removexattr(entry, req); err != nil { return err } - return file.saveEntry() + return file.saveEntry(entry) } @@ -226,11 +231,12 @@ func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, res glog.V(4).Infof("file Listxattr %s", file.fullpath()) - if err := file.maybeLoadEntry(ctx); err != nil { + entry, err := file.maybeLoadEntry(ctx) + if err != nil { return err } - if err := listxattr(file.entry, req, resp); err != nil { + if err := listxattr(entry, req, resp); err != nil { return err } @@ -252,27 +258,28 @@ func (file *File) Forget() { file.wfs.fsNodeCache.DeleteFsNode(t) } -func (file *File) maybeLoadEntry(ctx context.Context) error { +func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) { + entry = file.entry if file.isOpen > 0 { - return nil + return entry, nil } - if file.entry != nil { - if len(file.entry.HardLinkId) == 0 { + if entry != nil { + if len(entry.HardLinkId) == 0 { // only always reload hard link - return nil + return entry, nil } } - entry, err := file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name) + entry, err = file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name) if err != nil { glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err) - return err + return entry, err } if entry != nil { file.setEntry(entry) } else { glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err) } - return nil + return entry, nil } func (file *File) addChunks(chunks []*filer_pb.FileChunk) { @@ -297,7 +304,7 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) { func (file *File) setEntry(entry *filer_pb.Entry) { file.entry = entry - file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), file.entry.Chunks) + file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(file.wfs), entry.Chunks) file.reader = nil } @@ -307,15 +314,15 @@ func (file *File) clearEntry() { file.reader = nil } -func (file *File) saveEntry() error { +func (file *File) saveEntry(entry *filer_pb.Entry) error { return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - file.wfs.mapPbIdFromLocalToFiler(file.entry) - defer file.wfs.mapPbIdFromFilerToLocal(file.entry) + file.wfs.mapPbIdFromLocalToFiler(entry) + defer file.wfs.mapPbIdFromFilerToLocal(entry) request := &filer_pb.UpdateEntryRequest{ Directory: file.dir.FullPath(), - Entry: file.entry, + Entry: entry, Signatures: []int32{file.wfs.signature}, } From c057dd1f6413ae95264d63782a21692a2088919c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 30 Oct 2020 02:16:34 -0700 Subject: [PATCH 40/84] add back upload retries --- weed/operation/upload_content.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index a4148cb22..3e96b5909 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -95,7 +95,7 @@ func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, } func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { - for i := 0; i < 1; i++ { + for i := 0; i < 3; i++ { uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) if err == nil { return From 9cc6575ab51ab29df5d45975451d8ef330d2df3f Mon Sep 17 00:00:00 2001 From: Kenny Date: Fri, 30 Oct 2020 18:16:49 +0800 Subject: [PATCH 41/84] Update filer_copy.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改 weed filer.copy 上传目录时文件名变子目录的问题 --- weed/command/filer_copy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 2295faa8a..f2b1e963e 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -219,7 +219,7 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi fileCopyTaskChan <- FileCopyTask{ sourceLocation: fileOrDir, - destinationUrlPath: destPath+fi.Name(), + destinationUrlPath: destPath,//+fi.Name(), fileSize: fi.Size(), fileMode: fi.Mode(), uid: uid, From 7ecbb4b3c9435ad75fee29245c6165485dacb892 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 30 Oct 2020 10:46:31 -0700 Subject: [PATCH 42/84] clean up --- weed/command/filer_copy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index f2b1e963e..eacb44528 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -219,7 +219,7 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi fileCopyTaskChan <- FileCopyTask{ sourceLocation: fileOrDir, - destinationUrlPath: destPath,//+fi.Name(), + destinationUrlPath: destPath, fileSize: fi.Size(), fileMode: fi.Mode(), uid: uid, From eb32af25d43729d0430abb30d91b80dc718b4dcf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 30 Oct 2020 10:55:31 -0700 Subject: [PATCH 43/84] make space for the QR code --- weed/server/filer_ui/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go index f86dde5b1..3f0647119 100644 --- a/weed/server/filer_ui/templates.go +++ b/weed/server/filer_ui/templates.go @@ -25,7 +25,7 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`