gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Removing chunks on failed to write to replicas (#3591)

Browse source 
* Removing chunks on failed to write to replicas
https://github.com/seaweedfs/seaweedfs/issues/3578

* put with in the util.Retry

* just purge on any errors
This commit is contained in:
Konstantin Lebedev 2022-09-14 10:33:35 +05:00 committed by GitHub
parent 0baf3d38c2
commit f8ef25099c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
weed/server/filer_server_handlers_write_upload.go
View file

@ -107,7 +107,7 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
wg.Done() wg.Done()
}() }()
chunk, toChunkErr := fs.dataToChunk(fileName, contentType, bytesBuffer.Bytes(), offset, so) chunks, toChunkErr := fs.dataToChunk(fileName, contentType, bytesBuffer.Bytes(), offset, so)
if toChunkErr != nil { if toChunkErr != nil {
uploadErrLock.Lock() uploadErrLock.Lock()
if uploadErr == nil { if uploadErr == nil {
@ -115,13 +115,15 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
} }
uploadErrLock.Unlock() uploadErrLock.Unlock()
} }
if chunk != nil { if chunks != nil {
fileChunksLock.Lock() fileChunksLock.Lock()
fileChunksSize := len(fileChunks) + len(chunks)
for _, chunk := range chunks {
fileChunks = append(fileChunks, chunk) fileChunks = append(fileChunks, chunk)
fileChunksSize := len(fileChunks)
fileChunksLock.Unlock()
glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, fileChunksSize, chunk.FileId, offset, offset+int64(chunk.Size)) glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, fileChunksSize, chunk.FileId, offset, offset+int64(chunk.Size))
} }
fileChunksLock.Unlock()
}
}(chunkOffset) }(chunkOffset)
// reset variables for the next chunk // reset variables for the next chunk
@ -169,7 +171,7 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil
return uploadResult, err, data return uploadResult, err, data
} }
func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, chunkOffset int64, so *operation.StorageOption) (*filer_pb.FileChunk, error) { func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, chunkOffset int64, so *operation.StorageOption) ([]*filer_pb.FileChunk, error) {
dataReader := util.NewBytesReader(data) dataReader := util.NewBytesReader(data)
// retry to assign a different file id // retry to assign a different file id
@ -177,6 +179,7 @@ func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, ch
var auth security.EncodedJwt var auth security.EncodedJwt
var uploadErr error var uploadErr error
var uploadResult *operation.UploadResult var uploadResult *operation.UploadResult
var failedFileChunks []*filer_pb.FileChunk
err := util.Retry("filerDataToChunk", func() error { err := util.Retry("filerDataToChunk", func() error {
// assign one file id for one chunk // assign one file id for one chunk
@ -191,19 +194,25 @@ func (fs *FilerServer) dataToChunk(fileName, contentType string, data []byte, ch
if uploadErr != nil { if uploadErr != nil {
glog.V(4).Infof("retry later due to upload error: %v", uploadErr) glog.V(4).Infof("retry later due to upload error: %v", uploadErr)
stats.FilerRequestCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc() stats.FilerRequestCounter.WithLabelValues(stats.ChunkDoUploadRetry).Inc()
fid, _ := filer_pb.ToFileIdObject(fileId)
fileChunk := filer_pb.FileChunk{
FileId: fileId,
Offset: chunkOffset,
Fid: fid,
}
failedFileChunks = append(failedFileChunks, &fileChunk)
return uploadErr return uploadErr
} }
return nil return nil
}) })
if err != nil { if err != nil {
glog.Errorf("upload error: %v", err) glog.Errorf("upload error: %v", err)
return nil, err return failedFileChunks, err
} }
// if last chunk exhausted the reader exactly at the border // if last chunk exhausted the reader exactly at the border
if uploadResult.Size == 0 { if uploadResult.Size == 0 {
return nil, nil return nil, nil
} }
return []*filer_pb.FileChunk{uploadResult.ToPbFileChunk(fileId, chunkOffset)}, nil
return uploadResult.ToPbFileChunk(fileId, chunkOffset), nil
} }

2
weed/topology/store_replicate.go
View file

@ -197,7 +197,7 @@ func GetWritableRemoteReplications(s *storage.Store, grpcDialOption grpc.DialOpt
} }
} }
} else { } else {
err = fmt.Errorf("failed to lookup for %d: %v", volumeId, lookupErr) err = fmt.Errorf("replicating lookup failed for %d: %v", volumeId, lookupErr)
return return
} }