gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

adjust logs

Browse source
This commit is contained in:
Chris Lu 2020-08-15 19:55:28 -07:00
parent 6ee8d952d2
commit 003d48da21
13 changed files with 33 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
weed/filer2/reader_at.go
View file

@ -112,12 +112,12 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {
func (c *ChunkReadAt) fetchChunkData(chunkView *ChunkView) (data []byte, err error) { func (c *ChunkReadAt) fetchChunkData(chunkView *ChunkView) (data []byte, err error) {
glog.V(4).Infof("fetchChunkData %s [%d,%d)\n", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size)) glog.V(5).Infof("fetchChunkData %s [%d,%d)\n", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
hasDataInCache := false hasDataInCache := false
chunkData := c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize) chunkData := c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize)
if chunkData != nil { if chunkData != nil {
glog.V(4).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size)) glog.V(5).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
hasDataInCache = true hasDataInCache = true
} else { } else {
chunkData, err = c.doFetchFullChunkData(chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped) chunkData, err = c.doFetchFullChunkData(chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped)

4
weed/filesys/dir.go
View file

@ -240,7 +240,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
return nil, fuse.ENOENT return nil, fuse.ENOENT
} }
} else { } else {
glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath) glog.V(5).Infof("dir Lookup cache hit %s", fullFilePath)
} }
if entry != nil { if entry != nil {
@ -268,7 +268,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
glog.V(4).Infof("dir ReadDirAll %s", dir.FullPath()) glog.V(5).Infof("dir ReadDirAll %s", dir.FullPath())
processEachEntryFn := func(entry *filer_pb.Entry, isLast bool) error { processEachEntryFn := func(entry *filer_pb.Entry, isLast bool) error {
fullpath := util.NewFullPath(dir.FullPath(), entry.Name) fullpath := util.NewFullPath(dir.FullPath(), entry.Name)

4
weed/filesys/dirty_page.go
View file

@ -35,7 +35,7 @@ func (pages *ContinuousDirtyPages) AddPage(offset int64, data []byte) (chunks []
pages.lock.Lock() pages.lock.Lock()
defer pages.lock.Unlock() defer pages.lock.Unlock()
glog.V(4).Infof("%s AddPage [%d,%d) of %d bytes", pages.f.fullpath(), offset, offset+int64(len(data)), pages.f.entry.Attributes.FileSize) glog.V(5).Infof("%s AddPage [%d,%d) of %d bytes", pages.f.fullpath(), offset, offset+int64(len(data)), pages.f.entry.Attributes.FileSize)
if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) { if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) {
// this is more than what buffer can hold. // this is more than what buffer can hold.
@ -127,7 +127,7 @@ func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (chunk *fi
chunk, err = pages.saveToStorage(maxList.ToReader(), maxList.Offset(), chunkSize) chunk, err = pages.saveToStorage(maxList.ToReader(), maxList.Offset(), chunkSize)
if err == nil { if err == nil {
hasSavedData = true hasSavedData = true
glog.V(4).Infof("%s saveToStorage %s [%d,%d) of %d bytes", pages.f.fullpath(), chunk.FileId, maxList.Offset(), maxList.Offset()+chunkSize, fileSize) glog.V(4).Infof("saveToStorage %s %s [%d,%d) of %d bytes", pages.f.fullpath(), chunk.GetFileIdString(), maxList.Offset(), maxList.Offset()+chunkSize, fileSize)
return return
} else { } else {
glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+chunkSize, err) glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+chunkSize, err)

10
weed/filesys/file.go
View file

@ -85,7 +85,7 @@ func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp
func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
glog.V(4).Infof("file %v open %+v", file.fullpath(), req) glog.V(5).Infof("file %v open %+v", file.fullpath(), req)
file.isOpen++ file.isOpen++
@ -93,7 +93,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op
resp.Handle = fuse.HandleID(handle.handle) resp.Handle = fuse.HandleID(handle.handle)
glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle) glog.V(5).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)
return handle, nil return handle, nil
@ -101,7 +101,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op
func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
glog.V(4).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes) glog.V(5).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
if err := file.maybeLoadEntry(ctx); err != nil { if err := file.maybeLoadEntry(ctx); err != nil {
return err return err
@ -121,10 +121,10 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
int64Size = int64(req.Size) - chunk.Offset int64Size = int64(req.Size) - chunk.Offset
if int64Size > 0 { if int64Size > 0 {
chunks = append(chunks, chunk) chunks = append(chunks, chunk)
glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk, chunk.Size, int64Size) glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
chunk.Size = uint64(int64Size) chunk.Size = uint64(int64Size)
} else { } else {
glog.V(4).Infof("truncated whole chunk %+v\n", chunk) glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
truncatedChunks = append(truncatedChunks, chunk) truncatedChunks = append(truncatedChunks, chunk)
} }
} }

12
weed/filesys/filehandle.go
View file

@ -54,7 +54,7 @@ var _ = fs.HandleReleaser(&FileHandle{})
func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
glog.V(2).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size)) glog.V(5).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size))
buff := make([]byte, req.Size) buff := make([]byte, req.Size)
@ -125,7 +125,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
copy(data, req.Data) copy(data, req.Data)
fh.f.entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(fh.f.entry.Attributes.FileSize))) fh.f.entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(fh.f.entry.Attributes.FileSize)))
glog.V(2).Infof("%v write [%d,%d)", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data))) glog.V(5).Infof("%v write [%d,%d)", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)))
chunks, err := fh.dirtyPages.AddPage(req.Offset, data) chunks, err := fh.dirtyPages.AddPage(req.Offset, data)
if err != nil { if err != nil {
@ -153,7 +153,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
glog.V(4).Infof("%v release fh %d", fh.f.fullpath(), fh.handle) glog.V(4).Infof("Release %v fh %d", fh.f.fullpath(), fh.handle)
fh.f.isOpen-- fh.f.isOpen--
@ -170,7 +170,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
// fflush works at fh level // fflush works at fh level
// send the data to the OS // send the data to the OS
glog.V(4).Infof("%s fh %d flush %v", fh.f.fullpath(), fh.handle, req) glog.V(5).Infof("Flush %s fh %d %v", fh.f.fullpath(), fh.handle, req)
chunks, err := fh.dirtyPages.FlushToStorage() chunks, err := fh.dirtyPages.FlushToStorage()
if err != nil { if err != nil {
@ -213,7 +213,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
glog.V(4).Infof("%s set chunks: %v", fh.f.fullpath(), len(fh.f.entry.Chunks)) glog.V(4).Infof("%s set chunks: %v", fh.f.fullpath(), len(fh.f.entry.Chunks))
for i, chunk := range fh.f.entry.Chunks { for i, chunk := range fh.f.entry.Chunks {
glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
} }
chunks, garbages := filer2.CompactFileChunks(filer2.LookupFn(fh.f.wfs), fh.f.entry.Chunks) chunks, garbages := filer2.CompactFileChunks(filer2.LookupFn(fh.f.wfs), fh.f.entry.Chunks)
@ -238,7 +238,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
fh.f.wfs.deleteFileChunks(garbages) fh.f.wfs.deleteFileChunks(garbages)
for i, chunk := range garbages { for i, chunk := range garbages {
glog.V(4).Infof("garbage %s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) glog.V(4).Infof("garbage %s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
} }
return nil return nil

2
weed/filesys/meta_cache/meta_cache_init.go
View file

@ -14,7 +14,7 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full
mc.visitedBoundary.EnsureVisited(dirPath, func(path util.FullPath) (childDirectories []string, err error) { mc.visitedBoundary.EnsureVisited(dirPath, func(path util.FullPath) (childDirectories []string, err error) {
glog.V(4).Infof("ReadDirAllEntries %s ...", path) glog.V(5).Infof("ReadDirAllEntries %s ...", path)
err = filer_pb.ReadDirAllEntries(client, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error { err = filer_pb.ReadDirAllEntries(client, dirPath, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
entry := filer2.FromPbEntry(string(dirPath), pbEntry) entry := filer2.FromPbEntry(string(dirPath), pbEntry)

11
weed/filesys/wfs.go
View file

@ -113,7 +113,7 @@ func (wfs *WFS) Root() (fs.Node, error) {
func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) { func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {
fullpath := file.fullpath() fullpath := file.fullpath()
glog.V(4).Infof("%s AcquireHandle uid=%d gid=%d", fullpath, uid, gid) glog.V(4).Infof("AcquireHandle %s uid=%d gid=%d", fullpath, uid, gid)
wfs.handlesLock.Lock() wfs.handlesLock.Lock()
defer wfs.handlesLock.Unlock() defer wfs.handlesLock.Unlock()
@ -127,7 +127,6 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
fileHandle = newFileHandle(file, uid, gid) fileHandle = newFileHandle(file, uid, gid)
wfs.handles[inodeId] = fileHandle wfs.handles[inodeId] = fileHandle
fileHandle.handle = inodeId fileHandle.handle = inodeId
glog.V(4).Infof("%s new fh %d", fullpath, fileHandle.handle)
return return
} }
@ -136,7 +135,7 @@ func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) {
wfs.handlesLock.Lock() wfs.handlesLock.Lock()
defer wfs.handlesLock.Unlock() defer wfs.handlesLock.Unlock()
glog.V(4).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles)) glog.V(5).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles))
delete(wfs.handles, fullpath.AsInode()) delete(wfs.handles, fullpath.AsInode())
@ -146,7 +145,7 @@ func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) {
// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser // Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error { func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {
glog.V(4).Infof("reading fs stats: %+v", req) glog.V(5).Infof("reading fs stats: %+v", req)
if wfs.stats.lastChecked < time.Now().Unix()-20 { if wfs.stats.lastChecked < time.Now().Unix()-20 {
@ -158,13 +157,13 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.
Ttl: fmt.Sprintf("%ds", wfs.option.TtlSec), Ttl: fmt.Sprintf("%ds", wfs.option.TtlSec),
} }
glog.V(4).Infof("reading filer stats: %+v", request) glog.V(5).Infof("reading filer stats: %+v", request)
resp, err := client.Statistics(context.Background(), request) resp, err := client.Statistics(context.Background(), request)
if err != nil { if err != nil {
glog.V(0).Infof("reading filer stats %v: %v", request, err) glog.V(0).Infof("reading filer stats %v: %v", request, err)
return err return err
} }
glog.V(4).Infof("read filer stats: %+v", resp) glog.V(5).Infof("read filer stats: %+v", resp)
wfs.stats.TotalSize = resp.TotalSize wfs.stats.TotalSize = resp.TotalSize
wfs.stats.UsedSize = resp.UsedSize wfs.stats.UsedSize = resp.UsedSize

2
weed/filesys/wfs_deletion.go
View file

@ -38,7 +38,7 @@ func (wfs *WFS) deleteFileIds(grpcDialOption grpc.DialOption, client filer_pb.Se
m := make(map[string]operation.LookupResult) m := make(map[string]operation.LookupResult)
glog.V(4).Infof("remove file lookup volume id locations: %v", vids) glog.V(5).Infof("deleteFileIds lookup volume id locations: %v", vids)
resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{ resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
VolumeIds: vids, VolumeIds: vids,
}) })

2
weed/operation/upload_content.go
View file

@ -33,6 +33,7 @@ type UploadResult struct {
} }
func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk { func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *filer_pb.FileChunk {
fid, _ := filer_pb.ToFileIdObject(fileId)
return &filer_pb.FileChunk{ return &filer_pb.FileChunk{
FileId: fileId, FileId: fileId,
Offset: offset, Offset: offset,
@ -41,6 +42,7 @@ func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64) *fi
ETag: uploadResult.ETag, ETag: uploadResult.ETag,
CipherKey: uploadResult.CipherKey, CipherKey: uploadResult.CipherKey,
IsCompressed: uploadResult.Gzip > 0, IsCompressed: uploadResult.Gzip > 0,
Fid: fid,
} }
} }

2
weed/pb/filer_pb/filer_client.go
View file

@ -83,7 +83,7 @@ func doList(filerClient FilerClient, fullDirPath util.FullPath, prefix string, f
InclusiveStartFrom: inclusive, InclusiveStartFrom: inclusive,
} }
glog.V(4).Infof("read directory: %v", request) glog.V(5).Infof("read directory: %v", request)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
stream, err := client.ListEntries(ctx, request) stream, err := client.ListEntries(ctx, request)
if err != nil { if err != nil {

6
weed/pb/filer_pb/filer_pb_helper.go
View file

@ -10,7 +10,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/needle"
) )
func toFileIdObject(fileIdStr string) (*FileId, error) { func ToFileIdObject(fileIdStr string) (*FileId, error) {
t, err := needle.ParseFileIdFromString(fileIdStr) t, err := needle.ParseFileIdFromString(fileIdStr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -43,14 +43,14 @@ func BeforeEntrySerialization(chunks []*FileChunk) {
for _, chunk := range chunks { for _, chunk := range chunks {
if chunk.FileId != "" { if chunk.FileId != "" {
if fid, err := toFileIdObject(chunk.FileId); err == nil { if fid, err := ToFileIdObject(chunk.FileId); err == nil {
chunk.Fid = fid chunk.Fid = fid
chunk.FileId = "" chunk.FileId = ""
} }
} }
if chunk.SourceFileId != "" { if chunk.SourceFileId != "" {
if fid, err := toFileIdObject(chunk.SourceFileId); err == nil { if fid, err := ToFileIdObject(chunk.SourceFileId); err == nil {
chunk.SourceFid = fid chunk.SourceFid = fid
chunk.SourceFileId = "" chunk.SourceFileId = ""
} }

2
weed/pb/filer_pb/filer_pb_helper_test.go
View file

@ -9,7 +9,7 @@ import (
func TestFileIdSize(t *testing.T) { func TestFileIdSize(t *testing.T) {
fileIdStr := "11745,0293434534cbb9892b" fileIdStr := "11745,0293434534cbb9892b"
fid, _ := toFileIdObject(fileIdStr) fid, _ := ToFileIdObject(fileIdStr)
bytes, _ := proto.Marshal(fid) bytes, _ := proto.Marshal(fid)
println(len(fileIdStr)) println(len(fileIdStr))

2
weed/util/chunk_cache/chunk_cache.go
View file

@ -89,7 +89,7 @@ func (c *ChunkCache) SetChunk(fileId string, data []byte) {
c.Lock() c.Lock()
defer c.Unlock() defer c.Unlock()
glog.V(4).Infof("SetChunk %s size %d\n", fileId, len(data)) glog.V(5).Infof("SetChunk %s size %d\n", fileId, len(data))
c.doSetChunk(fileId, data) c.doSetChunk(fileId, data)
} }