gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

weed mount can work well

Browse source 
TODO: somehow filer url is returning empty content
This commit is contained in:
Chris Lu 2018-05-22 03:26:38 -07:00
parent 9dd228747c
commit 7362de9a18
10 changed files with 206 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
weed/filer2/embedded/embedded_store.go
View file

@ -26,7 +26,7 @@ func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int3
return nil return nil
} }
func (filer *EmbeddedStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { func (filer *EmbeddedStore) UpdateEntry(entry *filer2.Entry) (err error) {
return nil return nil
} }

15
weed/filer2/filechunks.go
View file

@ -37,6 +37,21 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file
return return
} }
func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) {
fileIds := make(map[string]bool)
for _, interval := range newChunks {
fileIds[interval.FileId] = true
}
for _, chunk := range oldChunks {
if found := fileIds[chunk.FileId]; !found {
unused = append(unused, chunk)
}
}
return
}
func logPrintf(name string, visibles []*visibleInterval) { func logPrintf(name string, visibles []*visibleInterval) {
// return // return

5
weed/filer2/filer.go
View file

@ -8,7 +8,6 @@ import (
"path/filepath" "path/filepath"
"time" "time"
"os" "os"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
) )
@ -113,8 +112,8 @@ func (f *Filer) CreateEntry(entry *Entry) (error) {
return nil return nil
} }
func (f *Filer) SetFileChunks(p FullPath, chunks []*filer_pb.FileChunk) (err error) { func (f *Filer) UpdateEntry(entry *Entry) (err error) {
return f.store.SetFileChunks(p, chunks) return f.store.UpdateEntry(entry)
} }
func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) {

2
weed/filer2/filer_structure.go
View file

@ -69,7 +69,7 @@ var ErrNotFound = errors.New("filer: no entry is found in filer store")
type FilerStore interface { type FilerStore interface {
InsertEntry(*Entry) (error) InsertEntry(*Entry) (error)
SetFileChunks(FullPath, []*filer_pb.FileChunk) (err error) UpdateEntry(*Entry) (err error)
FindEntry(FullPath) (found bool, entry *Entry, err error) FindEntry(FullPath) (found bool, entry *Entry, err error)
DeleteEntry(FullPath) (fileEntry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error)

9
weed/filer2/memdb/memdb_store.go
View file

@ -6,7 +6,6 @@ import (
"strings" "strings"
"fmt" "fmt"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
) )
type MemDbStore struct { type MemDbStore struct {
@ -29,16 +28,16 @@ func NewMemDbStore() (filer *MemDbStore) {
func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
// println("inserting", entry.FullPath) // println("inserting", entry.FullPath)
entry.Crtime = time.Now()
filer.tree.ReplaceOrInsert(Entry{entry}) filer.tree.ReplaceOrInsert(Entry{entry})
return nil return nil
} }
func (filer *MemDbStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { func (filer *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) {
found, entry, err := filer.FindEntry(fullpath) found, entry, err := filer.FindEntry(entry.FullPath)
if !found { if !found {
return fmt.Errorf("No such file: %s", fullpath) return fmt.Errorf("No such file: %s", entry.FullPath)
} }
entry.Chunks = fileChunks
entry.Mtime = time.Now() entry.Mtime = time.Now()
filer.tree.ReplaceOrInsert(Entry{entry}) filer.tree.ReplaceOrInsert(Entry{entry})
return nil return nil

14
weed/filesys/dir.go
View file

@ -76,6 +76,16 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
return nil return nil
} }
func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk) *File {
return &File{
Name: name,
dir: dir,
wfs: dir.wfs,
// attributes: &filer_pb.FuseAttributes{},
Chunks: chunks,
}
}
func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
@ -104,7 +114,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
}) })
if err == nil { if err == nil {
node := &File{Name: req.Name, dir: dir, wfs: dir.wfs} node := dir.newFile(req.Name, nil)
dir.NodeMap[req.Name] = node dir.NodeMap[req.Name] = node
return node, node, nil return node, node, nil
} }
@ -186,7 +196,7 @@ func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err erro
if entry.IsDirectory { if entry.IsDirectory {
node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs} node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs}
} else { } else {
node = &File{Chunks: entry.Chunks, Name: name, dir: dir, wfs: dir.wfs} node = dir.newFile(name, entry.Chunks)
} }
dir.NodeMap[name] = node dir.NodeMap[name] = node
return node, nil return node, nil

111
weed/filesys/file.go
View file

@ -17,7 +17,7 @@ import (
) )
var _ = fs.Node(&File{}) var _ = fs.Node(&File{})
// var _ = fs.NodeOpener(&File{}) var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{}) var _ = fs.NodeFsyncer(&File{})
var _ = fs.Handle(&File{}) var _ = fs.Handle(&File{})
var _ = fs.HandleReadAller(&File{}) var _ = fs.HandleReadAller(&File{})
@ -28,58 +28,63 @@ var _ = fs.HandleReleaser(&File{})
var _ = fs.NodeSetattrer(&File{}) var _ = fs.NodeSetattrer(&File{})
type File struct { type File struct {
Chunks []*filer_pb.FileChunk Chunks []*filer_pb.FileChunk
Name string Name string
dir *Dir dir *Dir
wfs *WFS wfs *WFS
isOpened bool
attributes *filer_pb.FuseAttributes
} }
func (file *File) Attr(context context.Context, attr *fuse.Attr) error { func (file *File) Attr(context context.Context, attr *fuse.Attr) error {
fullPath := filepath.Join(file.dir.Path, file.Name)
item := file.wfs.listDirectoryEntriesCache.Get(fullPath)
var attributes *filer_pb.FuseAttributes
if item != nil {
attributes = item.Value().(*filer_pb.FuseAttributes)
glog.V(1).Infof("read cached file %v attributes", file.Name)
} else {
err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.GetEntryAttributesRequest{ if !file.isOpened || file.attributes == nil {
Name: file.Name, fullPath := filepath.Join(file.dir.Path, file.Name)
ParentDir: file.dir.Path, item := file.wfs.listDirectoryEntriesCache.Get(fullPath)
} if item != nil {
file.attributes = item.Value().(*filer_pb.FuseAttributes)
glog.V(1).Infof("read cached file %v attributes", file.Name)
} else {
err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.GetEntryAttributesRequest{
Name: file.Name,
ParentDir: file.dir.Path,
}
glog.V(1).Infof("read file size: %v", request)
resp, err := client.GetEntryAttributes(context, request)
if err != nil {
glog.V(0).Infof("read file attributes %v: %v", request, err)
return err
}
file.attributes = resp.Attributes
return nil
})
glog.V(1).Infof("read file size: %v", request)
resp, err := client.GetEntryAttributes(context, request)
if err != nil { if err != nil {
glog.V(0).Infof("read file attributes %v: %v", request, err)
return err return err
} }
attributes = resp.Attributes
return nil
})
if err != nil {
return err
} }
} }
attr.Mode = os.FileMode(attributes.FileMode) attr.Mode = os.FileMode(file.attributes.FileMode)
attr.Size = attributes.FileSize attr.Size = filer2.TotalSize(file.Chunks)
attr.Mtime = time.Unix(attributes.Mtime, 0) attr.Mtime = time.Unix(file.attributes.Mtime, 0)
attr.Gid = attributes.Gid attr.Gid = file.attributes.Gid
attr.Uid = attributes.Uid attr.Uid = file.attributes.Uid
return nil return nil
} }
func (file *File) xOpen(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
fullPath := filepath.Join(file.dir.Path, file.Name) fullPath := filepath.Join(file.dir.Path, file.Name)
fmt.Printf("Open %v %+v\n", fullPath, req) fmt.Printf("Open %v %+v\n", fullPath, req)
file.isOpened = true
return file, nil return file, nil
@ -89,10 +94,28 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
fullPath := filepath.Join(file.dir.Path, file.Name) fullPath := filepath.Join(file.dir.Path, file.Name)
fmt.Printf("Setattr %v %+v\n", fullPath, req) fmt.Printf("Setattr %v %+v\n", fullPath, req)
if req.Valid.Size() && req.Size == 0 { if req.Valid.Size() {
fmt.Printf("truncate %v \n", fullPath)
file.Chunks = nil if req.Size == 0 {
resp.Attr.Size = 0 fmt.Printf("truncate %v \n", fullPath)
file.Chunks = nil
}
file.attributes.FileSize = req.Size
}
if req.Valid.Mode() {
file.attributes.FileMode = uint32(req.Mode)
}
if req.Valid.Uid() {
file.attributes.Uid = req.Uid
}
if req.Valid.Gid() {
file.attributes.Gid = req.Gid
}
if req.Valid.Mtime() {
file.attributes.Mtime = req.Mtime.Unix()
} }
return nil return nil
@ -152,16 +175,17 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error {
err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.SetFileChunksRequest{ request := &filer_pb.UpdateEntryRequest{
Directory: file.dir.Path, Directory: file.dir.Path,
Entry: &filer_pb.Entry{ Entry: &filer_pb.Entry{
Name: file.Name, Name: file.Name,
Chunks: file.Chunks, Attributes: file.attributes,
Chunks: file.Chunks,
}, },
} }
glog.V(1).Infof("append chunks: %v", request) glog.V(1).Infof("append chunks: %v", request)
if _, err := client.SetFileChunks(ctx, request); err != nil { if _, err := client.UpdateEntry(ctx, request); err != nil {
return fmt.Errorf("create file: %v", err) return fmt.Errorf("create file: %v", err)
} }
@ -224,7 +248,8 @@ func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.
func (file *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { func (file *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
// fmt.Printf("release file %+v\n", req) fmt.Printf("release file %+v\n", req)
file.isOpened = false
return nil return nil
} }

6
weed/pb/filer.proto
View file

@ -21,7 +21,7 @@ service SeaweedFiler {
rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
} }
rpc SetFileChunks (SetFileChunksRequest) returns (SetFileChunksResponse) { rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
} }
rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) { rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
@ -121,9 +121,9 @@ message AssignVolumeResponse {
int32 count = 4; int32 count = 4;
} }
message SetFileChunksRequest { message UpdateEntryRequest {
string directory = 1; string directory = 1;
Entry entry = 2; Entry entry = 2;
} }
message SetFileChunksResponse { message UpdateEntryResponse {
} }

154
weed/pb/filer_pb/filer.pb.go
View file

@ -26,8 +26,8 @@ It has these top-level messages:
DeleteEntryResponse DeleteEntryResponse
AssignVolumeRequest AssignVolumeRequest
AssignVolumeResponse AssignVolumeResponse
SetFileChunksRequest UpdateEntryRequest
SetFileChunksResponse UpdateEntryResponse
*/ */
package filer_pb package filer_pb
@ -475,37 +475,37 @@ func (m *AssignVolumeResponse) GetCount() int32 {
return 0 return 0
} }
type SetFileChunksRequest struct { type UpdateEntryRequest struct {
Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"`
} }
func (m *SetFileChunksRequest) Reset() { *m = SetFileChunksRequest{} } func (m *UpdateEntryRequest) Reset() { *m = UpdateEntryRequest{} }
func (m *SetFileChunksRequest) String() string { return proto.CompactTextString(m) } func (m *UpdateEntryRequest) String() string { return proto.CompactTextString(m) }
func (*SetFileChunksRequest) ProtoMessage() {} func (*UpdateEntryRequest) ProtoMessage() {}
func (*SetFileChunksRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
func (m *SetFileChunksRequest) GetDirectory() string { func (m *UpdateEntryRequest) GetDirectory() string {
if m != nil { if m != nil {
return m.Directory return m.Directory
} }
return "" return ""
} }
func (m *SetFileChunksRequest) GetEntry() *Entry { func (m *UpdateEntryRequest) GetEntry() *Entry {
if m != nil { if m != nil {
return m.Entry return m.Entry
} }
return nil return nil
} }
type SetFileChunksResponse struct { type UpdateEntryResponse struct {
} }
func (m *SetFileChunksResponse) Reset() { *m = SetFileChunksResponse{} } func (m *UpdateEntryResponse) Reset() { *m = UpdateEntryResponse{} }
func (m *SetFileChunksResponse) String() string { return proto.CompactTextString(m) } func (m *UpdateEntryResponse) String() string { return proto.CompactTextString(m) }
func (*SetFileChunksResponse) ProtoMessage() {} func (*UpdateEntryResponse) ProtoMessage() {}
func (*SetFileChunksResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
func init() { func init() {
proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest")
@ -525,8 +525,8 @@ func init() {
proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse") proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse")
proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest") proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest")
proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse") proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse")
proto.RegisterType((*SetFileChunksRequest)(nil), "filer_pb.SetFileChunksRequest") proto.RegisterType((*UpdateEntryRequest)(nil), "filer_pb.UpdateEntryRequest")
proto.RegisterType((*SetFileChunksResponse)(nil), "filer_pb.SetFileChunksResponse") proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse")
} }
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.
@ -545,7 +545,7 @@ type SeaweedFilerClient interface {
GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error) GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error)
GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error)
CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error)
SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error)
DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
} }
@ -603,9 +603,9 @@ func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryReq
return out, nil return out, nil
} }
func (c *seaweedFilerClient) SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) { func (c *seaweedFilerClient) UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) {
out := new(SetFileChunksResponse) out := new(UpdateEntryResponse)
err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/SetFileChunks", in, out, c.cc, opts...) err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/UpdateEntry", in, out, c.cc, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -638,7 +638,7 @@ type SeaweedFilerServer interface {
GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error) GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error)
GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error) GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error)
CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error)
SetFileChunks(context.Context, *SetFileChunksRequest) (*SetFileChunksResponse, error) UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error)
DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
} }
@ -737,20 +737,20 @@ func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _SeaweedFiler_SetFileChunks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { func _SeaweedFiler_UpdateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SetFileChunksRequest) in := new(UpdateEntryRequest)
if err := dec(in); err != nil { if err := dec(in); err != nil {
return nil, err return nil, err
} }
if interceptor == nil { if interceptor == nil {
return srv.(SeaweedFilerServer).SetFileChunks(ctx, in) return srv.(SeaweedFilerServer).UpdateEntry(ctx, in)
} }
info := &grpc.UnaryServerInfo{ info := &grpc.UnaryServerInfo{
Server: srv, Server: srv,
FullMethod: "/filer_pb.SeaweedFiler/SetFileChunks", FullMethod: "/filer_pb.SeaweedFiler/UpdateEntry",
} }
handler := func(ctx context.Context, req interface{}) (interface{}, error) { handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedFilerServer).SetFileChunks(ctx, req.(*SetFileChunksRequest)) return srv.(SeaweedFilerServer).UpdateEntry(ctx, req.(*UpdateEntryRequest))
} }
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
@ -816,8 +816,8 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
Handler: _SeaweedFiler_CreateEntry_Handler, Handler: _SeaweedFiler_CreateEntry_Handler,
}, },
{ {
MethodName: "SetFileChunks", MethodName: "UpdateEntry",
Handler: _SeaweedFiler_SetFileChunks_Handler, Handler: _SeaweedFiler_UpdateEntry_Handler,
}, },
{ {
MethodName: "DeleteEntry", MethodName: "DeleteEntry",
@ -835,53 +835,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 765 bytes of a gzipped FileDescriptorProto // 754 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x4e, 0xdb, 0x4a, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xd3, 0x4a,
0x10, 0xc6, 0x38, 0x09, 0x64, 0x12, 0x38, 0x47, 0x93, 0x00, 0x3e, 0xe1, 0x2f, 0xc7, 0x2d, 0x15, 0x10, 0xae, 0xeb, 0x24, 0x6d, 0x26, 0x69, 0xcf, 0xd1, 0x26, 0xed, 0xf1, 0x49, 0x7f, 0x08, 0x86,
0x55, 0x25, 0x54, 0xa5, 0x37, 0xbd, 0x2c, 0x02, 0x8a, 0x2a, 0x51, 0x21, 0x19, 0x81, 0x54, 0xf5, 0xa2, 0x22, 0xa4, 0x0a, 0x85, 0x1b, 0x2e, 0xa9, 0xda, 0x52, 0x21, 0x15, 0x55, 0x72, 0xd5, 0x4a,
0x22, 0x4a, 0xec, 0x49, 0xba, 0xc2, 0xb1, 0x53, 0x7b, 0xdd, 0x8a, 0x5e, 0xf7, 0x55, 0xfa, 0x1e, 0x5c, 0x45, 0x89, 0x3d, 0x09, 0xab, 0x3a, 0x76, 0xf0, 0xae, 0x41, 0xe5, 0x9a, 0x57, 0xe1, 0x25,
0x7d, 0xb4, 0x6a, 0xd7, 0x1b, 0x7b, 0x4d, 0x1c, 0x5a, 0xa4, 0xde, 0x79, 0xe7, 0xef, 0xfb, 0x76, 0x78, 0x3a, 0xb4, 0x3f, 0xb1, 0xd7, 0xd8, 0x29, 0xf4, 0x82, 0x3b, 0xef, 0xfc, 0x7c, 0xf3, 0xed,
0x66, 0xbe, 0x4d, 0xa0, 0x31, 0x62, 0x3e, 0x45, 0x47, 0xd3, 0x28, 0xe4, 0x21, 0xae, 0xca, 0x43, 0xcc, 0x7c, 0x9b, 0x40, 0x6b, 0x42, 0x43, 0x4c, 0x8e, 0xe6, 0x49, 0xcc, 0x63, 0xb2, 0x2e, 0x0f,
0x7f, 0x3a, 0xb4, 0x2f, 0x61, 0xfb, 0x22, 0x0c, 0x6f, 0x93, 0xe9, 0x29, 0x8b, 0xc8, 0xe5, 0x61, 0xc3, 0xf9, 0xd8, 0xbd, 0x84, 0x9d, 0x8b, 0x38, 0xbe, 0x4d, 0xe7, 0xa7, 0x34, 0x41, 0x9f, 0xc7,
0x74, 0x77, 0x16, 0xf0, 0xe8, 0xce, 0xa1, 0xcf, 0x09, 0xc5, 0x1c, 0x77, 0xa0, 0xee, 0xcd, 0x1c, 0xc9, 0xdd, 0x59, 0xc4, 0x93, 0x3b, 0x0f, 0x3f, 0xa5, 0xc8, 0x38, 0xd9, 0x85, 0x66, 0xb0, 0x70,
0x96, 0xd1, 0x35, 0x0e, 0xeb, 0x4e, 0x6e, 0x40, 0x84, 0x4a, 0x30, 0x98, 0x90, 0xb5, 0x2c, 0x1d, 0x38, 0x56, 0xdf, 0x3a, 0x6c, 0x7a, 0xb9, 0x81, 0x10, 0xa8, 0x45, 0xa3, 0x19, 0x3a, 0xab, 0xd2,
0xf2, 0xdb, 0x3e, 0x83, 0x9d, 0xf2, 0x82, 0xf1, 0x34, 0x0c, 0x62, 0xc2, 0x03, 0xa8, 0x92, 0x30, 0x21, 0xbf, 0xdd, 0x33, 0xd8, 0xad, 0x06, 0x64, 0xf3, 0x38, 0x62, 0x48, 0x0e, 0xa0, 0x8e, 0xc2,
0xc8, 0x6a, 0x8d, 0xde, 0x3f, 0x47, 0x33, 0x2a, 0x47, 0x69, 0x5c, 0xea, 0xb5, 0x7b, 0x80, 0x17, 0x20, 0xd1, 0x5a, 0x83, 0x7f, 0x8e, 0x16, 0x54, 0x8e, 0x54, 0x9c, 0xf2, 0xba, 0x03, 0x20, 0x17,
0x2c, 0xe6, 0xc2, 0xc6, 0x28, 0xfe, 0x23, 0x3a, 0xf6, 0x1b, 0x68, 0x15, 0x72, 0x14, 0xe2, 0x73, 0x94, 0x71, 0x61, 0xa3, 0xc8, 0xfe, 0x88, 0x8e, 0xfb, 0x06, 0x3a, 0x85, 0x1c, 0x5d, 0xf1, 0x39,
0x58, 0xa1, 0xd4, 0x64, 0x19, 0x5d, 0xb3, 0x0c, 0x73, 0xe6, 0xb7, 0x7f, 0x18, 0x50, 0x95, 0xa6, 0xac, 0xa1, 0x32, 0x39, 0x56, 0xdf, 0xae, 0xaa, 0xb9, 0xf0, 0xbb, 0xdf, 0x2d, 0xa8, 0x4b, 0x53,
0xec, 0x6a, 0x46, 0x7e, 0x35, 0xfc, 0x1f, 0x9a, 0x2c, 0xee, 0xe7, 0x04, 0xc4, 0xb5, 0x57, 0x9d, 0x76, 0x35, 0x2b, 0xbf, 0x1a, 0x79, 0x0c, 0x6d, 0xca, 0x86, 0x39, 0x01, 0x71, 0xed, 0x75, 0xaf,
0x06, 0x8b, 0xb3, 0xab, 0xe2, 0x0b, 0xa8, 0xb9, 0x9f, 0x92, 0xe0, 0x36, 0xb6, 0x4c, 0x09, 0xd5, 0x45, 0x59, 0x76, 0x55, 0xf2, 0x02, 0x1a, 0xfe, 0xc7, 0x34, 0xba, 0x65, 0x8e, 0x2d, 0x4b, 0x75,
0xca, 0xa1, 0xde, 0x32, 0x9f, 0x4e, 0x84, 0xcf, 0x51, 0x21, 0xf8, 0x1a, 0x60, 0xc0, 0x79, 0xc4, 0xf2, 0x52, 0x6f, 0x69, 0x88, 0x27, 0xc2, 0xe7, 0xe9, 0x10, 0xf2, 0x1a, 0x60, 0xc4, 0x79, 0x42,
0x86, 0x09, 0xa7, 0xd8, 0xaa, 0xc8, 0x7e, 0x58, 0x5a, 0x42, 0x12, 0xd3, 0x71, 0xe6, 0x77, 0xb4, 0xc7, 0x29, 0x47, 0xe6, 0xd4, 0x64, 0x3f, 0x1c, 0x23, 0x21, 0x65, 0x78, 0x9c, 0xf9, 0x3d, 0x23,
0x58, 0x7b, 0x04, 0xf5, 0xac, 0x1c, 0x6e, 0xc1, 0x8a, 0xc8, 0xe9, 0x33, 0x4f, 0xb1, 0xad, 0x89, 0xd6, 0x9d, 0x40, 0x33, 0x83, 0x23, 0xff, 0xc1, 0x9a, 0xc8, 0x19, 0xd2, 0x40, 0xb3, 0x6d, 0x88,
0xe3, 0x3b, 0x0f, 0x37, 0xa1, 0x16, 0x8e, 0x46, 0x31, 0x71, 0xc9, 0xd4, 0x74, 0xd4, 0x49, 0xdc, 0xe3, 0xbb, 0x80, 0x6c, 0x43, 0x23, 0x9e, 0x4c, 0x18, 0x72, 0xc9, 0xd4, 0xf6, 0xf4, 0x49, 0xdc,
0x2d, 0x66, 0xdf, 0xc8, 0x32, 0xbb, 0xc6, 0x61, 0xc5, 0x91, 0xdf, 0xd8, 0x86, 0xea, 0x84, 0xb3, 0x8d, 0xd1, 0xaf, 0xe8, 0xd8, 0x7d, 0xeb, 0xb0, 0xe6, 0xc9, 0x6f, 0xd2, 0x85, 0xfa, 0x8c, 0xd3,
0x09, 0x49, 0x1a, 0xa6, 0x93, 0x1e, 0xec, 0xef, 0x06, 0xac, 0x17, 0x69, 0xe0, 0x36, 0xd4, 0x25, 0x19, 0x4a, 0x1a, 0xb6, 0xa7, 0x0e, 0xee, 0x37, 0x0b, 0x36, 0x8b, 0x34, 0xc8, 0x0e, 0x34, 0x65,
0x9a, 0xac, 0x60, 0xc8, 0x0a, 0x72, 0x9b, 0xae, 0x0a, 0x55, 0x96, 0xb5, 0x2a, 0x59, 0xca, 0x24, 0x35, 0x89, 0x60, 0x49, 0x04, 0xb9, 0x4d, 0x57, 0x05, 0x94, 0x55, 0x03, 0x25, 0x4b, 0x99, 0xc5,
0xf4, 0x52, 0xd0, 0xb5, 0x34, 0xe5, 0x7d, 0xe8, 0x11, 0xfe, 0x0b, 0x66, 0xc2, 0x3c, 0x09, 0xbb, 0x81, 0x2a, 0xba, 0xa1, 0x52, 0xde, 0xc7, 0x01, 0x92, 0x7f, 0xc1, 0x4e, 0x69, 0x20, 0xcb, 0x6e,
0xe6, 0x88, 0x4f, 0x61, 0x19, 0x33, 0xcf, 0xaa, 0xa6, 0x96, 0x31, 0xf3, 0xec, 0x31, 0xfc, 0x77, 0x78, 0xe2, 0x53, 0x58, 0xa6, 0x34, 0x70, 0xea, 0xca, 0x32, 0xa5, 0x81, 0x3b, 0x85, 0xff, 0xcf,
0x4e, 0x72, 0xae, 0x77, 0x5a, 0x43, 0xd4, 0x4e, 0x94, 0x4d, 0x6a, 0x17, 0x60, 0x3a, 0x88, 0x28, 0x51, 0xce, 0xf5, 0xce, 0x68, 0x88, 0xde, 0x89, 0xaa, 0x49, 0xed, 0x01, 0xcc, 0x47, 0x09, 0x46,
0xe0, 0x62, 0x5a, 0x6a, 0x3d, 0xeb, 0xa9, 0xe5, 0x94, 0x45, 0x7a, 0xc7, 0x4c, 0xbd, 0x63, 0xf6, 0x5c, 0x4c, 0x4b, 0xaf, 0x67, 0x53, 0x59, 0x4e, 0x69, 0x62, 0x76, 0xcc, 0x36, 0x3b, 0xe6, 0xde,
0x0d, 0x74, 0xca, 0x80, 0xd4, 0x22, 0x15, 0xe7, 0x65, 0x3c, 0x62, 0x5e, 0x2f, 0x61, 0xe3, 0x9c, 0x40, 0xaf, 0xaa, 0x90, 0x5e, 0xa4, 0xe2, 0xbc, 0xac, 0x07, 0xcc, 0xeb, 0x25, 0x6c, 0x9d, 0x23,
0xb8, 0x1c, 0x59, 0x18, 0x70, 0x0a, 0xf8, 0x8c, 0xfc, 0xa2, 0xd9, 0xd9, 0x3d, 0xd8, 0xbc, 0x9f, 0x97, 0x23, 0x8b, 0x23, 0x8e, 0x11, 0x5f, 0x90, 0x5f, 0x36, 0x3b, 0x77, 0x00, 0xdb, 0xbf, 0x66,
0xa1, 0x58, 0x58, 0xb0, 0xe2, 0xa6, 0x26, 0x99, 0xd2, 0x74, 0x66, 0x47, 0xfb, 0x03, 0xe0, 0x49, 0x68, 0x16, 0x0e, 0xac, 0xf9, 0xca, 0x24, 0x53, 0xda, 0xde, 0xe2, 0xe8, 0x7e, 0x00, 0x72, 0x92,
0x44, 0x03, 0x4e, 0x8f, 0x90, 0x70, 0x26, 0xc7, 0xe5, 0x07, 0xe5, 0xb8, 0x01, 0xad, 0x42, 0xe9, 0xe0, 0x88, 0xe3, 0x03, 0x24, 0x9c, 0xc9, 0x71, 0xf5, 0x5e, 0x39, 0x6e, 0x41, 0xa7, 0x00, 0xad,
0x94, 0x8b, 0xcd, 0x00, 0x4f, 0xc9, 0xa7, 0x47, 0x21, 0x96, 0x3c, 0x1a, 0x73, 0xca, 0x32, 0xe7, 0xb8, 0xb8, 0x14, 0xc8, 0x29, 0x86, 0xf8, 0xa0, 0x8a, 0x15, 0x8f, 0x46, 0x49, 0x59, 0x76, 0x49,
0x94, 0x25, 0x18, 0x14, 0xa0, 0x14, 0x83, 0x09, 0xb4, 0x8e, 0xe3, 0x98, 0x8d, 0x83, 0x9b, 0xd0, 0x59, 0x82, 0x41, 0xa1, 0x94, 0x66, 0x30, 0x83, 0xce, 0x31, 0x63, 0x74, 0x1a, 0xdd, 0xc4, 0x61,
0x4f, 0x26, 0x34, 0xa3, 0xd0, 0x86, 0xaa, 0x1b, 0x26, 0xaa, 0x45, 0x55, 0x27, 0x3d, 0xe0, 0x1e, 0x3a, 0xc3, 0x05, 0x85, 0x2e, 0xd4, 0xfd, 0x38, 0xd5, 0x2d, 0xaa, 0x7b, 0xea, 0x40, 0xf6, 0x01,
0x80, 0x1b, 0xfa, 0x3e, 0xb9, 0x9c, 0x85, 0x81, 0x22, 0xa0, 0x59, 0xb0, 0x0b, 0x8d, 0x88, 0xa6, 0xfc, 0x38, 0x0c, 0xd1, 0xe7, 0x34, 0x8e, 0x34, 0x01, 0xc3, 0x42, 0xfa, 0xd0, 0x4a, 0x70, 0x1e,
0x3e, 0x73, 0x07, 0x32, 0x20, 0xdd, 0x0d, 0xdd, 0x64, 0x7f, 0x81, 0x76, 0x11, 0x4e, 0x0d, 0x65, 0x52, 0x7f, 0x24, 0x03, 0xd4, 0x6e, 0x98, 0x26, 0xf7, 0x33, 0x74, 0x8b, 0xe5, 0xf4, 0x50, 0x96,
0xa1, 0x06, 0xc5, 0x7a, 0x47, 0xbe, 0xc2, 0x12, 0x9f, 0x72, 0x37, 0x93, 0xa1, 0xcf, 0xdc, 0xbe, 0x6a, 0x50, 0xac, 0x77, 0x12, 0xea, 0x5a, 0xe2, 0x53, 0xee, 0x66, 0x3a, 0x0e, 0xa9, 0x3f, 0x14,
0x70, 0x98, 0x6a, 0x37, 0xa5, 0xe5, 0x3a, 0xf2, 0x73, 0xe6, 0x15, 0x8d, 0xb9, 0xfd, 0x11, 0xda, 0x0e, 0x5b, 0xef, 0xa6, 0xb4, 0x5c, 0x27, 0x61, 0xce, 0xbc, 0x66, 0x30, 0x17, 0xa3, 0xbd, 0x9e,
0x57, 0x6a, 0x1d, 0xe4, 0xdb, 0xf1, 0x57, 0x87, 0xbb, 0x05, 0x1b, 0xf7, 0x8a, 0xa7, 0xb7, 0xea, 0x07, 0x7f, 0x6b, 0xb4, 0x05, 0x68, 0x75, 0xa3, 0xc1, 0x8f, 0x3a, 0xb4, 0xaf, 0x70, 0xf4, 0x05,
0xfd, 0xac, 0x42, 0xf3, 0x8a, 0x06, 0x5f, 0x89, 0x3c, 0xe1, 0x8d, 0x70, 0x0c, 0xed, 0xb2, 0xc7, 0x31, 0x10, 0x5b, 0x98, 0x90, 0x29, 0x74, 0xab, 0x1e, 0x76, 0x72, 0x90, 0xe3, 0xde, 0xf3, 0x4b,
0x1d, 0x0f, 0xf2, 0xca, 0x0f, 0xfc, 0x9a, 0x74, 0x9e, 0xfd, 0x2e, 0x4c, 0x0d, 0x75, 0x09, 0x2f, 0xd2, 0x7b, 0xf6, 0xbb, 0x30, 0x3d, 0xd0, 0x15, 0x72, 0x01, 0x2d, 0xe3, 0x19, 0x27, 0xbb, 0x46,
0xa0, 0xa1, 0x3d, 0xe5, 0xb8, 0xa3, 0x25, 0xce, 0xfd, 0x2a, 0x74, 0x76, 0x17, 0x78, 0xb3, 0x6a, 0x62, 0xe9, 0x17, 0xa1, 0xb7, 0xb7, 0xc4, 0x9b, 0xa1, 0x8d, 0x80, 0x94, 0x25, 0x4d, 0x9e, 0xe4,
0x03, 0xc0, 0x79, 0x59, 0xe3, 0x93, 0x3c, 0x6d, 0xe1, 0xeb, 0xd2, 0x79, 0xfa, 0x70, 0x50, 0x06, 0x69, 0x4b, 0x5f, 0x96, 0xde, 0xd3, 0xfb, 0x83, 0xb2, 0x12, 0xd7, 0xb0, 0x59, 0xd4, 0x2a, 0x79,
0x71, 0x0d, 0xeb, 0x45, 0xbd, 0xe2, 0x7e, 0x21, 0x73, 0x5e, 0xfb, 0x9d, 0xee, 0xe2, 0x00, 0xbd, 0x54, 0xc8, 0x2c, 0xeb, 0xbe, 0xd7, 0x5f, 0x1e, 0x60, 0xf6, 0xc1, 0xd0, 0x9c, 0xd9, 0x87, 0xb2,
0x0f, 0x9a, 0xee, 0xf4, 0x3e, 0xcc, 0x2b, 0x5d, 0xef, 0x43, 0x99, 0x58, 0x97, 0xd0, 0x81, 0xb5, 0xca, 0xcd, 0x3e, 0x54, 0x09, 0x55, 0xa2, 0x19, 0x63, 0x36, 0xd1, 0xca, 0x8b, 0x65, 0xa2, 0x55,
0xc2, 0xa0, 0x71, 0x2f, 0xcf, 0x28, 0x5b, 0xaf, 0xce, 0xfe, 0x42, 0xbf, 0xce, 0x50, 0xd3, 0xa5, 0xec, 0x86, 0x42, 0x33, 0xd4, 0x68, 0xa2, 0x95, 0xdf, 0x03, 0x13, 0xad, 0x4a, 0xc2, 0x2b, 0xe4,
0xce, 0x70, 0xfe, 0x65, 0xd0, 0x19, 0x96, 0x89, 0x79, 0x09, 0x2f, 0xa1, 0xa9, 0xeb, 0x0b, 0xb5, 0x12, 0xda, 0xa6, 0xaa, 0x88, 0x91, 0x50, 0x21, 0xee, 0xde, 0xfe, 0x32, 0xf7, 0x02, 0x70, 0xdc,
0x84, 0x12, 0x99, 0x77, 0xf6, 0x16, 0xb9, 0x67, 0x05, 0x87, 0x35, 0xf9, 0x87, 0xe7, 0xd5, 0xaf, 0x90, 0x7f, 0x73, 0x5e, 0xfd, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x9c, 0xf1, 0x24, 0x43, 0xf5, 0x08,
0x00, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x51, 0xe7, 0x2b, 0xff, 0x08, 0x00, 0x00, 0x00, 0x00,
} }

34
weed/server/filer_grpc_server.go
View file

@ -120,34 +120,46 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
return &filer_pb.CreateEntryResponse{}, err return &filer_pb.CreateEntryResponse{}, err
} }
func (fs *FilerServer) SetFileChunks(ctx context.Context, req *filer_pb.SetFileChunksRequest) (*filer_pb.SetFileChunksResponse, error) { func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
fullpath := filepath.Join(req.Directory, req.Entry.Name) fullpath := filepath.Join(req.Directory, req.Entry.Name)
found, entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath)) found, entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath))
if err != nil { if err != nil {
return &filer_pb.SetFileChunksResponse{}, err return &filer_pb.UpdateEntryResponse{}, err
} }
if !found { if !found {
return &filer_pb.SetFileChunksResponse{}, fmt.Errorf("file not found: %s", fullpath) return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("file not found: %s", fullpath)
} }
chunks := append(entry.Chunks, req.Entry.Chunks...) // remove old chunks if not included in the new ones
unusedChunks := filer2.FindUnusedFileChunks(entry.Chunks, req.Entry.Chunks)
chunks, garbages := filer2.CompactFileChunks(chunks) chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
err = fs.filer.SetFileChunks( err = fs.filer.UpdateEntry(&filer2.Entry{
filer2.FullPath(fullpath), FullPath: filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)),
chunks, Attr: filer2.Attr{
) Mtime: time.Unix(req.Entry.Attributes.Mtime, 0),
Crtime: time.Unix(req.Entry.Attributes.Mtime, 0),
Mode: os.FileMode(req.Entry.Attributes.FileMode),
Uid: req.Entry.Attributes.Uid,
Gid: req.Entry.Attributes.Gid,
},
Chunks: chunks,
})
if err == nil { if err == nil {
for _, garbage := range garbages { for _, garbage := range unusedChunks {
glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size))
operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId))
} }
for _, garbage := range garbages {
glog.V(0).Infof("deleting %s garbage chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size))
operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId))
}
} }
return &filer_pb.SetFileChunksResponse{}, err return &filer_pb.UpdateEntryResponse{}, err
} }
func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) { func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {