seaweedfs/weed/server/filer_grpc_server.go

344 lines
10 KiB
Go
Raw Normal View History

2018-05-08 08:59:43 +00:00
package weed_server
import (
"context"
2018-05-27 18:52:26 +00:00
"fmt"
2018-05-27 06:59:56 +00:00
"os"
"path/filepath"
"strconv"
"strings"
2018-05-27 18:52:26 +00:00
"time"
2018-05-27 06:59:56 +00:00
2018-05-14 06:56:16 +00:00
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
2018-05-27 18:52:26 +00:00
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
2018-05-08 08:59:43 +00:00
)
2018-05-10 06:18:02 +00:00
func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
2018-05-08 08:59:43 +00:00
2019-03-15 22:55:34 +00:00
entry, err := fs.filer.FindEntry(ctx, filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Name))))
2020-03-08 01:01:39 +00:00
if err == filer_pb.ErrNotFound {
2020-01-25 02:07:34 +00:00
return &filer_pb.LookupDirectoryEntryResponse{}, nil
}
2018-05-08 08:59:43 +00:00
if err != nil {
2020-01-25 01:26:03 +00:00
glog.V(3).Infof("LookupDirectoryEntry %s: %+v, ", filepath.Join(req.Directory, req.Name), err)
return nil, err
2018-05-08 08:59:43 +00:00
}
2018-05-10 06:18:02 +00:00
return &filer_pb.LookupDirectoryEntryResponse{
Entry: &filer_pb.Entry{
2018-05-08 08:59:43 +00:00
Name: req.Name,
2018-05-14 06:56:16 +00:00
IsDirectory: entry.IsDirectory(),
Attributes: filer2.EntryAttributeToPb(entry),
Chunks: entry.Chunks,
2019-12-18 05:39:48 +00:00
Extended: entry.Extended,
2018-05-08 08:59:43 +00:00
},
}, nil
}
2019-12-13 08:22:37 +00:00
func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
2018-05-08 08:59:43 +00:00
2018-07-22 08:14:36 +00:00
limit := int(req.Limit)
if limit == 0 {
limit = fs.option.DirListingLimit
2018-05-08 08:59:43 +00:00
}
paginationLimit := filer2.PaginationSize
if limit < paginationLimit {
paginationLimit = limit
}
2018-07-22 08:14:36 +00:00
lastFileName := req.StartFromFileName
includeLastFile := req.InclusiveStartFrom
for limit > 0 {
entries, err := fs.filer.ListDirectoryEntries(stream.Context(), filer2.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit)
2018-07-22 08:14:36 +00:00
if err != nil {
2019-12-13 08:22:37 +00:00
return err
2018-07-22 08:14:36 +00:00
}
if len(entries) == 0 {
2019-12-13 08:22:37 +00:00
return nil
2018-07-22 08:14:36 +00:00
}
includeLastFile = false
for _, entry := range entries {
lastFileName = entry.Name()
if req.Prefix != "" {
if !strings.HasPrefix(entry.Name(), req.Prefix) {
continue
}
}
2019-12-13 08:22:37 +00:00
if err := stream.Send(&filer_pb.ListEntriesResponse{
Entry: &filer_pb.Entry{
Name: entry.Name(),
IsDirectory: entry.IsDirectory(),
Chunks: entry.Chunks,
Attributes: filer2.EntryAttributeToPb(entry),
2019-12-18 05:39:48 +00:00
Extended: entry.Extended,
2019-12-13 08:22:37 +00:00
},
}); err != nil {
return err
}
2018-07-22 08:14:36 +00:00
limit--
if limit == 0 {
2019-12-13 08:22:37 +00:00
return nil
}
2018-07-22 08:14:36 +00:00
}
if len(entries) < paginationLimit {
break
}
2018-05-08 08:59:43 +00:00
}
2019-12-13 08:22:37 +00:00
return nil
2018-05-08 08:59:43 +00:00
}
func (fs *FilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
2018-05-08 08:59:43 +00:00
resp := &filer_pb.LookupVolumeResponse{
LocationsMap: make(map[string]*filer_pb.Locations),
2018-05-08 08:59:43 +00:00
}
for _, vidString := range req.VolumeIds {
vid, err := strconv.Atoi(vidString)
if err != nil {
2018-07-29 09:25:03 +00:00
glog.V(1).Infof("Unknown volume id %d", vid)
return nil, err
}
var locs []*filer_pb.Location
locations, found := fs.filer.MasterClient.GetLocations(uint32(vid))
if !found {
continue
}
for _, loc := range locations {
locs = append(locs, &filer_pb.Location{
Url: loc.Url,
PublicUrl: loc.PublicUrl,
})
}
resp.LocationsMap[vidString] = &filer_pb.Locations{
Locations: locs,
}
}
return resp, nil
2018-05-08 08:59:43 +00:00
}
func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (resp *filer_pb.CreateEntryResponse, err error) {
2018-09-23 05:11:37 +00:00
resp = &filer_pb.CreateEntryResponse{}
fullpath := filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Entry.Name)))
2018-09-23 05:11:37 +00:00
chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
2018-12-03 06:57:59 +00:00
if req.Entry.Attributes == nil {
2020-01-25 01:55:39 +00:00
glog.V(3).Infof("CreateEntry %s: nil attributes", filepath.Join(req.Directory, req.Entry.Name))
resp.Error = fmt.Sprintf("can not create entry with empty attributes")
return
2018-12-03 06:57:59 +00:00
}
createErr := fs.filer.CreateEntry(ctx, &filer2.Entry{
2018-09-23 05:11:37 +00:00
FullPath: fullpath,
Attr: filer2.PbToEntryAttribute(req.Entry.Attributes),
2018-09-23 05:11:37 +00:00
Chunks: chunks,
}, req.OExcl)
if createErr == nil {
fs.filer.DeleteChunks(garbages)
2020-01-25 01:55:39 +00:00
} else {
glog.V(3).Infof("CreateEntry %s: %v", filepath.Join(req.Directory, req.Entry.Name), createErr)
resp.Error = createErr.Error()
}
return
}
func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
fullpath := filepath.ToSlash(filepath.Join(req.Directory, req.Entry.Name))
2019-03-15 22:55:34 +00:00
entry, err := fs.filer.FindEntry(ctx, filer2.FullPath(fullpath))
if err != nil {
return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err)
}
// remove old chunks if not included in the new ones
2019-06-23 03:04:56 +00:00
unusedChunks := filer2.MinusChunks(entry.Chunks, req.Entry.Chunks)
chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
2018-05-23 10:08:46 +00:00
newEntry := &filer2.Entry{
FullPath: filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Entry.Name))),
2018-05-23 10:08:46 +00:00
Attr: entry.Attr,
2019-12-18 05:39:48 +00:00
Extended: req.Entry.Extended,
2018-05-23 10:08:46 +00:00
Chunks: chunks,
}
2019-12-18 05:39:48 +00:00
glog.V(3).Infof("updating %s: %+v, chunks %d: %v => %+v, chunks %d: %v, extended: %v => %v",
2018-05-23 10:08:46 +00:00
fullpath, entry.Attr, len(entry.Chunks), entry.Chunks,
2019-12-18 05:39:48 +00:00
req.Entry.Attributes, len(req.Entry.Chunks), req.Entry.Chunks,
entry.Extended, req.Entry.Extended)
2018-05-23 10:08:46 +00:00
if req.Entry.Attributes != nil {
if req.Entry.Attributes.Mtime != 0 {
newEntry.Attr.Mtime = time.Unix(req.Entry.Attributes.Mtime, 0)
}
if req.Entry.Attributes.FileMode != 0 {
newEntry.Attr.Mode = os.FileMode(req.Entry.Attributes.FileMode)
}
newEntry.Attr.Uid = req.Entry.Attributes.Uid
newEntry.Attr.Gid = req.Entry.Attributes.Gid
newEntry.Attr.Mime = req.Entry.Attributes.Mime
newEntry.Attr.UserName = req.Entry.Attributes.UserName
newEntry.Attr.GroupNames = req.Entry.Attributes.GroupName
2018-05-23 10:08:46 +00:00
}
2018-08-13 08:20:49 +00:00
if filer2.EqualEntry(entry, newEntry) {
return &filer_pb.UpdateEntryResponse{}, err
}
2019-03-15 22:55:34 +00:00
if err = fs.filer.UpdateEntry(ctx, entry, newEntry); err == nil {
fs.filer.DeleteChunks(unusedChunks)
fs.filer.DeleteChunks(garbages)
2020-01-25 01:55:39 +00:00
} else {
glog.V(3).Infof("UpdateEntry %s: %v", filepath.Join(req.Directory, req.Entry.Name), err)
}
2018-09-17 07:27:56 +00:00
fs.filer.NotifyUpdateEvent(entry, newEntry, true)
2018-08-13 08:20:49 +00:00
return &filer_pb.UpdateEntryResponse{}, err
}
2018-05-10 06:18:02 +00:00
func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {
err = fs.filer.DeleteEntryMetaAndData(ctx, filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Name))), req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData)
2020-02-25 22:38:36 +00:00
resp = &filer_pb.DeleteEntryResponse{}
if err != nil {
resp.Error = err.Error()
}
return resp, nil
}
func (fs *FilerServer) StreamDeleteEntries(stream filer_pb.SeaweedFiler_StreamDeleteEntriesServer) error {
for {
req, err := stream.Recv()
if err != nil {
return fmt.Errorf("receive delete entry request: %v", err)
}
fullpath := filer2.FullPath(filepath.ToSlash(filepath.Join(req.Directory, req.Name)))
2020-02-26 06:23:59 +00:00
err = fs.filer.DeleteEntryMetaAndData(context.Background(), fullpath, req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData)
2020-02-25 22:38:36 +00:00
resp := &filer_pb.DeleteEntryResponse{}
if err != nil {
resp.Error = err.Error()
}
if err := stream.Send(resp); err != nil {
return err
}
}
return nil
2018-05-08 08:59:43 +00:00
}
func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) {
2018-06-12 06:13:33 +00:00
ttlStr := ""
if req.TtlSec > 0 {
ttlStr = strconv.Itoa(int(req.TtlSec))
}
collection, replication := fs.detectCollection(req.ParentPath, req.Collection, req.Replication)
2018-06-12 06:13:33 +00:00
2018-07-10 07:20:50 +00:00
var altRequest *operation.VolumeAssignRequest
dataCenter := req.DataCenter
2018-07-14 21:09:45 +00:00
if dataCenter == "" {
dataCenter = fs.option.DataCenter
}
assignRequest := &operation.VolumeAssignRequest{
Count: uint64(req.Count),
Replication: replication,
Collection: collection,
2018-06-12 06:13:33 +00:00
Ttl: ttlStr,
DataCenter: dataCenter,
}
if dataCenter != "" {
2018-07-10 07:20:50 +00:00
altRequest = &operation.VolumeAssignRequest{
Count: uint64(req.Count),
Replication: replication,
Collection: collection,
Ttl: ttlStr,
DataCenter: "",
}
}
2019-02-18 20:11:52 +00:00
assignResult, err := operation.Assign(fs.filer.GetMaster(), fs.grpcDialOption, assignRequest, altRequest)
if err != nil {
2020-01-25 02:07:34 +00:00
glog.V(3).Infof("AssignVolume: %v", err)
return &filer_pb.AssignVolumeResponse{Error: fmt.Sprintf("assign volume: %v", err)}, nil
}
if assignResult.Error != "" {
2020-01-25 02:07:34 +00:00
glog.V(3).Infof("AssignVolume error: %v", assignResult.Error)
return &filer_pb.AssignVolumeResponse{Error: fmt.Sprintf("assign volume result: %v", assignResult.Error)}, nil
}
return &filer_pb.AssignVolumeResponse{
FileId: assignResult.Fid,
Count: int32(assignResult.Count),
Url: assignResult.Url,
PublicUrl: assignResult.PublicUrl,
Auth: string(assignResult.Auth),
Collection: collection,
Replication: replication,
}, nil
}
func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.DeleteCollectionRequest) (resp *filer_pb.DeleteCollectionResponse, err error) {
err = fs.filer.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
_, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{
2019-03-19 12:19:37 +00:00
Name: req.GetCollection(),
})
return err
})
return &filer_pb.DeleteCollectionResponse{}, err
}
func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error) {
input := &master_pb.StatisticsRequest{
Replication: req.Replication,
Collection: req.Collection,
Ttl: req.Ttl,
}
2019-02-18 20:11:52 +00:00
output, err := operation.Statistics(fs.filer.GetMaster(), fs.grpcDialOption, input)
if err != nil {
return nil, err
}
return &filer_pb.StatisticsResponse{
TotalSize: output.TotalSize,
UsedSize: output.UsedSize,
FileCount: output.FileCount,
}, nil
}
func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) {
return &filer_pb.GetFilerConfigurationResponse{
Masters: fs.option.Masters,
Collection: fs.option.Collection,
Replication: fs.option.DefaultReplication,
MaxMb: uint32(fs.option.MaxMB),
2020-02-27 08:07:13 +00:00
DirBuckets: fs.filer.DirBucketsPath,
DirQueues: fs.filer.DirQueuesPath,
Cipher: fs.filer.Cipher,
}, nil
}