seaweedfs/weed/command/filer_copy.go

420 lines
12 KiB
Go
Raw Normal View History

package command
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
2019-02-18 20:11:52 +00:00
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/wdclient"
2019-02-18 20:11:52 +00:00
"github.com/spf13/viper"
"google.golang.org/grpc"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
2018-05-30 07:54:56 +00:00
"strconv"
"strings"
"sync"
2018-05-30 07:54:56 +00:00
"time"
)
var (
copy CopyOptions
waitGroup sync.WaitGroup
)
type CopyOptions struct {
2019-02-18 20:11:52 +00:00
filerGrpcPort *int
master *string
include *string
replication *string
collection *string
ttl *string
maxMB *int
grpcDialOption grpc.DialOption
masterClient *wdclient.MasterClient
concurrency *int
}
func init() {
cmdCopy.Run = runCopy // break init cycle
cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
2018-06-06 06:37:41 +00:00
copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
copy.concurrency = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
}
var cmdCopy = &Command{
2016-07-21 22:00:07 +00:00
UsageLine: "filer.copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
Short: "copy one or a list of files to a filer folder",
Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder
It can copy one or a list of files or folders.
If copying a whole folder recursively:
All files under the folder and subfolders will be copyed.
Optional parameter "-include" allows you to specify the file name patterns.
2018-09-28 08:58:34 +00:00
If "maxMB" is set to a positive number, files larger than it would be split into chunks.
`,
}
func runCopy(cmd *Command, args []string) bool {
2019-02-18 20:11:52 +00:00
weed_server.LoadConfiguration("security", false)
if len(args) <= 1 {
return false
}
filerDestination := args[len(args)-1]
2018-07-22 00:39:10 +00:00
fileOrDirs := args[0 : len(args)-1]
filerUrl, err := url.Parse(filerDestination)
if err != nil {
fmt.Printf("The last argument should be a URL on filer: %v\n", err)
return false
}
urlPath := filerUrl.Path
if !strings.HasSuffix(urlPath, "/") {
fmt.Printf("The last argument should be a folder and end with \"/\": %v\n", err)
return false
}
2018-06-06 06:37:41 +00:00
if filerUrl.Port() == "" {
fmt.Printf("The filer port should be specified.\n")
return false
}
filerPort, parseErr := strconv.ParseUint(filerUrl.Port(), 10, 64)
if parseErr != nil {
fmt.Printf("The filer port parse error: %v\n", parseErr)
return false
}
filerGrpcPort := filerPort + 10000
if *copy.filerGrpcPort != 0 {
filerGrpcPort = uint64(*copy.filerGrpcPort)
}
filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
2019-02-18 20:11:52 +00:00
copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
2018-06-06 06:37:41 +00:00
copy.masterClient = wdclient.NewMasterClient(context.Background(), copy.grpcDialOption, "client", strings.Split(*copy.master, ","))
go copy.masterClient.KeepConnectedToMaster()
copy.masterClient.WaitUntilConnected()
if *cmdCopy.IsDebug {
util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
}
fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrency)
ctx := context.Background()
go func() {
defer close(fileCopyTaskChan)
for _, fileOrDir := range fileOrDirs {
if err := genFileCopyTask(fileOrDir, urlPath, fileCopyTaskChan); err != nil {
fmt.Fprintf(os.Stderr, "gen file list error: %v\n", err)
break
}
}
}()
for i := 0; i < *copy.concurrency; i++ {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
worker := FileCopyWorker{
options: &copy,
filerHost: filerUrl.Host,
filerGrpcAddress: filerGrpcAddress,
}
if err := worker.copyFiles(ctx, fileCopyTaskChan); err != nil {
fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
return
}
}()
}
waitGroup.Wait()
return true
}
func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan FileCopyTask) error {
fi, err := os.Stat(fileOrDir)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to get stat for file %s: %v\n", fileOrDir, err)
return nil
}
mode := fi.Mode()
if mode.IsDir() {
files, _ := ioutil.ReadDir(fileOrDir)
for _, subFileOrDir := range files {
if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
return err
}
}
return nil
}
uid, gid := util.GetFileUidGid(fi)
fileCopyTaskChan <- FileCopyTask{
sourceLocation: fileOrDir,
destinationUrlPath: destPath,
fileSize: fi.Size(),
fileMode: fi.Mode(),
uid: uid,
gid: gid,
}
return nil
}
type FileCopyWorker struct {
options *CopyOptions
filerHost string
filerGrpcAddress string
}
func (worker *FileCopyWorker) copyFiles(ctx context.Context, fileCopyTaskChan chan FileCopyTask) error {
for task := range fileCopyTaskChan {
if err := worker.doEachCopy(ctx, task); err != nil {
return err
}
}
return nil
}
type FileCopyTask struct {
sourceLocation string
destinationUrlPath string
fileSize int64
fileMode os.FileMode
uid uint32
gid uint32
}
func (worker *FileCopyWorker) doEachCopy(ctx context.Context, task FileCopyTask) error {
f, err := os.Open(task.sourceLocation)
if err != nil {
fmt.Printf("Failed to open file %s: %v\n", task.sourceLocation, err)
if _, ok := err.(*os.PathError); ok {
fmt.Printf("skipping %s\n", task.sourceLocation)
return nil
}
return err
}
defer f.Close()
// this is a regular file
if *worker.options.include != "" {
if ok, _ := filepath.Match(*worker.options.include, filepath.Base(task.sourceLocation)); !ok {
return nil
}
}
// find the chunk count
chunkSize := int64(*worker.options.maxMB * 1024 * 1024)
chunkCount := 1
if chunkSize > 0 && task.fileSize > chunkSize {
chunkCount = int(task.fileSize/chunkSize) + 1
}
if chunkCount == 1 {
return worker.uploadFileAsOne(ctx, task, f)
}
return worker.uploadFileInChunks(ctx, task, f, chunkCount, chunkSize)
}
func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopyTask, f *os.File) error {
// upload the file content
fileName := filepath.Base(f.Name())
2018-05-30 06:52:27 +00:00
mimeType := detectMimeType(f)
2018-05-31 05:28:14 +00:00
var chunks []*filer_pb.FileChunk
if task.fileSize > 0 {
2018-05-31 05:28:14 +00:00
// assign a volume
assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
2018-05-31 05:28:14 +00:00
Count: 1,
Replication: *worker.options.replication,
Collection: *worker.options.collection,
Ttl: *worker.options.ttl,
2018-05-31 05:28:14 +00:00
})
if err != nil {
fmt.Printf("Failed to assign from %s: %v\n", *worker.options.master, err)
2018-05-31 05:28:14 +00:00
}
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
2019-02-15 08:09:19 +00:00
uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth)
2018-05-31 05:28:14 +00:00
if err != nil {
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
2018-05-31 05:28:14 +00:00
}
if uploadResult.Error != "" {
return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
2018-05-31 05:28:14 +00:00
}
fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
chunks = append(chunks, &filer_pb.FileChunk{
FileId: assignResult.Fid,
Offset: 0,
Size: uint64(uploadResult.Size),
Mtime: time.Now().UnixNano(),
2018-09-23 05:12:21 +00:00
ETag: uploadResult.ETag,
2018-05-31 05:28:14 +00:00
})
fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
}
if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
2018-05-31 05:28:14 +00:00
request := &filer_pb.CreateEntryRequest{
Directory: task.destinationUrlPath,
2018-05-31 05:28:14 +00:00
Entry: &filer_pb.Entry{
Name: fileName,
Attributes: &filer_pb.FuseAttributes{
Crtime: time.Now().Unix(),
Mtime: time.Now().Unix(),
Gid: task.gid,
Uid: task.uid,
FileSize: uint64(task.fileSize),
FileMode: uint32(task.fileMode),
Mime: mimeType,
Replication: *worker.options.replication,
Collection: *worker.options.collection,
TtlSec: int32(util.ParseInt(*worker.options.ttl, 0)),
2018-05-31 05:28:14 +00:00
},
Chunks: chunks,
},
}
2019-03-16 00:20:24 +00:00
if _, err := client.CreateEntry(ctx, request); err != nil {
2018-05-31 05:28:14 +00:00
return fmt.Errorf("update fh: %v", err)
}
return nil
}); err != nil {
return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
}
return nil
}
func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
2018-05-30 07:54:56 +00:00
fileName := filepath.Base(f.Name())
mimeType := detectMimeType(f)
2018-05-30 07:54:56 +00:00
var chunks []*filer_pb.FileChunk
for i := int64(0); i < int64(chunkCount); i++ {
// assign a volume
assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
Count: 1,
Replication: *worker.options.replication,
Collection: *worker.options.collection,
Ttl: *worker.options.ttl,
})
if err != nil {
fmt.Printf("Failed to assign from %s: %v\n", *worker.options.master, err)
2018-05-30 07:54:56 +00:00
}
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
2018-05-30 07:54:56 +00:00
uploadResult, err := operation.Upload(targetUrl,
fileName+"-"+strconv.FormatInt(i+1, 10),
2018-05-30 07:54:56 +00:00
io.LimitReader(f, chunkSize),
2019-02-15 08:09:19 +00:00
false, "application/octet-stream", nil, assignResult.Auth)
2018-05-30 07:54:56 +00:00
if err != nil {
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
2018-05-30 07:54:56 +00:00
}
if uploadResult.Error != "" {
return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
2018-05-30 07:54:56 +00:00
}
chunks = append(chunks, &filer_pb.FileChunk{
FileId: assignResult.Fid,
2018-05-30 07:54:56 +00:00
Offset: i * chunkSize,
Size: uint64(uploadResult.Size),
Mtime: time.Now().UnixNano(),
2018-09-23 05:12:21 +00:00
ETag: uploadResult.ETag,
2018-05-30 07:54:56 +00:00
})
fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
2018-05-30 07:54:56 +00:00
}
if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
2018-05-30 07:54:56 +00:00
request := &filer_pb.CreateEntryRequest{
Directory: task.destinationUrlPath,
2018-05-30 07:54:56 +00:00
Entry: &filer_pb.Entry{
Name: fileName,
2018-05-30 07:54:56 +00:00
Attributes: &filer_pb.FuseAttributes{
Crtime: time.Now().Unix(),
Mtime: time.Now().Unix(),
Gid: task.gid,
Uid: task.uid,
FileSize: uint64(task.fileSize),
FileMode: uint32(task.fileMode),
Mime: mimeType,
Replication: *worker.options.replication,
Collection: *worker.options.collection,
TtlSec: int32(util.ParseInt(*worker.options.ttl, 0)),
2018-05-30 07:54:56 +00:00
},
Chunks: chunks,
},
}
2019-03-16 00:20:24 +00:00
if _, err := client.CreateEntry(ctx, request); err != nil {
2018-05-30 07:54:56 +00:00
return fmt.Errorf("update fh: %v", err)
}
return nil
}); err != nil {
return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
2018-05-30 07:54:56 +00:00
}
fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
return nil
}
2018-05-30 06:52:27 +00:00
func detectMimeType(f *os.File) string {
head := make([]byte, 512)
2019-01-17 01:17:19 +00:00
f.Seek(0, io.SeekStart)
2018-05-30 06:52:27 +00:00
n, err := f.Read(head)
2018-05-31 05:28:14 +00:00
if err == io.EOF {
return ""
}
2018-05-30 06:52:27 +00:00
if err != nil {
fmt.Printf("read head of %v: %v\n", f.Name(), err)
return "application/octet-stream"
}
2019-01-17 01:17:19 +00:00
f.Seek(0, io.SeekStart)
2018-05-30 06:52:27 +00:00
mimeType := http.DetectContentType(head[:n])
return mimeType
}
2018-05-30 07:54:56 +00:00
2019-03-16 00:20:24 +00:00
func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
2018-05-30 07:54:56 +00:00
return util.WithCachedGrpcClient(ctx, func(clientConn *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(clientConn)
return fn(client)
}, filerAddress, grpcDialOption)
2018-05-30 07:54:56 +00:00
}