seaweedfs/weed/command/upload.go

143 lines
4.9 KiB
Go
Raw Normal View History

package command
import (
"context"
"encoding/json"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"google.golang.org/grpc"
"os"
"path/filepath"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
)
var (
upload UploadOptions
)
type UploadOptions struct {
master *string
dir *string
include *string
replication *string
collection *string
dataCenter *string
ttl *string
2020-12-16 17:14:05 +00:00
diskType *string
maxMB *int
usePublicUrl *bool
}
func init() {
cmdUpload.Run = runUpload // break init cycle
2013-01-20 03:49:57 +00:00
cmdUpload.IsDebug = cmdUpload.Flag.Bool("debug", false, "verbose debug information")
upload.master = cmdUpload.Flag.String("master", "localhost:9333", "SeaweedFS master location")
upload.dir = cmdUpload.Flag.String("dir", "", "Upload the whole folder recursively if specified.")
upload.include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
upload.replication = cmdUpload.Flag.String("replication", "", "replication type")
upload.collection = cmdUpload.Flag.String("collection", "", "optional collection name")
upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
2021-02-22 10:03:12 +00:00
upload.diskType = cmdUpload.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
2021-04-01 09:21:59 +00:00
upload.maxMB = cmdUpload.Flag.Int("maxMB", 4, "split files larger than the limit")
upload.usePublicUrl = cmdUpload.Flag.Bool("usePublicUrl", false, "upload to public url from volume server")
}
var cmdUpload = &Command{
UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n weed upload -master=localhost:9333 -dir=one_directory -include=*.pdf",
Short: "upload one or a list of files",
Long: `upload one or a list of files, or batch upload one whole folder recursively.
2015-02-07 23:35:28 +00:00
2013-11-19 07:04:33 +00:00
If uploading a list of files:
It uses consecutive file keys for the list of files.
e.g. If the file1 uses key k, file2 can be read via k_1
2013-07-20 22:14:12 +00:00
If uploading a whole folder recursively:
All files under the folder and subfolders will be uploaded, each with its own file key.
Optional parameter "-include" allows you to specify the file name patterns.
2015-02-07 23:35:28 +00:00
2019-02-06 13:59:15 +00:00
If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separately.
2015-02-07 23:35:28 +00:00
The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
2013-07-20 22:14:12 +00:00
`,
}
func runUpload(cmd *Command, args []string) bool {
2019-02-18 20:11:52 +00:00
util.LoadConfiguration("security", false)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
2019-02-18 20:11:52 +00:00
defaultCollection, err := readMasterConfiguration(grpcDialOption, *upload.master)
if err != nil {
fmt.Printf("upload: %v", err)
return false
}
if *upload.replication == "" {
*upload.replication = defaultCollection
}
if len(args) == 0 {
if *upload.dir == "" {
return false
}
err = filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error {
if err == nil {
if !info.IsDir() {
if *upload.include != "" {
if ok, _ := filepath.Match(*upload.include, filepath.Base(path)); !ok {
return nil
}
}
2013-08-14 17:07:42 +00:00
parts, e := operation.NewFileParts([]string{path})
if e != nil {
return e
}
2021-02-18 04:57:08 +00:00
results, e := operation.SubmitFiles(func() string { return *upload.master }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl)
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
if e != nil {
return e
}
}
} else {
fmt.Println(err)
}
return err
})
if err != nil {
fmt.Println(err.Error())
2021-07-01 08:21:14 +00:00
return false
}
} else {
2013-08-14 17:07:42 +00:00
parts, e := operation.NewFileParts(args)
if e != nil {
fmt.Println(e.Error())
return false
}
results, err := operation.SubmitFiles(func() string { return *upload.master }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl)
if err != nil {
fmt.Println(err.Error())
return false
2013-08-14 17:07:42 +00:00
}
bytes, _ := json.Marshal(results)
fmt.Println(string(bytes))
}
return true
}
func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress string) (replication string, err error) {
err = pb.WithMasterClient(masterAddress, grpcDialOption, func(client master_pb.SeaweedClient) error {
resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return fmt.Errorf("get master %s configuration: %v", masterAddress, err)
}
replication = resp.DefaultReplication
return nil
})
return
}