From 13e215ee5cb5f4c2873f89c263d8c970e9978b19 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 6 Mar 2020 00:49:47 -0800 Subject: [PATCH] filer: option to encrypt data on volume server --- other/java/client/src/main/proto/filer.proto | 2 + .../repeated_vacuum/repeated_vacuum.go | 2 +- weed/command/filer.go | 3 + weed/command/filer_copy.go | 36 +-- weed/command/mount_std.go | 5 +- weed/command/scaffold.go | 4 +- weed/command/server.go | 1 + weed/command/webdav.go | 40 ++- weed/filer2/filechunks.go | 30 +-- weed/filer2/filer.go | 1 + weed/filer2/filer_client_util.go | 7 +- weed/filer2/stream.go | 2 +- weed/filesys/dirty_page.go | 13 +- weed/filesys/wfs.go | 5 +- weed/operation/submit.go | 7 +- weed/operation/upload_content.go | 47 +++- weed/pb/filer.proto | 2 + weed/pb/filer_pb/filer.pb.go | 235 ++++++++++-------- weed/pb/master_pb/master.pb.go | 20 +- weed/pb/volume_server_pb/volume_server.pb.go | 26 +- weed/replication/sink/azuresink/azure_sink.go | 2 +- weed/replication/sink/b2sink/b2_sink.go | 2 +- .../replication/sink/filersink/fetch_write.go | 5 +- weed/replication/sink/gcssink/gcs_sink.go | 5 +- weed/replication/sink/s3sink/s3_write.go | 2 +- weed/server/common.go | 2 +- weed/server/filer_grpc_server.go | 1 + weed/server/filer_server.go | 2 + weed/server/filer_server_handlers_read.go | 25 +- .../filer_server_handlers_write_autochunk.go | 2 +- weed/server/webdav_server.go | 14 +- weed/topology/store_replicate.go | 5 +- weed/util/cipher.go | 60 +++++ weed/util/http_util.go | 51 +++- 34 files changed, 419 insertions(+), 247 deletions(-) create mode 100644 weed/util/cipher.go diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 9ee552561..5983c84d8 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -99,6 +99,7 @@ message FileChunk { string source_file_id = 6; // to be deprecated FileId fid = 7; FileId source_fid = 8; + bytes cipher_key = 9; } message FileId { @@ -229,4 +230,5 @@ message GetFilerConfigurationResponse { uint32 max_mb = 4; string dir_buckets = 5; string dir_queues = 6; + bool cipher = 7; } diff --git a/unmaintained/repeated_vacuum/repeated_vacuum.go b/unmaintained/repeated_vacuum/repeated_vacuum.go index 718b6faa1..4a0464eda 100644 --- a/unmaintained/repeated_vacuum/repeated_vacuum.go +++ b/unmaintained/repeated_vacuum/repeated_vacuum.go @@ -35,7 +35,7 @@ func main() { targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) - _, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, assignResult.Auth) + _, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), false, reader, false, "", nil, assignResult.Auth) if err != nil { log.Fatalf("upload: %v", err) } diff --git a/weed/command/filer.go b/weed/command/filer.go index b5b595215..327ee8316 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -34,6 +34,7 @@ type FilerOptions struct { dataCenter *string enableNotification *bool disableHttp *bool + cipher *bool // default leveldb directory, used in "weed server" mode defaultLevelDbDirectory *string @@ -53,6 +54,7 @@ func init() { f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size") f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center") f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed") + f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers") } var cmdFiler = &Command{ @@ -111,6 +113,7 @@ func (fo *FilerOptions) startFiler() { DefaultLevelDbDir: defaultLevelDbDirectory, DisableHttp: *fo.disableHttp, Port: uint32(*fo.port), + Cipher: *fo.cipher, }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 3e7ae1db2..8c01cfd74 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -41,6 +41,7 @@ type CopyOptions struct { compressionLevel *int grpcDialOption grpc.DialOption masters []string + cipher bool } func init() { @@ -108,7 +109,7 @@ func runCopy(cmd *Command, args []string) bool { filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort) copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") - masters, collection, replication, maxMB, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress) + masters, collection, replication, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress) if err != nil { fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err) return false @@ -123,6 +124,7 @@ func runCopy(cmd *Command, args []string) bool { *copy.maxMB = int(maxMB) } copy.masters = masters + copy.cipher = cipher if *cmdCopy.IsDebug { util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof") @@ -159,13 +161,14 @@ func runCopy(cmd *Command, args []string) bool { return true } -func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, err error) { +func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, cipher bool, err error) { err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) } masters, collection, replication, maxMB = resp.Masters, resp.Collection, resp.Replication, resp.MaxMb + cipher = resp.Cipher return nil }) return @@ -300,7 +303,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId - uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, security.EncodedJwt(assignResult.Auth), *worker.options.compressionLevel) + uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, worker.options.cipher, f, false, mimeType, nil, security.EncodedJwt(assignResult.Auth), *worker.options.compressionLevel) if err != nil { return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) } @@ -310,11 +313,12 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err fmt.Printf("uploaded %s to %s\n", fileName, targetUrl) chunks = append(chunks, &filer_pb.FileChunk{ - FileId: assignResult.FileId, - Offset: 0, - Size: uint64(uploadResult.Size), - Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + FileId: assignResult.FileId, + Offset: 0, + Size: uint64(uploadResult.Size), + Mtime: time.Now().UnixNano(), + ETag: uploadResult.ETag, + CipherKey: uploadResult.CipherKey, }) fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName) @@ -409,10 +413,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, replication = assignResult.Replication } - uploadResult, err := operation.Upload(targetUrl, - fileName+"-"+strconv.FormatInt(i+1, 10), - io.NewSectionReader(f, i*chunkSize, chunkSize), - false, "", nil, security.EncodedJwt(assignResult.Auth)) + uploadResult, err := operation.Upload(targetUrl, fileName+"-"+strconv.FormatInt(i+1, 10), false, io.NewSectionReader(f, i*chunkSize, chunkSize), false, "", nil, security.EncodedJwt(assignResult.Auth)) if err != nil { uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) return @@ -422,11 +423,12 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, return } chunksChan <- &filer_pb.FileChunk{ - FileId: assignResult.FileId, - Offset: i * chunkSize, - Size: uint64(uploadResult.Size), - Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + FileId: assignResult.FileId, + Offset: i * chunkSize, + Size: uint64(uploadResult.Size), + Mtime: time.Now().UnixNano(), + ETag: uploadResult.ETag, + CipherKey: uploadResult.CipherKey, } fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size)) }(i) diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index b195bf143..9177091a5 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -145,11 +145,13 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente // try to connect to filer, filerBucketsPath may be useful later grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + var cipher bool err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - _, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) } + cipher = resp.Cipher return nil }) if err != nil { @@ -183,6 +185,7 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente MountMtime: time.Now(), Umask: umask, OutsideContainerClusterMode: outsideContainerClusterMode, + Cipher: cipher, })) if err != nil { fuse.Unmount(dir) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 5b246b7c0..f4a08fb51 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -75,9 +75,9 @@ const ( # recursive_delete will delete all sub folders and files, similar to "rm -Rf" recursive_delete = false # directories under this folder will be automatically creating a separate bucket -buckets_folder = /buckets +buckets_folder = "/buckets" # directories under this folder will be store message queue data -queues_folder = /queues +queues_folder = "/queues" #################################################### # The following are filer store options diff --git a/weed/command/server.go b/weed/command/server.go index c9d27555c..f45429193 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -82,6 +82,7 @@ func init() { filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") + filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") diff --git a/weed/command/webdav.go b/weed/command/webdav.go index ba88a17be..4f5d5f5ce 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -1,6 +1,7 @@ package command import ( + "context" "fmt" "net/http" "os/user" @@ -9,6 +10,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" @@ -55,12 +57,6 @@ func runWebDav(cmd *Command, args []string) bool { func (wo *WebDavOption) startWebDav() bool { - filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*wo.filer) - if err != nil { - glog.Fatal(err) - return false - } - // detect current user uid, gid := uint32(0), uint32(0) if u, err := user.Current(); err == nil { @@ -72,13 +68,43 @@ func (wo *WebDavOption) startWebDav() bool { } } + // parse filer grpc address + filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*wo.filer) + if err != nil { + glog.Fatal(err) + return false + } + + grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + + var cipher bool + // connect to filer + for { + err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) + } + cipher = resp.Cipher + return nil + }) + if err != nil { + glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerGrpcAddress) + time.Sleep(time.Second) + } else { + glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerGrpcAddress) + break + } + } + ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{ Filer: *wo.filer, FilerGrpcAddress: filerGrpcAddress, - GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.client"), + GrpcDialOption: grpcDialOption, Collection: *wo.collection, Uid: uid, Gid: gid, + Cipher: cipher, }) if webdavServer_err != nil { glog.Fatalf("WebDav Server startup error: %v", webdavServer_err) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index b5876df82..98a965337 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -71,6 +71,7 @@ type ChunkView struct { Size uint64 LogicOffset int64 IsFullChunk bool + CipherKey []byte } func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) { @@ -94,6 +95,7 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int Size: uint64(min(chunk.stop, stop) - offset), LogicOffset: offset, IsFullChunk: isFullChunk, + CipherKey: chunk.cipherKey, }) offset = min(chunk.stop, stop) } @@ -120,13 +122,7 @@ var bufPool = sync.Pool{ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval { - newV := newVisibleInterval( - chunk.Offset, - chunk.Offset+int64(chunk.Size), - chunk.GetFileIdString(), - chunk.Mtime, - true, - ) + newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey) length := len(visibles) if length == 0 { @@ -140,23 +136,11 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb. logPrintf(" before", visibles) for _, v := range visibles { if v.start < chunk.Offset && chunk.Offset < v.stop { - newVisibles = append(newVisibles, newVisibleInterval( - v.start, - chunk.Offset, - v.fileId, - v.modifiedTime, - false, - )) + newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey)) } chunkStop := chunk.Offset + int64(chunk.Size) if v.start < chunkStop && chunkStop < v.stop { - newVisibles = append(newVisibles, newVisibleInterval( - chunkStop, - v.stop, - v.fileId, - v.modifiedTime, - false, - )) + newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey)) } if chunkStop <= v.start || v.stop <= chunk.Offset { newVisibles = append(newVisibles, v) @@ -208,15 +192,17 @@ type VisibleInterval struct { modifiedTime int64 fileId string isFullChunk bool + cipherKey []byte } -func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) VisibleInterval { +func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte) VisibleInterval { return VisibleInterval{ start: start, stop: stop, fileId: fileId, modifiedTime: modifiedTime, isFullChunk: isFullChunk, + cipherKey: cipherKey, } } diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index bf43c3c52..cbb14a5c1 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -33,6 +33,7 @@ type Filer struct { DirBucketsPath string DirQueuesPath string buckets *FilerBuckets + Cipher bool } func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32) *Filer { diff --git a/weed/filer2/filer_client_util.go b/weed/filer2/filer_client_util.go index ab9db2992..e80c4bf36 100644 --- a/weed/filer2/filer_client_util.go +++ b/weed/filer2/filer_client_util.go @@ -70,12 +70,7 @@ func ReadIntoBuffer(filerClient FilerClient, fullFilePath FullPath, buff []byte, volumeServerAddress := filerClient.AdjustedUrl(locations.Locations[0].Url) var n int64 - n, err = util.ReadUrl( - fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), - chunkView.Offset, - int(chunkView.Size), - buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)], - !chunkView.IsFullChunk) + n, err = util.ReadUrl(fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)]) if err != nil { diff --git a/weed/filer2/stream.go b/weed/filer2/stream.go index 01b87cad1..0a1f943ea 100644 --- a/weed/filer2/stream.go +++ b/weed/filer2/stream.go @@ -27,7 +27,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f for _, chunkView := range chunkViews { urlString := fileId2Url[chunkView.FileId] - _, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) { + err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) { w.Write(data) }) if err != nil { diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 67e1d57ef..ff3b8f885 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -174,7 +174,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - uploadResult, err := operation.Upload(fileUrl, pages.f.Name, reader, false, "", nil, auth) + uploadResult, err := operation.Upload(fileUrl, pages.f.Name, pages.f.wfs.option.Cipher, reader, false, "", nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err) return nil, fmt.Errorf("upload data: %v", err) @@ -185,11 +185,12 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, } return &filer_pb.FileChunk{ - FileId: fileId, - Offset: offset, - Size: uint64(size), - Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + FileId: fileId, + Offset: offset, + Size: uint64(size), + Mtime: time.Now().UnixNano(), + ETag: uploadResult.ETag, + CipherKey: uploadResult.CipherKey, }, nil } diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 83826fed5..77438b58e 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -39,8 +39,9 @@ type Option struct { MountCtime time.Time MountMtime time.Time - // whether the mount runs outside SeaweedFS containers - OutsideContainerClusterMode bool + OutsideContainerClusterMode bool // whether the mount runs outside SeaweedFS containers + Cipher bool // whether encrypt data on volume server + } var _ = fs.FS(&WFS{}) diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 62f067430..8c7ed5d7b 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -189,7 +189,7 @@ func (fi FilePart) Upload(maxMB int, master string, jwt security.EncodedJwt, grp cm.DeleteChunks(master, grpcDialOption) } } else { - ret, e := Upload(fileUrl, baseName, fi.Reader, false, fi.MimeType, nil, jwt) + ret, e := Upload(fileUrl, baseName, false, fi.Reader, false, fi.MimeType, nil, jwt) if e != nil { return 0, e } @@ -202,8 +202,7 @@ func upload_one_chunk(filename string, reader io.Reader, master, fileUrl string, jwt security.EncodedJwt, ) (size uint32, e error) { glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...") - uploadResult, uploadError := Upload(fileUrl, filename, reader, false, - "", nil, jwt) + uploadResult, uploadError := Upload(fileUrl, filename, false, reader, false, "", nil, jwt) if uploadError != nil { return 0, uploadError } @@ -221,6 +220,6 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s q := u.Query() q.Set("cm", "true") u.RawQuery = q.Encode() - _, e = Upload(u.String(), manifest.Name, bufReader, false, "application/json", nil, jwt) + _, e = Upload(u.String(), manifest.Name, false, bufReader, false, "application/json", nil, jwt) return e } diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index c387d0230..ba15aea78 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -22,10 +22,11 @@ import ( ) type UploadResult struct { - Name string `json:"name,omitempty"` - Size uint32 `json:"size,omitempty"` - Error string `json:"error,omitempty"` - ETag string `json:"eTag,omitempty"` + Name string `json:"name,omitempty"` + Size uint32 `json:"size,omitempty"` + Error string `json:"error,omitempty"` + ETag string `json:"eTag,omitempty"` + CipherKey []byte `json:"cipherKey,omitempty"` } var ( @@ -41,22 +42,22 @@ func init() { var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") // Upload sends a POST request to a volume server to upload the content with adjustable compression level -func UploadWithLocalCompressionLevel(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) { +func UploadWithLocalCompressionLevel(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) { if compressionLevel < 1 { compressionLevel = 1 } if compressionLevel > 9 { compressionLevel = 9 } - return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, compressionLevel, jwt) + return doUpload(uploadUrl, filename, cipher, reader, isGzipped, mtype, pairMap, compressionLevel, jwt) } // Upload sends a POST request to a volume server to upload the content with fast compression -func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { - return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt) +func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { + return doUpload(uploadUrl, filename, cipher, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt) } -func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) { +func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) { contentIsGzipped := isGzipped shouldGzipNow := false if !isGzipped { @@ -65,7 +66,25 @@ func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped boo contentIsGzipped = true } } - return upload_content(uploadUrl, func(w io.Writer) (err error) { + // encrypt data + var cipherKey util.CipherKey + var clearDataLen int + if cipher { + clearData, err := ioutil.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read raw input: %v", err) + } + clearDataLen = len(clearData) + cipherKey = util.GenCipherKey() + encryptedData, err := util.Encrypt(clearData, cipherKey) + if err != nil { + return nil, fmt.Errorf("encrypt input: %v", err) + } + reader = bytes.NewReader(encryptedData) + } + + // upload data + uploadResult, err := upload_content(uploadUrl, func(w io.Writer) (err error) { if shouldGzipNow { gzWriter, _ := gzip.NewWriterLevel(w, compression) _, err = io.Copy(gzWriter, reader) @@ -75,6 +94,14 @@ func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped boo } return }, filename, contentIsGzipped, mtype, pairMap, jwt) + + // remember cipher key + if uploadResult != nil && cipherKey != nil { + uploadResult.CipherKey = cipherKey + uploadResult.Size = uint32(clearDataLen) + } + + return uploadResult, err } func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 9ee552561..5983c84d8 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -99,6 +99,7 @@ message FileChunk { string source_file_id = 6; // to be deprecated FileId fid = 7; FileId source_fid = 8; + bytes cipher_key = 9; } message FileId { @@ -229,4 +230,5 @@ message GetFilerConfigurationResponse { uint32 max_mb = 4; string dir_buckets = 5; string dir_queues = 6; + bool cipher = 7; } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 269abb8c7..92a12321c 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -287,6 +287,7 @@ type FileChunk struct { SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"` Fid *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"` SourceFid *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"` + CipherKey []byte `protobuf:"bytes,9,opt,name=cipher_key,json=cipherKey,proto3" json:"cipher_key,omitempty"` } func (m *FileChunk) Reset() { *m = FileChunk{} } @@ -350,6 +351,13 @@ func (m *FileChunk) GetSourceFid() *FileId { return nil } +func (m *FileChunk) GetCipherKey() []byte { + if m != nil { + return m.CipherKey + } + return nil +} + type FileId struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` FileKey uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"` @@ -1014,6 +1022,7 @@ type GetFilerConfigurationResponse struct { MaxMb uint32 `protobuf:"varint,4,opt,name=max_mb,json=maxMb" json:"max_mb,omitempty"` DirBuckets string `protobuf:"bytes,5,opt,name=dir_buckets,json=dirBuckets" json:"dir_buckets,omitempty"` DirQueues string `protobuf:"bytes,6,opt,name=dir_queues,json=dirQueues" json:"dir_queues,omitempty"` + Cipher bool `protobuf:"varint,7,opt,name=cipher" json:"cipher,omitempty"` } func (m *GetFilerConfigurationResponse) Reset() { *m = GetFilerConfigurationResponse{} } @@ -1063,6 +1072,13 @@ func (m *GetFilerConfigurationResponse) GetDirQueues() string { return "" } +func (m *GetFilerConfigurationResponse) GetCipher() bool { + if m != nil { + return m.Cipher + } + return false +} + func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse") @@ -1594,113 +1610,114 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1713 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xcb, 0x6e, 0xdb, 0xca, - 0x19, 0x36, 0x75, 0xe7, 0x2f, 0x29, 0xb1, 0x47, 0x76, 0xa2, 0xc8, 0x97, 0x3a, 0x74, 0x93, 0xba, - 0x48, 0xe0, 0x1a, 0x6e, 0x16, 0x49, 0xd3, 0x2e, 0x12, 0x5f, 0x0a, 0xa3, 0xce, 0xa5, 0x74, 0x52, - 0xa4, 0x28, 0x50, 0x82, 0x26, 0x47, 0xf2, 0xd4, 0x24, 0x47, 0x19, 0x0e, 0x6d, 0xa7, 0x8f, 0x52, - 0xa0, 0x8b, 0xbe, 0x47, 0xd1, 0x4d, 0x51, 0xa0, 0xeb, 0xf3, 0x08, 0xe7, 0x01, 0xce, 0xf2, 0xac, - 0x0f, 0x66, 0x86, 0xa4, 0x86, 0xa2, 0x6c, 0x27, 0xe7, 0x20, 0x3b, 0xce, 0x7f, 0x9b, 0x7f, 0xbe, - 0xff, 0x2a, 0x41, 0x7b, 0x48, 0x02, 0xcc, 0xb6, 0xc6, 0x8c, 0x72, 0x8a, 0x5a, 0xf2, 0xe0, 0x8c, - 0x4f, 0xac, 0x37, 0xb0, 0x7c, 0x44, 0xe9, 0x59, 0x32, 0xde, 0x23, 0x0c, 0x7b, 0x9c, 0xb2, 0x4f, - 0xfb, 0x11, 0x67, 0x9f, 0x6c, 0xfc, 0x31, 0xc1, 0x31, 0x47, 0x2b, 0x60, 0xfa, 0x19, 0xa3, 0x6f, - 0xac, 0x1b, 0x9b, 0xa6, 0x3d, 0x21, 0x20, 0x04, 0xb5, 0xc8, 0x0d, 0x71, 0xbf, 0x22, 0x19, 0xf2, - 0xdb, 0xda, 0x87, 0x95, 0xd9, 0x06, 0xe3, 0x31, 0x8d, 0x62, 0x8c, 0x1e, 0x40, 0x1d, 0x0b, 0x82, - 0xb4, 0xd6, 0xde, 0xb9, 0xbd, 0x95, 0xb9, 0xb2, 0xa5, 0xe4, 0x14, 0xd7, 0xfa, 0x8f, 0x01, 0xe8, - 0x88, 0xc4, 0x5c, 0x10, 0x09, 0x8e, 0x3f, 0xcf, 0x9f, 0x3b, 0xd0, 0x18, 0x33, 0x3c, 0x24, 0x97, - 0xa9, 0x47, 0xe9, 0x09, 0x3d, 0x86, 0x85, 0x98, 0xbb, 0x8c, 0x1f, 0x30, 0x1a, 0x1e, 0x90, 0x00, - 0xbf, 0x16, 0x4e, 0x57, 0xa5, 0x48, 0x99, 0x81, 0xb6, 0x00, 0x91, 0xc8, 0x0b, 0x92, 0x98, 0x9c, - 0xe3, 0xe3, 0x8c, 0xdb, 0xaf, 0xad, 0x1b, 0x9b, 0x2d, 0x7b, 0x06, 0x07, 0x2d, 0x42, 0x3d, 0x20, - 0x21, 0xe1, 0xfd, 0xfa, 0xba, 0xb1, 0xd9, 0xb5, 0xd5, 0xc1, 0xfa, 0x2d, 0xf4, 0x0a, 0xfe, 0x7f, - 0xd9, 0xf3, 0xff, 0x59, 0x81, 0xba, 0x24, 0xe4, 0x18, 0x1b, 0x13, 0x8c, 0xd1, 0x7d, 0xe8, 0x90, - 0xd8, 0x99, 0x00, 0x51, 0x91, 0xbe, 0xb5, 0x49, 0x9c, 0x63, 0x8e, 0x1e, 0x41, 0xc3, 0x3b, 0x4d, - 0xa2, 0xb3, 0xb8, 0x5f, 0x5d, 0xaf, 0x6e, 0xb6, 0x77, 0x7a, 0x93, 0x8b, 0xc4, 0x43, 0x77, 0x05, - 0xcf, 0x4e, 0x45, 0xd0, 0x53, 0x00, 0x97, 0x73, 0x46, 0x4e, 0x12, 0x8e, 0x63, 0xf9, 0xd2, 0xf6, - 0x4e, 0x5f, 0x53, 0x48, 0x62, 0xfc, 0x22, 0xe7, 0xdb, 0x9a, 0x2c, 0x7a, 0x06, 0x2d, 0x7c, 0xc9, - 0x71, 0xe4, 0x63, 0xbf, 0x5f, 0x97, 0x17, 0xad, 0x4e, 0xbd, 0x68, 0x6b, 0x3f, 0xe5, 0xab, 0xf7, - 0xe5, 0xe2, 0x83, 0xe7, 0xd0, 0x2d, 0xb0, 0xd0, 0x3c, 0x54, 0xcf, 0x70, 0x16, 0x55, 0xf1, 0x29, - 0x90, 0x3d, 0x77, 0x83, 0x44, 0x25, 0x58, 0xc7, 0x56, 0x87, 0xdf, 0x54, 0x9e, 0x1a, 0xd6, 0x1e, - 0x98, 0x07, 0x49, 0x10, 0xe4, 0x8a, 0x3e, 0x61, 0x99, 0xa2, 0x4f, 0xd8, 0x04, 0xe5, 0xca, 0xb5, - 0x28, 0xff, 0xdb, 0x80, 0x85, 0xfd, 0x73, 0x1c, 0xf1, 0xd7, 0x94, 0x93, 0x21, 0xf1, 0x5c, 0x4e, - 0x68, 0x84, 0x1e, 0x83, 0x49, 0x03, 0xdf, 0xb9, 0x36, 0x4c, 0x2d, 0x1a, 0xa4, 0x5e, 0x3f, 0x06, - 0x33, 0xc2, 0x17, 0xce, 0xb5, 0xd7, 0xb5, 0x22, 0x7c, 0xa1, 0xa4, 0x37, 0xa0, 0xeb, 0xe3, 0x00, - 0x73, 0xec, 0xe4, 0xd1, 0x11, 0xa1, 0xeb, 0x28, 0xe2, 0xae, 0x0a, 0xc7, 0x43, 0xb8, 0x2d, 0x4c, - 0x8e, 0x5d, 0x86, 0x23, 0xee, 0x8c, 0x5d, 0x7e, 0x2a, 0x63, 0x62, 0xda, 0xdd, 0x08, 0x5f, 0xbc, - 0x95, 0xd4, 0xb7, 0x2e, 0x3f, 0xb5, 0xbe, 0x37, 0xc0, 0xcc, 0x83, 0x89, 0xee, 0x42, 0x53, 0x5c, - 0xeb, 0x10, 0x3f, 0x45, 0xa2, 0x21, 0x8e, 0x87, 0xbe, 0xa8, 0x0a, 0x3a, 0x1c, 0xc6, 0x98, 0x4b, - 0xf7, 0xaa, 0x76, 0x7a, 0x12, 0x99, 0x15, 0x93, 0xbf, 0xab, 0x42, 0xa8, 0xd9, 0xf2, 0x5b, 0x20, - 0x1e, 0x72, 0x12, 0x62, 0x79, 0x61, 0xd5, 0x56, 0x07, 0xd4, 0x83, 0x3a, 0x76, 0xb8, 0x3b, 0x92, - 0x19, 0x6e, 0xda, 0x35, 0xfc, 0xce, 0x1d, 0xa1, 0x9f, 0xc3, 0xad, 0x98, 0x26, 0xcc, 0xc3, 0x4e, - 0x76, 0x6d, 0x43, 0x72, 0x3b, 0x8a, 0x7a, 0xa0, 0x2e, 0xb7, 0xa0, 0x3a, 0x24, 0x7e, 0xbf, 0x29, - 0x81, 0x99, 0x2f, 0x26, 0xe1, 0xa1, 0x6f, 0x0b, 0x26, 0xfa, 0x15, 0x40, 0x6e, 0xc9, 0xef, 0xb7, - 0xae, 0x10, 0x35, 0x33, 0xbb, 0xbe, 0xf5, 0x01, 0x1a, 0xa9, 0xf9, 0x65, 0x30, 0xcf, 0x69, 0x90, - 0x84, 0xf9, 0xb3, 0xbb, 0x76, 0x4b, 0x11, 0x0e, 0x7d, 0x74, 0x0f, 0x64, 0x9f, 0x73, 0x44, 0x56, - 0x55, 0xe4, 0x23, 0x25, 0x42, 0x7f, 0xc0, 0xb2, 0x53, 0x78, 0x94, 0x9e, 0x11, 0xf5, 0xfa, 0xa6, - 0x9d, 0x9e, 0xac, 0xef, 0x2a, 0x70, 0xab, 0x98, 0xee, 0xe2, 0x0a, 0x69, 0x45, 0x62, 0x65, 0x48, - 0x33, 0xd2, 0xec, 0x71, 0x01, 0xaf, 0x8a, 0x8e, 0x57, 0xa6, 0x12, 0x52, 0x5f, 0x5d, 0xd0, 0x55, - 0x2a, 0xaf, 0xa8, 0x8f, 0x45, 0xb6, 0x26, 0xc4, 0x97, 0x00, 0x77, 0x6d, 0xf1, 0x29, 0x28, 0x23, - 0xe2, 0xa7, 0xed, 0x43, 0x7c, 0x4a, 0xf7, 0x98, 0xb4, 0xdb, 0x50, 0x21, 0x53, 0x27, 0x11, 0xb2, - 0x50, 0x50, 0x9b, 0x2a, 0x0e, 0xe2, 0x1b, 0xad, 0x43, 0x9b, 0xe1, 0x71, 0x90, 0x66, 0xaf, 0x84, - 0xcf, 0xb4, 0x75, 0x12, 0x5a, 0x03, 0xf0, 0x68, 0x10, 0x60, 0x4f, 0x0a, 0x98, 0x52, 0x40, 0xa3, - 0x88, 0xcc, 0xe1, 0x3c, 0x70, 0x62, 0xec, 0xf5, 0x61, 0xdd, 0xd8, 0xac, 0xdb, 0x0d, 0xce, 0x83, - 0x63, 0xec, 0x89, 0x77, 0x24, 0x31, 0x66, 0x8e, 0x6c, 0x40, 0x6d, 0xa9, 0xd7, 0x12, 0x04, 0xd9, - 0x26, 0x57, 0x01, 0x46, 0x8c, 0x26, 0x63, 0xc5, 0xed, 0xac, 0x57, 0x45, 0x2f, 0x96, 0x14, 0xc9, - 0x7e, 0x00, 0xb7, 0xe2, 0x4f, 0x61, 0x40, 0xa2, 0x33, 0x87, 0xbb, 0x6c, 0x84, 0x79, 0xbf, 0xab, - 0x72, 0x38, 0xa5, 0xbe, 0x93, 0x44, 0x6b, 0x0c, 0x68, 0x97, 0x61, 0x97, 0xe3, 0x2f, 0x18, 0x3b, - 0x9f, 0x57, 0xdd, 0x68, 0x09, 0x1a, 0xd4, 0xc1, 0x97, 0x5e, 0x90, 0x16, 0x59, 0x9d, 0xee, 0x5f, - 0x7a, 0x81, 0xf5, 0x08, 0x7a, 0x85, 0x1b, 0xd3, 0xc6, 0xbc, 0x08, 0x75, 0xcc, 0x18, 0xcd, 0xda, - 0x88, 0x3a, 0x58, 0x7f, 0x06, 0xf4, 0x7e, 0xec, 0x7f, 0x0d, 0xf7, 0xac, 0x25, 0xe8, 0x15, 0x4c, - 0x2b, 0x3f, 0xac, 0xff, 0x19, 0x80, 0xf6, 0x64, 0x37, 0xf8, 0x69, 0x83, 0x58, 0xd4, 0xa7, 0x18, - 0x12, 0xaa, 0xdb, 0xf8, 0x2e, 0x77, 0xd3, 0x11, 0xd6, 0x21, 0xb1, 0xb2, 0xbf, 0xe7, 0x72, 0x37, - 0x1d, 0x25, 0x0c, 0x7b, 0x09, 0x13, 0x53, 0x4d, 0x26, 0xa1, 0x1c, 0x25, 0x76, 0x46, 0x42, 0x4f, - 0xe0, 0x0e, 0x19, 0x45, 0x94, 0xe1, 0x89, 0x98, 0xa3, 0xa0, 0x6a, 0x48, 0xe1, 0x45, 0xc5, 0xcd, - 0x15, 0xf6, 0x25, 0x72, 0x8f, 0xa0, 0x57, 0x78, 0xc6, 0xb5, 0x30, 0xff, 0xc3, 0x80, 0xfe, 0x0b, - 0x4e, 0x43, 0xe2, 0xd9, 0x58, 0x38, 0x5f, 0x78, 0xfa, 0x06, 0x74, 0x45, 0x3f, 0x9e, 0x7e, 0x7e, - 0x87, 0x06, 0xfe, 0x64, 0xde, 0xdd, 0x03, 0xd1, 0x92, 0x1d, 0x0d, 0x85, 0x26, 0x0d, 0x7c, 0x99, - 0x89, 0x1b, 0x20, 0xfa, 0xa6, 0xa6, 0xaf, 0x26, 0x7f, 0x27, 0xc2, 0x17, 0x05, 0x7d, 0x21, 0x24, - 0xf5, 0x55, 0xb3, 0x6d, 0x46, 0xf8, 0x42, 0xe8, 0x5b, 0xcb, 0x70, 0x6f, 0x86, 0x6f, 0x69, 0xb8, - 0xfe, 0x6f, 0x40, 0xef, 0x45, 0x1c, 0x93, 0x51, 0xf4, 0x27, 0xd9, 0x76, 0x32, 0xa7, 0x17, 0xa1, - 0xee, 0xd1, 0x24, 0xe2, 0xd2, 0xd9, 0xba, 0xad, 0x0e, 0x53, 0x95, 0x58, 0x29, 0x55, 0xe2, 0x54, - 0x2d, 0x57, 0xcb, 0xb5, 0xac, 0xd5, 0x6a, 0xad, 0x50, 0xab, 0x3f, 0x83, 0xb6, 0x08, 0xb2, 0xe3, - 0xe1, 0x88, 0x63, 0x96, 0x76, 0x6a, 0x10, 0xa4, 0x5d, 0x49, 0x11, 0x02, 0xfa, 0x44, 0x51, 0xcd, - 0x1a, 0xc6, 0x93, 0x71, 0xf2, 0xad, 0x01, 0x8b, 0xc5, 0xa7, 0xa4, 0x31, 0xbb, 0x72, 0xb2, 0x88, - 0x56, 0xc6, 0x82, 0xf4, 0x1d, 0xe2, 0x53, 0x34, 0x85, 0x71, 0x72, 0x12, 0x10, 0xcf, 0x11, 0x0c, - 0xe5, 0xbf, 0xa9, 0x28, 0xef, 0x59, 0x30, 0x41, 0xa5, 0xa6, 0xa3, 0x82, 0xa0, 0xe6, 0x26, 0xfc, - 0x34, 0x9b, 0x2e, 0xe2, 0x7b, 0x0a, 0xa9, 0xc6, 0x4d, 0x48, 0x35, 0xcb, 0x48, 0xe5, 0x99, 0xd6, - 0xd2, 0x33, 0xed, 0x09, 0xf4, 0xd4, 0x7a, 0x5a, 0x0c, 0xd7, 0x2a, 0x40, 0x3e, 0x47, 0xe2, 0xbe, - 0xa1, 0x9a, 0x59, 0x36, 0x48, 0x62, 0xeb, 0x77, 0x60, 0x1e, 0x51, 0x65, 0x37, 0x46, 0xdb, 0x60, - 0x06, 0xd9, 0x41, 0x8a, 0xb6, 0x77, 0xd0, 0xa4, 0xc6, 0x33, 0x39, 0x7b, 0x22, 0x64, 0x3d, 0x87, - 0x56, 0x46, 0xce, 0x30, 0x33, 0xae, 0xc2, 0xac, 0x32, 0x85, 0x99, 0xf5, 0x5f, 0x03, 0x16, 0x8b, - 0x2e, 0xa7, 0x61, 0x79, 0x0f, 0xdd, 0xfc, 0x0a, 0x27, 0x74, 0xc7, 0xa9, 0x2f, 0xdb, 0xba, 0x2f, - 0x65, 0xb5, 0xdc, 0xc1, 0xf8, 0x95, 0x3b, 0x56, 0xb9, 0xdc, 0x09, 0x34, 0xd2, 0xe0, 0x1d, 0x2c, - 0x94, 0x44, 0x66, 0xec, 0x66, 0xbf, 0xd4, 0x77, 0xb3, 0xc2, 0x7e, 0x99, 0x6b, 0xeb, 0x0b, 0xdb, - 0x33, 0xb8, 0xab, 0xda, 0xc1, 0x6e, 0x1e, 0xc3, 0x0c, 0xfb, 0x62, 0xa8, 0x8d, 0xe9, 0x50, 0x5b, - 0x03, 0xe8, 0x97, 0x55, 0xd3, 0xf2, 0x1b, 0xc1, 0xc2, 0x31, 0x77, 0x39, 0x89, 0x39, 0xf1, 0xf2, - 0x1f, 0x09, 0x53, 0xb9, 0x61, 0xdc, 0x34, 0x11, 0xcb, 0x75, 0x38, 0x0f, 0x55, 0xce, 0xb3, 0xfc, - 0x15, 0x9f, 0x22, 0x0a, 0x48, 0xbf, 0x29, 0x8d, 0xc1, 0x57, 0xb8, 0x4a, 0xe4, 0x03, 0xa7, 0xdc, - 0x0d, 0xd4, 0xc6, 0x51, 0x93, 0x1b, 0x87, 0x29, 0x29, 0x72, 0xe5, 0x50, 0x43, 0xd9, 0x57, 0xdc, - 0xba, 0xda, 0x47, 0x04, 0x41, 0x32, 0x57, 0x01, 0x64, 0xa9, 0xaa, 0x2a, 0x6b, 0x28, 0x5d, 0x41, - 0xd9, 0x15, 0x04, 0x6b, 0x0d, 0x56, 0x7e, 0x8f, 0xb9, 0xd8, 0x9d, 0xd8, 0x2e, 0x8d, 0x86, 0x64, - 0x94, 0x30, 0x57, 0x0b, 0x85, 0xf5, 0x8d, 0x01, 0xab, 0x57, 0x08, 0xa4, 0x0f, 0xee, 0x43, 0x33, - 0x74, 0x63, 0x8e, 0x59, 0x56, 0x25, 0xd9, 0x71, 0x1a, 0x8a, 0xca, 0x4d, 0x50, 0x54, 0x4b, 0x50, - 0x2c, 0x41, 0x23, 0x74, 0x2f, 0x9d, 0xf0, 0x24, 0x5d, 0x8e, 0xea, 0xa1, 0x7b, 0xf9, 0xea, 0x44, - 0x76, 0x36, 0xc2, 0x9c, 0x93, 0xc4, 0x3b, 0xc3, 0x3c, 0xce, 0x3b, 0x1b, 0x61, 0x2f, 0x15, 0x45, - 0x3c, 0x5a, 0x08, 0x7c, 0x4c, 0x70, 0x82, 0xe3, 0xb4, 0x57, 0x88, 0xe1, 0xf8, 0x47, 0x49, 0xd8, - 0xf9, 0x57, 0x0b, 0x3a, 0xc7, 0xd8, 0xbd, 0xc0, 0xd8, 0x97, 0x0f, 0x43, 0xa3, 0xac, 0xa0, 0x8a, - 0x3f, 0x51, 0xd1, 0x83, 0xe9, 0xca, 0x99, 0xf9, 0x9b, 0x78, 0xf0, 0xf0, 0x26, 0xb1, 0x34, 0x37, - 0xe7, 0xd0, 0x6b, 0x68, 0x6b, 0xbf, 0x01, 0xd1, 0x8a, 0xa6, 0x58, 0xfa, 0x69, 0x3b, 0x58, 0xbd, - 0x82, 0x9b, 0x59, 0xdb, 0x36, 0xd0, 0x11, 0xb4, 0xb5, 0xd5, 0x45, 0xb7, 0x57, 0xde, 0xa1, 0x74, - 0x7b, 0x33, 0xf6, 0x1d, 0x6b, 0x4e, 0x58, 0xd3, 0x16, 0x10, 0xdd, 0x5a, 0x79, 0xe5, 0xd1, 0xad, - 0xcd, 0xda, 0x5a, 0xa4, 0x35, 0x6d, 0xde, 0xeb, 0xd6, 0xca, 0xdb, 0x8c, 0x6e, 0x6d, 0xc6, 0x92, - 0x60, 0xcd, 0xa1, 0x0f, 0xd0, 0x3b, 0xe6, 0x0c, 0xbb, 0xe1, 0x84, 0x3d, 0x85, 0xe0, 0x8f, 0xb0, - 0xba, 0x69, 0x6c, 0x1b, 0xe8, 0xaf, 0xb0, 0x50, 0x9a, 0xe6, 0xc8, 0x9a, 0x68, 0x5e, 0xb5, 0x86, - 0x0c, 0x36, 0xae, 0x95, 0xc9, 0x3d, 0x7f, 0x03, 0x1d, 0x7d, 0x88, 0x22, 0xcd, 0xa9, 0x19, 0x7b, - 0xc2, 0x60, 0xed, 0x2a, 0xb6, 0x6e, 0x50, 0xef, 0xe3, 0xba, 0xc1, 0x19, 0x93, 0x4c, 0x37, 0x38, - 0xab, 0xfd, 0x5b, 0x73, 0xe8, 0x2f, 0x30, 0x3f, 0xdd, 0x4f, 0xd1, 0xfd, 0x69, 0xe8, 0x4a, 0x6d, - 0x7a, 0x60, 0x5d, 0x27, 0x92, 0x1b, 0x3f, 0x04, 0x98, 0xb4, 0x49, 0xb4, 0x3c, 0xd1, 0x29, 0xb5, - 0xe9, 0xc1, 0xca, 0x6c, 0x66, 0x6e, 0xea, 0x6f, 0xb0, 0x34, 0xb3, 0x17, 0x21, 0xad, 0x00, 0xaf, - 0xeb, 0x66, 0x83, 0x5f, 0xdc, 0x28, 0x97, 0xdd, 0xf5, 0x72, 0x0d, 0xe6, 0x63, 0xd5, 0x22, 0x86, - 0xf1, 0x96, 0x17, 0x10, 0x1c, 0xf1, 0x97, 0x20, 0x35, 0xde, 0x32, 0xca, 0xe9, 0x49, 0x43, 0xfe, - 0x6f, 0xf6, 0xeb, 0x1f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xdb, 0x25, 0xa7, 0x6f, 0x46, 0x13, 0x00, - 0x00, + // 1742 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x4f, 0x6f, 0xdc, 0xc6, + 0x15, 0x37, 0xf7, 0x3f, 0xdf, 0xee, 0x3a, 0xd2, 0xac, 0x9c, 0xac, 0xd7, 0x92, 0xab, 0xd0, 0x75, + 0xaa, 0xc2, 0x86, 0x6a, 0xa8, 0x39, 0x24, 0x4d, 0x7b, 0xb0, 0x65, 0xb9, 0x30, 0x62, 0x3b, 0x2e, + 0x65, 0x17, 0x29, 0x0a, 0x94, 0xa0, 0xc8, 0xd9, 0xd5, 0x54, 0x24, 0x87, 0x19, 0x0e, 0x2d, 0xb9, + 0xdf, 0xa3, 0x97, 0x02, 0x3d, 0xf4, 0x7b, 0x14, 0xbd, 0x14, 0x05, 0xfa, 0x39, 0x7a, 0xec, 0xa1, + 0x9f, 0xa1, 0x98, 0x37, 0x24, 0x77, 0xb8, 0x5c, 0x49, 0x49, 0x8b, 0xdc, 0x38, 0xef, 0xdf, 0xbc, + 0xf9, 0xbd, 0xbf, 0xbb, 0x30, 0x9c, 0xb3, 0x88, 0x8a, 0xfd, 0x54, 0x70, 0xc9, 0xc9, 0x00, 0x0f, + 0x5e, 0x7a, 0xe2, 0x7c, 0x05, 0x77, 0x5e, 0x70, 0x7e, 0x96, 0xa7, 0x4f, 0x99, 0xa0, 0x81, 0xe4, + 0xe2, 0xfd, 0x51, 0x22, 0xc5, 0x7b, 0x97, 0x7e, 0x93, 0xd3, 0x4c, 0x92, 0x6d, 0xb0, 0xc3, 0x92, + 0x31, 0xb5, 0x76, 0xad, 0x3d, 0xdb, 0x5d, 0x12, 0x08, 0x81, 0x4e, 0xe2, 0xc7, 0x74, 0xda, 0x42, + 0x06, 0x7e, 0x3b, 0x47, 0xb0, 0xbd, 0xde, 0x60, 0x96, 0xf2, 0x24, 0xa3, 0xe4, 0x3e, 0x74, 0xa9, + 0x22, 0xa0, 0xb5, 0xe1, 0xc1, 0x07, 0xfb, 0xa5, 0x2b, 0xfb, 0x5a, 0x4e, 0x73, 0x9d, 0xbf, 0x59, + 0x40, 0x5e, 0xb0, 0x4c, 0x2a, 0x22, 0xa3, 0xd9, 0xb7, 0xf3, 0xe7, 0x43, 0xe8, 0xa5, 0x82, 0xce, + 0xd9, 0x45, 0xe1, 0x51, 0x71, 0x22, 0x0f, 0x61, 0x33, 0x93, 0xbe, 0x90, 0xcf, 0x04, 0x8f, 0x9f, + 0xb1, 0x88, 0xbe, 0x52, 0x4e, 0xb7, 0x51, 0xa4, 0xc9, 0x20, 0xfb, 0x40, 0x58, 0x12, 0x44, 0x79, + 0xc6, 0xde, 0xd1, 0xe3, 0x92, 0x3b, 0xed, 0xec, 0x5a, 0x7b, 0x03, 0x77, 0x0d, 0x87, 0x6c, 0x41, + 0x37, 0x62, 0x31, 0x93, 0xd3, 0xee, 0xae, 0xb5, 0x37, 0x76, 0xf5, 0xc1, 0xf9, 0x39, 0x4c, 0x6a, + 0xfe, 0x7f, 0xb7, 0xe7, 0xff, 0xb9, 0x05, 0x5d, 0x24, 0x54, 0x18, 0x5b, 0x4b, 0x8c, 0xc9, 0xc7, + 0x30, 0x62, 0x99, 0xb7, 0x04, 0xa2, 0x85, 0xbe, 0x0d, 0x59, 0x56, 0x61, 0x4e, 0x1e, 0x40, 0x2f, + 0x38, 0xcd, 0x93, 0xb3, 0x6c, 0xda, 0xde, 0x6d, 0xef, 0x0d, 0x0f, 0x26, 0xcb, 0x8b, 0xd4, 0x43, + 0x0f, 0x15, 0xcf, 0x2d, 0x44, 0xc8, 0x67, 0x00, 0xbe, 0x94, 0x82, 0x9d, 0xe4, 0x92, 0x66, 0xf8, + 0xd2, 0xe1, 0xc1, 0xd4, 0x50, 0xc8, 0x33, 0xfa, 0xb8, 0xe2, 0xbb, 0x86, 0x2c, 0xf9, 0x1c, 0x06, + 0xf4, 0x42, 0xd2, 0x24, 0xa4, 0xe1, 0xb4, 0x8b, 0x17, 0xed, 0xac, 0xbc, 0x68, 0xff, 0xa8, 0xe0, + 0xeb, 0xf7, 0x55, 0xe2, 0xb3, 0x2f, 0x60, 0x5c, 0x63, 0x91, 0x0d, 0x68, 0x9f, 0xd1, 0x32, 0xaa, + 0xea, 0x53, 0x21, 0xfb, 0xce, 0x8f, 0x72, 0x9d, 0x60, 0x23, 0x57, 0x1f, 0x7e, 0xd6, 0xfa, 0xcc, + 0x72, 0x9e, 0x82, 0xfd, 0x2c, 0x8f, 0xa2, 0x4a, 0x31, 0x64, 0xa2, 0x54, 0x0c, 0x99, 0x58, 0xa2, + 0xdc, 0xba, 0x12, 0xe5, 0xbf, 0x5a, 0xb0, 0x79, 0xf4, 0x8e, 0x26, 0xf2, 0x15, 0x97, 0x6c, 0xce, + 0x02, 0x5f, 0x32, 0x9e, 0x90, 0x87, 0x60, 0xf3, 0x28, 0xf4, 0xae, 0x0c, 0xd3, 0x80, 0x47, 0x85, + 0xd7, 0x0f, 0xc1, 0x4e, 0xe8, 0xb9, 0x77, 0xe5, 0x75, 0x83, 0x84, 0x9e, 0x6b, 0xe9, 0x7b, 0x30, + 0x0e, 0x69, 0x44, 0x25, 0xf5, 0xaa, 0xe8, 0xa8, 0xd0, 0x8d, 0x34, 0xf1, 0x50, 0x87, 0xe3, 0x13, + 0xf8, 0x40, 0x99, 0x4c, 0x7d, 0x41, 0x13, 0xe9, 0xa5, 0xbe, 0x3c, 0xc5, 0x98, 0xd8, 0xee, 0x38, + 0xa1, 0xe7, 0xaf, 0x91, 0xfa, 0xda, 0x97, 0xa7, 0xce, 0x1f, 0x5b, 0x60, 0x57, 0xc1, 0x24, 0x1f, + 0x41, 0x5f, 0x5d, 0xeb, 0xb1, 0xb0, 0x40, 0xa2, 0xa7, 0x8e, 0xcf, 0x43, 0x55, 0x15, 0x7c, 0x3e, + 0xcf, 0xa8, 0x44, 0xf7, 0xda, 0x6e, 0x71, 0x52, 0x99, 0x95, 0xb1, 0x3f, 0xe8, 0x42, 0xe8, 0xb8, + 0xf8, 0xad, 0x10, 0x8f, 0x25, 0x8b, 0x29, 0x5e, 0xd8, 0x76, 0xf5, 0x81, 0x4c, 0xa0, 0x4b, 0x3d, + 0xe9, 0x2f, 0x30, 0xc3, 0x6d, 0xb7, 0x43, 0xdf, 0xf8, 0x0b, 0xf2, 0x43, 0xb8, 0x99, 0xf1, 0x5c, + 0x04, 0xd4, 0x2b, 0xaf, 0xed, 0x21, 0x77, 0xa4, 0xa9, 0xcf, 0xf4, 0xe5, 0x0e, 0xb4, 0xe7, 0x2c, + 0x9c, 0xf6, 0x11, 0x98, 0x8d, 0x7a, 0x12, 0x3e, 0x0f, 0x5d, 0xc5, 0x24, 0x3f, 0x01, 0xa8, 0x2c, + 0x85, 0xd3, 0xc1, 0x25, 0xa2, 0x76, 0x69, 0x37, 0x24, 0x3b, 0x00, 0x01, 0x4b, 0x4f, 0xa9, 0xf0, + 0x54, 0xc2, 0xd8, 0x98, 0x1c, 0xb6, 0xa6, 0x7c, 0x49, 0xdf, 0x3b, 0x5f, 0x43, 0xaf, 0xb8, 0xfd, + 0x0e, 0xd8, 0xef, 0x78, 0x94, 0xc7, 0x15, 0x2a, 0x63, 0x77, 0xa0, 0x09, 0xcf, 0x43, 0x72, 0x1b, + 0xb0, 0x0d, 0xa2, 0x8d, 0x16, 0x62, 0x80, 0x00, 0x7e, 0x49, 0xb1, 0x91, 0x04, 0x9c, 0x9f, 0x31, + 0x0d, 0x4e, 0xdf, 0x2d, 0x4e, 0xce, 0x7f, 0x5a, 0x70, 0xb3, 0x5e, 0x0d, 0xea, 0x0a, 0xb4, 0x82, + 0x50, 0x5a, 0x68, 0x06, 0xcd, 0x1e, 0xd7, 0xe0, 0x6c, 0x99, 0x70, 0x96, 0x2a, 0x31, 0x0f, 0xf5, + 0x05, 0x63, 0xad, 0xf2, 0x92, 0x87, 0x54, 0x25, 0x73, 0xce, 0x42, 0xc4, 0x7f, 0xec, 0xaa, 0x4f, + 0x45, 0x59, 0xb0, 0xb0, 0xe8, 0x2e, 0xea, 0x13, 0xdd, 0x13, 0x68, 0xb7, 0xa7, 0x23, 0xaa, 0x4f, + 0x2a, 0xa2, 0xb1, 0xa2, 0xf6, 0x75, 0x98, 0xd4, 0x37, 0xd9, 0x85, 0xa1, 0xa0, 0x69, 0x54, 0x24, + 0x37, 0xa2, 0x6b, 0xbb, 0x26, 0x89, 0xdc, 0x05, 0x08, 0x78, 0x14, 0xd1, 0x00, 0x05, 0x6c, 0x14, + 0x30, 0x28, 0x2a, 0xb1, 0xa4, 0x8c, 0xbc, 0x8c, 0x06, 0x53, 0xd8, 0xb5, 0xf6, 0xba, 0x6e, 0x4f, + 0xca, 0xe8, 0x98, 0x06, 0xea, 0x1d, 0x79, 0x46, 0x85, 0x87, 0xfd, 0x69, 0x88, 0x7a, 0x03, 0x45, + 0xc0, 0x2e, 0xba, 0x03, 0xb0, 0x10, 0x3c, 0x4f, 0x35, 0x77, 0xb4, 0xdb, 0x56, 0xad, 0x1a, 0x29, + 0xc8, 0xbe, 0x0f, 0x37, 0xb3, 0xf7, 0x71, 0xc4, 0x92, 0x33, 0x4f, 0xfa, 0x62, 0x41, 0xe5, 0x74, + 0xac, 0x53, 0xbc, 0xa0, 0xbe, 0x41, 0xa2, 0x93, 0x02, 0x39, 0x14, 0xd4, 0x97, 0xf4, 0x3b, 0x4c, + 0xa5, 0x6f, 0x57, 0xfc, 0xe4, 0x16, 0xf4, 0xb8, 0x47, 0x2f, 0x82, 0xa8, 0xa8, 0xc1, 0x2e, 0x3f, + 0xba, 0x08, 0x22, 0xe7, 0x01, 0x4c, 0x6a, 0x37, 0x16, 0x7d, 0x7b, 0x0b, 0xba, 0x54, 0x08, 0x5e, + 0x76, 0x19, 0x7d, 0x70, 0x7e, 0x03, 0xe4, 0x6d, 0x1a, 0x7e, 0x1f, 0xee, 0x39, 0xb7, 0x60, 0x52, + 0x33, 0xad, 0xfd, 0x70, 0xfe, 0x61, 0x01, 0x79, 0x8a, 0xcd, 0xe2, 0xff, 0x9b, 0xd3, 0xaa, 0x7c, + 0xd5, 0x0c, 0xd1, 0xcd, 0x28, 0xf4, 0xa5, 0x5f, 0x4c, 0xb8, 0x11, 0xcb, 0xb4, 0xfd, 0xa7, 0xbe, + 0xf4, 0x8b, 0x49, 0x23, 0x68, 0x90, 0x0b, 0x35, 0xf4, 0x30, 0x09, 0x71, 0xd2, 0xb8, 0x25, 0x89, + 0x7c, 0x0a, 0x1f, 0xb2, 0x45, 0xc2, 0x05, 0x5d, 0x8a, 0x79, 0x1a, 0xaa, 0x1e, 0x0a, 0x6f, 0x69, + 0x6e, 0xa5, 0x70, 0x84, 0xc8, 0x3d, 0x80, 0x49, 0xed, 0x19, 0x57, 0xc2, 0xfc, 0x27, 0x0b, 0xa6, + 0x8f, 0x25, 0x8f, 0x59, 0xe0, 0x52, 0xe5, 0x7c, 0xed, 0xe9, 0xf7, 0x60, 0xac, 0xda, 0xf5, 0xea, + 0xf3, 0x47, 0x3c, 0x0a, 0x97, 0xe3, 0xf0, 0x36, 0xa8, 0x8e, 0xed, 0x19, 0x28, 0xf4, 0x79, 0x14, + 0x62, 0x26, 0xde, 0x03, 0xd5, 0x56, 0x0d, 0x7d, 0xbd, 0x18, 0x8c, 0x12, 0x7a, 0x5e, 0xd3, 0x57, + 0x42, 0xa8, 0xaf, 0x7b, 0x71, 0x3f, 0xa1, 0xe7, 0x4a, 0xdf, 0xb9, 0x03, 0xb7, 0xd7, 0xf8, 0x56, + 0x84, 0xeb, 0x9f, 0x16, 0x4c, 0x1e, 0x67, 0x19, 0x5b, 0x24, 0xbf, 0xc6, 0xb6, 0x53, 0x3a, 0xbd, + 0x05, 0xdd, 0x80, 0xe7, 0x89, 0x44, 0x67, 0xbb, 0xae, 0x3e, 0xac, 0x54, 0x62, 0xab, 0x51, 0x89, + 0x2b, 0xb5, 0xdc, 0x6e, 0xd6, 0xb2, 0x51, 0xab, 0x9d, 0x5a, 0xad, 0xfe, 0x00, 0x86, 0x2a, 0xc8, + 0x5e, 0x40, 0x13, 0x49, 0x45, 0xd1, 0xc8, 0x41, 0x91, 0x0e, 0x91, 0xa2, 0x04, 0xcc, 0x81, 0xa3, + 0x7b, 0x39, 0xa4, 0xcb, 0x69, 0xf3, 0x2f, 0x0b, 0xb6, 0xea, 0x4f, 0x29, 0x62, 0x76, 0xe9, 0xe0, + 0x51, 0xad, 0x4c, 0x44, 0xc5, 0x3b, 0xd4, 0xa7, 0x6a, 0x0a, 0x69, 0x7e, 0x12, 0xb1, 0xc0, 0x53, + 0x0c, 0xed, 0xbf, 0xad, 0x29, 0x6f, 0x45, 0xb4, 0x44, 0xa5, 0x63, 0xa2, 0x42, 0xa0, 0xe3, 0xe7, + 0xf2, 0xb4, 0x1c, 0x3e, 0xea, 0x7b, 0x05, 0xa9, 0xde, 0x75, 0x48, 0xf5, 0x9b, 0x48, 0x55, 0x99, + 0x36, 0x30, 0x33, 0xed, 0x53, 0x98, 0xe8, 0xed, 0xb5, 0x1e, 0xae, 0x1d, 0x80, 0x6a, 0x8e, 0x64, + 0x53, 0x4b, 0x37, 0xb3, 0x72, 0x90, 0x64, 0xce, 0x2f, 0xc0, 0x7e, 0xc1, 0xb5, 0xdd, 0x8c, 0x3c, + 0x02, 0x3b, 0x2a, 0x0f, 0x28, 0x3a, 0x3c, 0x20, 0xcb, 0x1a, 0x2f, 0xe5, 0xdc, 0xa5, 0x90, 0xf3, + 0x05, 0x0c, 0x4a, 0x72, 0x89, 0x99, 0x75, 0x19, 0x66, 0xad, 0x15, 0xcc, 0x9c, 0xbf, 0x5b, 0xb0, + 0x55, 0x77, 0xb9, 0x08, 0xcb, 0x5b, 0x18, 0x57, 0x57, 0x78, 0xb1, 0x9f, 0x16, 0xbe, 0x3c, 0x32, + 0x7d, 0x69, 0xaa, 0x55, 0x0e, 0x66, 0x2f, 0xfd, 0x54, 0xe7, 0xf2, 0x28, 0x32, 0x48, 0xb3, 0x37, + 0xb0, 0xd9, 0x10, 0x59, 0xb3, 0xba, 0xfd, 0xd8, 0x5c, 0xdd, 0x6a, 0xeb, 0x67, 0xa5, 0x6d, 0xee, + 0x73, 0x9f, 0xc3, 0x47, 0xba, 0x1d, 0x1c, 0x56, 0x31, 0x2c, 0xb1, 0xaf, 0x87, 0xda, 0x5a, 0x0d, + 0xb5, 0x33, 0x83, 0x69, 0x53, 0xb5, 0x28, 0xbf, 0x05, 0x6c, 0x1e, 0x4b, 0x5f, 0xb2, 0x4c, 0xb2, + 0xa0, 0xfa, 0x0d, 0xb1, 0x92, 0x1b, 0xd6, 0x75, 0x13, 0xb1, 0x59, 0x87, 0x1b, 0xd0, 0x96, 0xb2, + 0xcc, 0x5f, 0xf5, 0xa9, 0xa2, 0x40, 0xcc, 0x9b, 0x8a, 0x18, 0x7c, 0x0f, 0x57, 0xa9, 0x7c, 0x90, + 0x5c, 0xfa, 0x91, 0xde, 0x38, 0x3a, 0xb8, 0x71, 0xd8, 0x48, 0xc1, 0x95, 0x43, 0x0f, 0xe5, 0x50, + 0x73, 0xbb, 0x7a, 0x1f, 0x51, 0x04, 0x64, 0xee, 0x00, 0x60, 0xa9, 0xea, 0x2a, 0xeb, 0x69, 0x5d, + 0x45, 0x39, 0x54, 0x04, 0xe7, 0x2e, 0x6c, 0xff, 0x92, 0x4a, 0xb5, 0x3b, 0x89, 0x43, 0x9e, 0xcc, + 0xd9, 0x22, 0x17, 0xbe, 0x11, 0x0a, 0xe7, 0xdf, 0x16, 0xec, 0x5c, 0x22, 0x50, 0x3c, 0x78, 0x0a, + 0xfd, 0xd8, 0xcf, 0x24, 0x15, 0x65, 0x95, 0x94, 0xc7, 0x55, 0x28, 0x5a, 0xd7, 0x41, 0xd1, 0x6e, + 0x40, 0x71, 0x0b, 0x7a, 0xb1, 0x7f, 0xe1, 0xc5, 0x27, 0xc5, 0x72, 0xd4, 0x8d, 0xfd, 0x8b, 0x97, + 0x27, 0xd8, 0xd9, 0x98, 0xf0, 0x4e, 0xf2, 0xe0, 0x8c, 0xca, 0xac, 0xea, 0x6c, 0x4c, 0x3c, 0xd1, + 0x14, 0xf5, 0x68, 0x25, 0xf0, 0x4d, 0x4e, 0x73, 0x9a, 0x15, 0xbd, 0x42, 0x0d, 0xc7, 0x5f, 0x21, + 0x01, 0x97, 0x29, 0x5c, 0x1d, 0xb1, 0x4b, 0x0c, 0xdc, 0xe2, 0x74, 0xf0, 0x97, 0x01, 0x8c, 0x8e, + 0xa9, 0x7f, 0x4e, 0x69, 0x88, 0x0f, 0x26, 0x8b, 0xb2, 0xd0, 0xea, 0xbf, 0x6c, 0xc9, 0xfd, 0xd5, + 0x8a, 0x5a, 0xfb, 0x53, 0x7a, 0xf6, 0xc9, 0x75, 0x62, 0x45, 0xce, 0xde, 0x20, 0xaf, 0x60, 0x68, + 0xfc, 0x74, 0x24, 0xdb, 0x86, 0x62, 0xe3, 0x17, 0xf1, 0x6c, 0xe7, 0x12, 0x6e, 0x69, 0xed, 0x91, + 0x45, 0x5e, 0xc0, 0xd0, 0x58, 0x69, 0x4c, 0x7b, 0xcd, 0xdd, 0xca, 0xb4, 0xb7, 0x66, 0x0f, 0x72, + 0x6e, 0x28, 0x6b, 0xc6, 0x62, 0x62, 0x5a, 0x6b, 0xae, 0x42, 0xa6, 0xb5, 0x75, 0xdb, 0x0c, 0x5a, + 0x33, 0xf6, 0x00, 0xd3, 0x5a, 0x73, 0xcb, 0x31, 0xad, 0xad, 0x59, 0x1e, 0x9c, 0x1b, 0xe4, 0x6b, + 0x98, 0x1c, 0x4b, 0x41, 0xfd, 0x78, 0xc9, 0x5e, 0x41, 0xf0, 0x7f, 0xb0, 0xba, 0x67, 0x3d, 0xb2, + 0xc8, 0xef, 0x60, 0xb3, 0x31, 0xe5, 0x89, 0xb3, 0xd4, 0xbc, 0x6c, 0x3d, 0x99, 0xdd, 0xbb, 0x52, + 0xa6, 0xf2, 0xfc, 0x2b, 0x18, 0x99, 0xc3, 0x95, 0x18, 0x4e, 0xad, 0xd9, 0x1f, 0x66, 0x77, 0x2f, + 0x63, 0x9b, 0x06, 0xcd, 0xfe, 0x6e, 0x1a, 0x5c, 0x33, 0xe1, 0x4c, 0x83, 0xeb, 0xc6, 0x82, 0x73, + 0x83, 0xfc, 0x16, 0x36, 0x56, 0xfb, 0x2c, 0xf9, 0x78, 0x15, 0xba, 0x46, 0xfb, 0x9e, 0x39, 0x57, + 0x89, 0x54, 0xc6, 0x9f, 0x03, 0x2c, 0xdb, 0x27, 0xb9, 0xb3, 0xd4, 0x69, 0xb4, 0xef, 0xd9, 0xf6, + 0x7a, 0x66, 0x65, 0xea, 0xf7, 0x70, 0x6b, 0x6d, 0x8f, 0x22, 0x46, 0x01, 0x5e, 0xd5, 0xe5, 0x66, + 0x3f, 0xba, 0x56, 0xae, 0xbc, 0xeb, 0xc9, 0x5d, 0xd8, 0xc8, 0x74, 0x8b, 0x98, 0x67, 0xfb, 0x41, + 0xc4, 0x68, 0x22, 0x9f, 0x00, 0x6a, 0xbc, 0x16, 0x5c, 0xf2, 0x93, 0x1e, 0xfe, 0xdd, 0xf6, 0xd3, + 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xf6, 0xb5, 0x1d, 0x19, 0x7d, 0x13, 0x00, 0x00, } diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 95c9533a1..c33e2b768 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -428,12 +428,10 @@ type VolumeEcShardInformationMessage struct { EcIndexBits uint32 `protobuf:"varint,3,opt,name=ec_index_bits,json=ecIndexBits" json:"ec_index_bits,omitempty"` } -func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} } -func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardInformationMessage) ProtoMessage() {} -func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{4} -} +func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} } +func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardInformationMessage) ProtoMessage() {} +func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (m *VolumeEcShardInformationMessage) GetId() uint32 { if m != nil { @@ -1424,12 +1422,10 @@ type GetMasterConfigurationResponse struct { MetricsIntervalSeconds uint32 `protobuf:"varint,2,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds" json:"metrics_interval_seconds,omitempty"` } -func (m *GetMasterConfigurationResponse) Reset() { *m = GetMasterConfigurationResponse{} } -func (m *GetMasterConfigurationResponse) String() string { return proto.CompactTextString(m) } -func (*GetMasterConfigurationResponse) ProtoMessage() {} -func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{32} -} +func (m *GetMasterConfigurationResponse) Reset() { *m = GetMasterConfigurationResponse{} } +func (m *GetMasterConfigurationResponse) String() string { return proto.CompactTextString(m) } +func (*GetMasterConfigurationResponse) ProtoMessage() {} +func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} } func (m *GetMasterConfigurationResponse) GetMetricsAddress() string { if m != nil { diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 588b18f2e..56baa0cf7 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -1035,12 +1035,10 @@ func (m *VolumeEcShardsGenerateRequest) GetCollection() string { type VolumeEcShardsGenerateResponse struct { } -func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } -func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} -func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{41} -} +func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } +func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} +func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} } type VolumeEcShardsRebuildRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -1413,12 +1411,10 @@ func (m *VolumeEcShardsToVolumeRequest) GetCollection() string { type VolumeEcShardsToVolumeResponse struct { } -func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } -func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} -func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{57} -} +func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } +func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} +func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{57} } type ReadVolumeFileStatusRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -2085,10 +2081,8 @@ type QueryRequest_InputSerialization_JSONInput struct { func (m *QueryRequest_InputSerialization_JSONInput) Reset() { *m = QueryRequest_InputSerialization_JSONInput{} } -func (m *QueryRequest_InputSerialization_JSONInput) String() string { - return proto.CompactTextString(m) -} -func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} +func (m *QueryRequest_InputSerialization_JSONInput) String() string { return proto.CompactTextString(m) } +func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{70, 1, 1} } diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go index f7c0eef0d..66e7b9c3e 100644 --- a/weed/replication/sink/azuresink/azure_sink.go +++ b/weed/replication/sink/azuresink/azure_sink.go @@ -115,7 +115,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error { } var writeErr error - _, readErr := util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) { + readErr := util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { _, writeErr = appendBlobURL.AppendBlock(context.Background(), bytes.NewReader(data), azblob.AppendBlobAccessConditions{}, nil) }) diff --git a/weed/replication/sink/b2sink/b2_sink.go b/weed/replication/sink/b2sink/b2_sink.go index e687170dd..0f8313638 100644 --- a/weed/replication/sink/b2sink/b2_sink.go +++ b/weed/replication/sink/b2sink/b2_sink.go @@ -103,7 +103,7 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error { } var writeErr error - _, readErr := util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) { + readErr := util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { _, err := writer.Write(data) if err != nil { writeErr = err diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index 232b68fec..512fbd46b 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -50,6 +50,7 @@ func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, dir stri Mtime: sourceChunk.Mtime, ETag: sourceChunk.ETag, SourceFileId: sourceChunk.GetFileIdString(), + CipherKey: sourceChunk.CipherKey, }, nil } @@ -95,8 +96,8 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, dir string) glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header) - uploadResult, err := operation.Upload(fileUrl, filename, readCloser, - "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth) + // fetch data as is, regardless whether it is encrypted or not + uploadResult, err := operation.Upload(fileUrl, filename, false, readCloser, "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) return "", fmt.Errorf("upload data: %v", err) diff --git a/weed/replication/sink/gcssink/gcs_sink.go b/weed/replication/sink/gcssink/gcs_sink.go index d5b1e137f..dd8567e0e 100644 --- a/weed/replication/sink/gcssink/gcs_sink.go +++ b/weed/replication/sink/gcssink/gcs_sink.go @@ -6,13 +6,14 @@ import ( "os" "cloud.google.com/go/storage" + "google.golang.org/api/option" + "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/replication/sink" "github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/chrislusf/seaweedfs/weed/util" - "google.golang.org/api/option" ) type GcsSink struct { @@ -100,7 +101,7 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error { return err } - _, err = util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) { + err = util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { wc.Write(data) }) diff --git a/weed/replication/sink/s3sink/s3_write.go b/weed/replication/sink/s3sink/s3_write.go index 60885c30c..dcc041642 100644 --- a/weed/replication/sink/s3sink/s3_write.go +++ b/weed/replication/sink/s3sink/s3_write.go @@ -162,6 +162,6 @@ func (s3sink *S3Sink) buildReadSeeker(chunk *filer2.ChunkView) (io.ReadSeeker, e return nil, err } buf := make([]byte, chunk.Size) - util.ReadUrl(fileUrl, chunk.Offset, int(chunk.Size), buf, true) + util.ReadUrl(fileUrl, nil, false, chunk.Offset, int(chunk.Size), buf) return bytes.NewReader(buf), nil } diff --git a/weed/server/common.go b/weed/server/common.go index 31a9a73b8..d7ab8d1ee 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -134,7 +134,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st } debug("upload file to store", url) - uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType, pairMap, assignResult.Auth) + uploadResult, err := operation.Upload(url, fname, false, bytes.NewReader(data), isGzipped, mimeType, pairMap, assignResult.Auth) if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) return diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 35539acca..b000bf3a6 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -338,5 +338,6 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb. MaxMb: uint32(fs.option.MaxMB), DirBuckets: fs.filer.DirBucketsPath, DirQueues: fs.filer.DirQueuesPath, + Cipher: fs.filer.Cipher, }, nil } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 57caeb6d4..4c493b6b3 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -46,6 +46,7 @@ type FilerOption struct { DisableHttp bool Port uint32 recursiveDelete bool + Cipher bool } type FilerServer struct { @@ -67,6 +68,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) } fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000) + fs.filer.Cipher = option.Cipher go fs.filer.KeepConnectedToMaster() diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index ba21298ba..aff2b9159 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -14,6 +14,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -93,7 +94,7 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request, return } - if fs.option.RedirectOnRead { + if fs.option.RedirectOnRead && entry.Chunks[0].CipherKey == nil { stats.FilerRequestCounter.WithLabelValues("redirect").Inc() http.Redirect(w, r, urlString, http.StatusFound) return @@ -136,7 +137,27 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request, w.Header().Set("Content-Type", entry.Attr.Mime) } w.WriteHeader(resp.StatusCode) - io.Copy(w, resp.Body) + if entry.Chunks[0].CipherKey == nil { + io.Copy(w, resp.Body) + } else { + fs.writeEncryptedChunk(w, resp, entry.Chunks[0]) + } +} + +func (fs *FilerServer) writeEncryptedChunk(w http.ResponseWriter, resp *http.Response, chunk *filer_pb.FileChunk) { + encryptedData, err := ioutil.ReadAll(resp.Body) + if err != nil { + glog.V(1).Infof("read encrypted %s failed, err: %v", chunk.FileId, err) + w.WriteHeader(http.StatusNotFound) + return + } + decryptedData, err := util.Decrypt(encryptedData, util.CipherKey(chunk.CipherKey)) + if err != nil { + glog.V(1).Infof("decrypt %s failed, err: %v", chunk.FileId, err) + w.WriteHeader(http.StatusNotFound) + return + } + w.Write(decryptedData) } func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) { diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 45cad480d..c8eadf82a 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -182,7 +182,7 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht stats.FilerRequestHistogram.WithLabelValues("postAutoChunkUpload").Observe(time.Since(start).Seconds()) }() - uploadResult, uploadError := operation.Upload(urlLocation, fileName, limitedReader, false, contentType, nil, auth) + uploadResult, uploadError := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, nil, auth) if uploadError != nil { return 0, uploadError } diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index a07f6be01..9451fdf99 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -32,6 +32,7 @@ type WebDavOption struct { Collection string Uid uint32 Gid uint32 + Cipher bool } type WebDavServer struct { @@ -418,7 +419,7 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) bufReader := bytes.NewReader(buf) - uploadResult, err := operation.Upload(fileUrl, f.name, bufReader, false, "", nil, auth) + uploadResult, err := operation.Upload(fileUrl, f.name, f.fs.option.Cipher, bufReader, false, "", nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, err) return 0, fmt.Errorf("upload data: %v", err) @@ -429,11 +430,12 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { } chunk := &filer_pb.FileChunk{ - FileId: fileId, - Offset: f.off, - Size: uint64(len(buf)), - Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + FileId: fileId, + Offset: f.off, + Size: uint64(len(buf)), + Mtime: time.Now().UnixNano(), + ETag: uploadResult.ETag, + CipherKey: uploadResult.CipherKey, } f.entry.Chunks = append(f.entry.Chunks, chunk) diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index b195b48ed..332ec4d65 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -72,9 +72,8 @@ func ReplicatedWrite(masterNode string, s *storage.Store, } } - _, err := operation.Upload(u.String(), - string(n.Name), bytes.NewReader(n.Data), n.IsGzipped(), string(n.Mime), - pairMap, jwt) + // volume server do not know about encryption + _, err := operation.Upload(u.String(), string(n.Name), false, bytes.NewReader(n.Data), n.IsGzipped(), string(n.Mime), pairMap, jwt) return err }); err != nil { size = 0 diff --git a/weed/util/cipher.go b/weed/util/cipher.go new file mode 100644 index 000000000..f044c2ca3 --- /dev/null +++ b/weed/util/cipher.go @@ -0,0 +1,60 @@ +package util + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "errors" + "io" + + "github.com/chrislusf/seaweedfs/weed/glog" +) + +type CipherKey []byte + +func GenCipherKey() CipherKey { + key := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + glog.Fatalf("random key gen: %v", err) + } + return CipherKey(key) +} + +func Encrypt(plaintext []byte, key CipherKey) ([]byte, error) { + c, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(c) + if err != nil { + return nil, err + } + + nonce := make([]byte, gcm.NonceSize()) + if _, err = io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + return gcm.Seal(nonce, nonce, plaintext, nil), nil +} + +func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error) { + c, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(c) + if err != nil { + return nil, err + } + + nonceSize := gcm.NonceSize() + if len(ciphertext) < nonceSize { + return nil, errors.New("ciphertext too short") + } + + nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] + return gcm.Open(nil, nonce, ciphertext, nil) +} diff --git a/weed/util/http_util.go b/weed/util/http_util.go index f819d8497..833db910c 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -189,13 +189,21 @@ func NormalizeUrl(url string) string { return "http://" + url } -func ReadUrl(fileUrl string, offset int64, size int, buf []byte, isReadRange bool) (int64, error) { +func ReadUrl(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) { + + if cipherKey != nil { + var n int + err := readEncryptedUrl(fileUrl, cipherKey, offset, size, func(data []byte) { + n = copy(buf, data) + }) + return int64(n), err + } req, err := http.NewRequest("GET", fileUrl, nil) if err != nil { return 0, err } - if isReadRange { + if !isFullChunk { req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) } else { req.Header.Set("Accept-Encoding", "gzip") @@ -250,43 +258,64 @@ func ReadUrl(fileUrl string, offset int64, size int, buf []byte, isReadRange boo return n, err } -func ReadUrlAsStream(fileUrl string, offset int64, size int, fn func(data []byte)) (int64, error) { +func ReadUrlAsStream(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, fn func(data []byte)) error { + + if cipherKey != nil { + return readEncryptedUrl(fileUrl, cipherKey, offset, size, fn) + } req, err := http.NewRequest("GET", fileUrl, nil) if err != nil { - return 0, err + return err + } + + if !isFullChunk { + req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) } - req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) r, err := client.Do(req) if err != nil { - return 0, err + return err } defer CloseResponse(r) if r.StatusCode >= 400 { - return 0, fmt.Errorf("%s: %s", fileUrl, r.Status) + return fmt.Errorf("%s: %s", fileUrl, r.Status) } var ( m int - n int64 ) buf := make([]byte, 64*1024) for { m, err = r.Body.Read(buf) fn(buf[:m]) - n += int64(m) if err == io.EOF { - return n, nil + return nil } if err != nil { - return n, err + return err } } } +func readEncryptedUrl(fileUrl string, cipherKey []byte, offset int64, size int, fn func(data []byte)) error { + encryptedData, err := Get(fileUrl) + if err != nil { + return fmt.Errorf("fetch %s: %v", fileUrl, err) + } + decryptedData, err := Decrypt(encryptedData, CipherKey(cipherKey)) + if err != nil { + return fmt.Errorf("decrypt %s: %v", fileUrl, err) + } + if len(decryptedData) < int(offset)+size { + return fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size) + } + fn(decryptedData[int(offset) : int(offset)+size]) + return nil +} + func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) { req, err := http.NewRequest("GET", fileUrl, nil)