diff --git a/weed/command/mount.go b/weed/command/mount.go index 151c6bedf..f0514b93a 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -1,10 +1,11 @@ package command type MountOptions struct { - filer *string - dir *string - collection *string - replication *string + filer *string + dir *string + collection *string + replication *string + chunkSizeLimitMB *int } var ( @@ -13,11 +14,11 @@ var ( func init() { cmdMount.Run = runMount // break init cycle - cmdMount.IsDebug = cmdMount.Flag.Bool("debug", false, "verbose debug information") mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location") mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") mountOptions.replication = cmdMount.Flag.String("replication", "000", "replication to create to files") + mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 0, "if set, limit the chunk size in MB") } var cmdMount = &Command{ diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index fcf663e1c..7e7a35f3d 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -48,7 +48,7 @@ func runMount(cmd *Command, args []string) bool { }) err = fs.Serve(c, filesys.NewSeaweedFileSystem( - *mountOptions.filer, *mountOptions.collection, *mountOptions.replication)) + *mountOptions.filer, *mountOptions.collection, *mountOptions.replication, *mountOptions.chunkSizeLimitMB)) if err != nil { fuse.Unmount(*mountOptions.dir) } diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index da442d6c6..a78fdb47d 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -59,7 +59,13 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da Offset: offset, Data: data, }) - return nil, nil + + if pages.totalSize() >= pages.f.wfs.chunkSizeLimit { + chunk, err = pages.saveToStorage(ctx) + pages.pages = nil + glog.V(3).Infof("%s/%s add split [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } + return } chunk, err = pages.saveToStorage(ctx) diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index b9cb0210b..4b9e20b95 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -13,14 +13,16 @@ type WFS struct { listDirectoryEntriesCache *ccache.Cache collection string replication string + chunkSizeLimit int64 } -func NewSeaweedFileSystem(filer string, collection string, replication string) *WFS { +func NewSeaweedFileSystem(filer string, collection string, replication string, chunkSizeLimitMB int) *WFS { return &WFS{ filer: filer, listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)), collection: collection, replication: replication, + chunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, } }