mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Merge pull request #145 from dyeldandi/incrementalExport
Using LastModified in export to export only new files and fill up ctime and mtime in tar archive
This commit is contained in:
commit
ebf8e9f663
|
@ -21,10 +21,11 @@ func init() {
|
|||
|
||||
const (
|
||||
defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
|
||||
timeFormat = "2006-01-02T15:04:05"
|
||||
)
|
||||
|
||||
var cmdExport = &Command{
|
||||
UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}",
|
||||
UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='"+timeFormat+"'",
|
||||
Short: "list or export files from one volume data file",
|
||||
Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
|
||||
|
||||
|
@ -39,19 +40,32 @@ var (
|
|||
exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
|
||||
dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
|
||||
format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
|
||||
newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone")
|
||||
tarFh *tar.Writer
|
||||
tarHeader tar.Header
|
||||
fnTmpl *template.Template
|
||||
fnTmplBuf = bytes.NewBuffer(nil)
|
||||
newerThan time.Time
|
||||
newerThanUnix int64 = -1
|
||||
localLocation, _ = time.LoadLocation("Local")
|
||||
)
|
||||
|
||||
func runExport(cmd *Command, args []string) bool {
|
||||
|
||||
var err error
|
||||
|
||||
if *newer != "" {
|
||||
if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil {
|
||||
fmt.Println("cannot parse 'newer' argument: " + err.Error())
|
||||
return false
|
||||
}
|
||||
newerThanUnix = newerThan.Unix()
|
||||
}
|
||||
|
||||
if *exportVolumeId == -1 {
|
||||
return false
|
||||
}
|
||||
|
||||
var err error
|
||||
if *dest != "" {
|
||||
if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
|
||||
fmt.Println("the output file", *dest, "should be '-' or end with .tar")
|
||||
|
@ -109,6 +123,11 @@ func runExport(cmd *Command, args []string) bool {
|
|||
glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
|
||||
n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
|
||||
if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
|
||||
if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
|
||||
glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
|
||||
n.LastModified, newerThanUnix)
|
||||
return nil
|
||||
}
|
||||
return walker(vid, n, version)
|
||||
}
|
||||
if !ok {
|
||||
|
@ -151,6 +170,12 @@ func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (e
|
|||
}
|
||||
|
||||
tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
|
||||
if n.HasLastModifiedDate() {
|
||||
tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
|
||||
} else {
|
||||
tarHeader.ModTime = time.Unix(0,0)
|
||||
}
|
||||
tarHeader.ChangeTime = tarHeader.ModTime
|
||||
if err = tarFh.WriteHeader(&tarHeader); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue