Merge pull request #145 from dyeldandi/incrementalExport

Using LastModified in export to export only new files and fill up ctime and mtime in tar archive

go/weed/export.go

@@ -21,10 +21,11 @@ func init() {
 
 const (
 	defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
+	timeFormat = "2006-01-02T15:04:05"
 )
 
 var cmdExport = &Command{
-	UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}",
+	UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='"+timeFormat+"'",
 	Short:     "list or export files from one volume data file",
 	Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
 
@@ -39,19 +40,32 @@ var (
 	exportVolumeId   = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
 	dest             = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
 	format           = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
+	newer            = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone")
 	tarFh            *tar.Writer
 	tarHeader        tar.Header
 	fnTmpl           *template.Template
 	fnTmplBuf        = bytes.NewBuffer(nil)
+	newerThan        time.Time
+	newerThanUnix    int64 = -1
+	localLocation, _    = time.LoadLocation("Local")
 )
 
 func runExport(cmd *Command, args []string) bool {
 
+	var err error
+
+	if *newer != "" {
+		if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil {
+			fmt.Println("cannot parse 'newer' argument: " + err.Error())
+			return false
+		}
+		newerThanUnix = newerThan.Unix()
+	}
+
 	if *exportVolumeId == -1 {
 		return false
 	}
 
-	var err error
 	if *dest != "" {
 		if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
 			fmt.Println("the output file", *dest, "should be '-' or end with .tar")
@@ -109,6 +123,11 @@ func runExport(cmd *Command, args []string) bool {
 			glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
 				n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
 			if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
+				if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
+					glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
+						n.LastModified, newerThanUnix)
+					return nil
+				}
 				return walker(vid, n, version)
 			}
 			if !ok {
@@ -151,6 +170,12 @@ func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (e
 		}
 
 		tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
+		if n.HasLastModifiedDate() {
+			tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
+		} else {
+			tarHeader.ModTime = time.Unix(0,0)
+		}
+		tarHeader.ChangeTime = tarHeader.ModTime
 		if err = tarFh.WriteHeader(&tarHeader); err != nil {
 			return err
 		}