extend export command to show tombstone + change output format to CSV

weed/command/export.go

@@ -52,6 +52,7 @@ var (
 	output      = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
 	format      = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename formatted with {{.Mime}} {{.Id}} {{.Name}} {{.Ext}}")
 	newer       = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone, e.g. 2006-01-02T15:04:05")
+	showDeleted = cmdExport.Flag.Bool("deleted", false, "export deleted files. only applies if -o is not specified")
 
 	tarOutputFile          *tar.Writer
 	tarHeader              tar.Header
@@ -62,6 +63,24 @@ var (
 	localLocation, _             = time.LoadLocation("Local")
 )
 
+func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Version, deleted bool) {
+	key := storage.NewFileIdFromNeedle(vid, n).String()
+	size := n.DataSize
+	if version == storage.Version1 {
+		size = n.Size
+	}
+	fmt.Printf("\"%s\",\"%s\",%d,%t,%s,%s,%s,%t\n",
+		key,
+		strings.Replace(string(n.Name), "\"", "\"\"", -1),
+		size,
+		n.IsGzipped(),
+		n.Mime,
+		n.LastModifiedString(),
+		n.Ttl.String(),
+		deleted,
+	)
+}
+
 func runExport(cmd *Command, args []string) bool {
 
 	var err error
@@ -125,6 +144,10 @@ func runExport(cmd *Command, args []string) bool {
 
 	var version storage.Version
 
+	if tarOutputFile == nil {
+		fmt.Printf("key,name,size,gzip,mime,modified,ttl,deleted\n")
+	}
+
 	err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
 		storage.NeedleMapInMemory,
 		func(superBlock storage.SuperBlock) error {
@@ -140,9 +163,24 @@ func runExport(cmd *Command, args []string) bool {
 						n.LastModified, newerThanUnix)
 					return nil
 				}
-				return walker(vid, n, version)
+
+				if tarOutputFile != nil {
+					return writeFile(vid, n, version)
+				} else {
+					printNeedle(vid, n, version, false)
+					return nil
+				}
 			}
 			if !ok {
+				if *showDeleted && tarOutputFile == nil {
+					if (n.DataSize > 0) {
+						printNeedle(vid, n, version, true)
+					} else {
+						n.Name = []byte("*tombstone")
+						printNeedle(vid, n, version, true)
+					}
+				}
+
 				glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
 			} else {
 				glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
@@ -163,9 +201,8 @@ type nameParams struct {
 	Ext  string
 }
 
-func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
+func writeFile(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
 	key := storage.NewFileIdFromNeedle(vid, n).String()
-	if tarOutputFile != nil {
 	fileNameTemplateBuffer.Reset()
 	if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
 		nameParams{
@@ -196,18 +233,5 @@ func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (e
 		return err
 	}
 	_, err = tarOutputFile.Write(n.Data)
-	} else {
-		size := n.DataSize
-		if version == storage.Version1 {
-			size = n.Size
-		}
-		fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
-			key,
-			n.Name,
-			size,
-			n.IsGzipped(),
-			n.Mime,
-		)
-	}
 	return
 }

weed/storage/needle.go

@@ -262,3 +262,7 @@ func ParseKeyHash(key_hash_string string) (uint64, uint32, error) {
 	}
 	return key, uint32(hash), nil
 }
+
+func (n *Needle) LastModifiedString() string {
+	return time.Unix(int64(n.LastModified), 0).Format("2006-01-02T15:04:05")
+}