From a10b838f4d182c52758edac728c63cd70f10f5f2 Mon Sep 17 00:00:00 2001 From: Matt Hazinski Date: Sun, 27 Sep 2015 22:17:12 -0400 Subject: [PATCH] Add preliminary metadata support --- .gitignore | 1 + expiry.go | 44 ++++++++++++++++++++++++++ fileserve.go | 12 ++++++- meta.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++ server.go | 30 +++++++++++++----- upload.go | 56 ++++++++++++++++++++++++++++++--- 6 files changed, 220 insertions(+), 12 deletions(-) create mode 100644 expiry.go create mode 100644 meta.go diff --git a/.gitignore b/.gitignore index f6fb889..79de414 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ _testmain.go linx-server files/ +meta/ diff --git a/expiry.go b/expiry.go new file mode 100644 index 0000000..6ffd779 --- /dev/null +++ b/expiry.go @@ -0,0 +1,44 @@ +package main + +import ( + "time" +) + +// Get what the unix timestamp will be in "seconds". +func getFutureTimestamp(seconds int32) (ts int32) { + now := int32(time.Now().Unix()) + + if seconds == 0 { + ts = 0 + } else { + ts = now + seconds + } + + return +} + +// Determine if a file with expiry set to "ts" has expired yet +func isTsExpired(ts int32) (expired bool) { + now := int32(time.Now().Unix()) + + if ts == 0 { + expired = false + } else if now > ts { + expired = true + } else { + expired = false + } + + return +} + +// Determine if the given filename is expired +func isFileExpired(filename string) (bool, error) { + exp, err := metadataGetExpiry(filename) + + if err != nil { + return true, err + } else { + return isTsExpired(exp), err + } +} diff --git a/fileserve.go b/fileserve.go index ba99455..8fb0be1 100644 --- a/fileserve.go +++ b/fileserve.go @@ -18,7 +18,17 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { return } - // plug file expiry checking here + expired, expErr := isFileExpired(fileName) + + if expErr != nil { + // Error reading metadata, pretend it's expired + notFoundHandler(c, w, r) + // TODO log error internally + return + } else if expired { + notFoundHandler(c, w, r) + // TODO delete the file + } http.ServeFile(w, r, filePath) } diff --git a/meta.go b/meta.go new file mode 100644 index 0000000..7874611 --- /dev/null +++ b/meta.go @@ -0,0 +1,89 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "os" + "path" + "strconv" +) + +// Write metadata from Upload struct to file +func metadataWrite(filename string, upload *Upload) error { + // Write metadata, overwriting if necessary + + file, err := os.Create(path.Join(Config.metaDir, upload.Filename)) + if err != nil { + return err + } + + defer file.Close() + + w := bufio.NewWriter(file) + + fmt.Fprintln(w, upload.Expiry) + fmt.Fprintln(w, upload.DeleteKey) + fmt.Fprintln(w, upload.DebugInfo) + + return w.Flush() +} + +// Return list of strings from a filename's metadata source +func metadataRead(filename string) ([]string, error) { + file, err := os.Create(path.Join(Config.metaDir, filename)) + + if err != nil { + return nil, err + } + + defer file.Close() + + var lines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + return lines, scanner.Err() +} + +func metadataGetExpiry(filename string) (int32, error) { + metadata, err := metadataRead(filename) + + if len(metadata) < 1 { + err := errors.New("ERR: Metadata file does not contain expiry") + return 0, err + } + + // XXX in this case it's up to the caller to determine proper behavior + // for a nonexistant metadata file or broken file + + if err != nil { + return 0, err + } + + var expiry int64 + expiry, err = strconv.ParseInt(metadata[0], 10, 32) + + if err != nil { + return 0, err + } else { + return int32(expiry), err + } +} + +func metadataGetDeleteKey(filename string) (string, error) { + metadata, err := metadataRead(filename) + + if len(metadata) < 2 { + err := errors.New("ERR: Metadata file does not contain deletion key") + return "", err + } + + if err != nil { + return "", err + } else { + return metadata[1], err + } +} diff --git a/server.go b/server.go index 3dec36e..785af29 100644 --- a/server.go +++ b/server.go @@ -2,12 +2,12 @@ package main import ( "flag" + "fmt" "log" "net" "net/http" - "regexp" "os" - "fmt" + "regexp" "github.com/flosch/pongo2" "github.com/zenazn/goji" @@ -17,6 +17,7 @@ import ( var Config struct { bind string filesDir string + metaDir string noLogs bool siteName string siteURL string @@ -26,26 +27,41 @@ func main() { flag.StringVar(&Config.bind, "b", "127.0.0.1:8080", "host to bind to (default: 127.0.0.1:8080)") flag.StringVar(&Config.filesDir, "filespath", "files/", - "path to files directory (including trailing slash)") + "path to files directory") + flag.StringVar(&Config.metaDir, "metapath", "meta/", + "path to metadata directory") flag.BoolVar(&Config.noLogs, "nologs", false, "remove stdout output for each request") flag.StringVar(&Config.siteName, "sitename", "linx", "name of the site") flag.StringVar(&Config.siteURL, "siteurl", "http://"+Config.bind+"/", - "site base url (including trailing slash)") + "site base url") flag.Parse() if Config.noLogs { goji.Abandon(middleware.Logger) } - // make directory if needed - err := os.MkdirAll(Config.filesDir, 0755) + // make directories if needed + var err error + + err = os.MkdirAll(Config.filesDir, 0755) if err != nil { - fmt.Printf("Error: could not create files directory") + fmt.Printf("Error: could not create files directory\n") os.Exit(1) } + err = os.MkdirAll(Config.metaDir, 0700) + if err != nil { + fmt.Printf("Error: could not create metadata directory\n") + os.Exit(1) + } + + // ensure siteURL ends wth '/' + if lastChar := Config.siteURL[len(Config.siteURL)-1:]; lastChar != "/" { + Config.siteURL = Config.siteURL + "/" + } + // Template Globals pongo2.DefaultSet.Globals["sitename"] = Config.siteName diff --git a/upload.go b/upload.go index f98f9d4..5efde03 100644 --- a/upload.go +++ b/upload.go @@ -15,21 +15,55 @@ import ( "github.com/zenazn/goji/web" ) +// Describes metadata directly from the user request type UploadRequest struct { src io.Reader filename string - expiry int + expiry int32 // Seconds until expiry, 0 = never randomBarename bool + deletionKey string // Empty string if not defined } +// Metadata associated with a file as it would actually be stored type Upload struct { - Filename string - Size int64 - Expiry int + Filename string // Final filename on disk + Size int64 + Expiry int32 // Unix timestamp of expiry, 0=never + DeleteKey string // Deletion key, one generated if not provided + DebugInfo string // Optional field to store whatever +} + +func uploadHeaderProcess(r *http.Request, upReq *UploadRequest) { + // For legacy reasons + upReq.randomBarename = false + if r.Header.Get("X-Randomized-Filename") == "yes" { + upReq.randomBarename = true + } + + if r.Header.Get("X-Randomized-Barename") == "yes" { + upReq.randomBarename = true + } + + upReq.deletionKey = r.Header.Get("X-Delete-Key") + + // Get seconds until expiry. Non-integer responses never expire. + expStr := r.Header.Get("X-File-Expiry") + if expStr == "" { + upReq.expiry = 0 + } else { + expiry, err := strconv.ParseInt(expStr, 10, 32) + if err != nil { + upReq.expiry = 0 + } else { + upReq.expiry = int32(expiry) + } + } + } func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { upReq := UploadRequest{} + uploadHeaderProcess(r, &upReq) if r.Header.Get("Content-Type") == "application/octet-stream" { defer r.Body.Close() @@ -71,6 +105,7 @@ func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) { upReq := UploadRequest{} + uploadHeaderProcess(r, &upReq) defer r.Body.Close() upReq.filename = c.URLParams["name"] @@ -86,6 +121,7 @@ func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) { } func processUpload(upReq UploadRequest) (upload Upload, err error) { + // Determine the appropriate filename, then write to disk barename, extension := barePlusExt(upReq.filename) if upReq.randomBarename || len(barename) == 0 { @@ -120,6 +156,18 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { } defer dst.Close() + // Get the rest of the metadata needed for storage + upload.Expiry = getFutureTimestamp(upReq.expiry) + + // If no delete key specified, pick a random one. + if upReq.deletionKey == "" { + upload.DeleteKey = uuid.New()[:30] + } else { + upload.DeleteKey = upReq.deletionKey + } + + metadataWrite(upload.Filename, &upload) + bytes, err := io.Copy(dst, upReq.src) if err != nil { return