Browse Source

Metadata holds mimetype, sha256sum, archiveFiles

pull/49/head v0.8.2
andreimarcu 9 years ago
parent
commit
6e33fe6ac8
  1. 5
      delete.go
  2. 115
      display.go
  3. 10
      expiry.go
  4. 21
      fileserve.go
  5. 198
      meta.go
  6. 6
      torrent.go
  7. 41
      upload.go

5
delete.go

@ -23,14 +23,13 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} }
// Ensure delete key is correct // Ensure delete key is correct
deleteKey, err := metadataGetDeleteKey(filename)
metadata, err := metadataRead(filename)
if err != nil { if err != nil {
unauthorizedHandler(c, w, r) // 401 - no metadata available unauthorizedHandler(c, w, r) // 401 - no metadata available
return return
} }
if deleteKey == requestKey {
if metadata.DeleteKey == requestKey {
fileDelErr := os.Remove(filePath) fileDelErr := os.Remove(filePath)
metaDelErr := os.Remove(metaPath) metaDelErr := os.Remove(metaPath)

115
display.go

@ -1,23 +1,16 @@
package main package main
import ( import (
"archive/tar"
"archive/zip"
"compress/bzip2"
"compress/gzip"
"encoding/json" "encoding/json"
"io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"bitbucket.org/taruti/mimemagic"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"github.com/flosch/pongo2" "github.com/flosch/pongo2"
"github.com/microcosm-cc/bluemonday" "github.com/microcosm-cc/bluemonday"
@ -30,21 +23,24 @@ const maxDisplayFileSizeBytes = 1024 * 512
func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"] fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName) filePath := path.Join(Config.filesDir, fileName)
fileInfo, err := os.Stat(filePath)
if !fileExistsAndNotExpired(fileName) {
err := checkFile(fileName)
if err == NotFoundErr {
notFoundHandler(c, w, r) notFoundHandler(c, w, r)
return return
} }
expiry, _ := metadataGetExpiry(fileName)
metadata, err := metadataRead(fileName)
if err != nil {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
}
var expiryHuman string var expiryHuman string
if expiry != neverExpire {
expiryHuman = humanize.RelTime(time.Now(), expiry, "", "")
if metadata.Expiry != neverExpire {
expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "")
} }
sizeHuman := humanize.Bytes(uint64(fileInfo.Size()))
sizeHuman := humanize.Bytes(uint64(metadata.Size))
extra := make(map[string]string) extra := make(map[string]string)
files := []string{}
file, _ := os.Open(filePath) file, _ := os.Open(filePath)
defer file.Close() defer file.Close()
@ -52,15 +48,15 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
header := make([]byte, 512) header := make([]byte, 512)
file.Read(header) file.Read(header)
mimetype := mimemagic.Match("", header)
extension := strings.TrimPrefix(filepath.Ext(fileName), ".") extension := strings.TrimPrefix(filepath.Ext(fileName), ".")
if strings.EqualFold("application/json", r.Header.Get("Accept")) { if strings.EqualFold("application/json", r.Header.Get("Accept")) {
js, _ := json.Marshal(map[string]string{ js, _ := json.Marshal(map[string]string{
"filename": fileName, "filename": fileName,
"mimetype": mimetype,
"expiry": strconv.FormatInt(expiry.Unix(), 10),
"size": strconv.FormatInt(fileInfo.Size(), 10),
"expiry": strconv.FormatInt(metadata.Expiry.Unix(), 10),
"size": strconv.FormatInt(metadata.Size, 10),
"mimetype": metadata.Mimetype,
"sha256sum": metadata.Sha256sum,
}) })
w.Write(js) w.Write(js)
return return
@ -68,81 +64,20 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
var tpl *pongo2.Template var tpl *pongo2.Template
if strings.HasPrefix(mimetype, "image/") {
if strings.HasPrefix(metadata.Mimetype, "image/") {
tpl = Templates["display/image.html"] tpl = Templates["display/image.html"]
} else if strings.HasPrefix(mimetype, "video/") {
tpl = Templates["display/video.html"]
} else if strings.HasPrefix(mimetype, "audio/") {
tpl = Templates["display/audio.html"]
} else if mimetype == "application/pdf" {
tpl = Templates["display/pdf.html"]
} else if mimetype == "application/x-tar" {
f, _ := os.Open(filePath)
defer f.Close()
tReadr := tar.NewReader(f)
for {
header, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg {
files = append(files, header.Name)
}
}
sort.Strings(files)
} else if mimetype == "application/x-gzip" {
f, _ := os.Open(filePath)
defer f.Close()
gzf, err := gzip.NewReader(f)
if err == nil {
tReadr := tar.NewReader(gzf)
for {
header, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg {
files = append(files, header.Name)
}
}
sort.Strings(files)
}
} else if mimetype == "application/x-bzip" {
f, _ := os.Open(filePath)
defer f.Close()
bzf := bzip2.NewReader(f)
tReadr := tar.NewReader(bzf)
for {
header, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if header.Typeflag == tar.TypeDir || header.Typeflag == tar.TypeReg {
files = append(files, header.Name)
}
}
sort.Strings(files)
} else if strings.HasPrefix(metadata.Mimetype, "video/") {
tpl = Templates["display/video.html"]
} else if mimetype == "application/zip" {
f, _ := os.Open(filePath)
defer f.Close()
} else if strings.HasPrefix(metadata.Mimetype, "audio/") {
tpl = Templates["display/audio.html"]
zf, err := zip.NewReader(f, fileInfo.Size())
if err == nil {
for _, f := range zf.File {
files = append(files, f.Name)
}
}
} else if metadata.Mimetype == "application/pdf" {
tpl = Templates["display/pdf.html"]
} else if supportedBinExtension(extension) { } else if supportedBinExtension(extension) {
if fileInfo.Size() < maxDisplayFileSizeBytes {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath) bytes, err := ioutil.ReadFile(filePath)
if err == nil { if err == nil {
extra["extension"] = extension extra["extension"] = extension
@ -152,7 +87,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} }
} }
} else if extension == "md" { } else if extension == "md" {
if fileInfo.Size() < maxDisplayFileSizeBytes {
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := ioutil.ReadFile(filePath) bytes, err := ioutil.ReadFile(filePath)
if err == nil { if err == nil {
unsafe := blackfriday.MarkdownCommon(bytes) unsafe := blackfriday.MarkdownCommon(bytes)
@ -170,12 +105,12 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} }
err = tpl.ExecuteWriter(pongo2.Context{ err = tpl.ExecuteWriter(pongo2.Context{
"mime": mimetype,
"mime": metadata.Mimetype,
"filename": fileName, "filename": fileName,
"size": sizeHuman, "size": sizeHuman,
"expiry": expiryHuman, "expiry": expiryHuman,
"extra": extra, "extra": extra,
"files": files,
"files": metadata.ArchiveFiles,
}, w) }, w)
if err != nil { if err != nil {

10
expiry.go

@ -13,7 +13,11 @@ func isTsExpired(ts time.Time) bool {
} }
// Determine if the given filename is expired // Determine if the given filename is expired
func isFileExpired(filename string) bool {
exp, _ := metadataGetExpiry(filename)
return isTsExpired(exp)
func isFileExpired(filename string) (bool, error) {
metadata, err := metadataRead(filename)
if err != nil {
return false, err
}
return isTsExpired(metadata.Expiry), nil
} }

21
fileserve.go

@ -13,9 +13,13 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"] fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName) filePath := path.Join(Config.filesDir, fileName)
if !fileExistsAndNotExpired(fileName) {
err := checkFile(fileName)
if err == NotFoundErr {
notFoundHandler(c, w, r) notFoundHandler(c, w, r)
return return
} else if err == BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
} }
if !Config.allowHotlink { if !Config.allowHotlink {
@ -55,19 +59,24 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
} }
} }
func fileExistsAndNotExpired(filename string) bool {
func checkFile(filename string) error {
filePath := path.Join(Config.filesDir, filename) filePath := path.Join(Config.filesDir, filename)
_, err := os.Stat(filePath) _, err := os.Stat(filePath)
if err != nil { if err != nil {
return false
return NotFoundErr
}
expired, err := isFileExpired(filename)
if err != nil {
return err
} }
if isFileExpired(filename) {
if expired {
os.Remove(path.Join(Config.filesDir, filename)) os.Remove(path.Join(Config.filesDir, filename))
os.Remove(path.Join(Config.metaDir, filename)) os.Remove(path.Join(Config.metaDir, filename))
return false
return NotFoundErr
} }
return true
return nil
} }

198
meta.go

@ -1,84 +1,188 @@
package main package main
import ( import (
"bufio"
"archive/tar"
"archive/zip"
"compress/bzip2"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors" "errors"
"fmt"
"io"
"io/ioutil"
"os" "os"
"path" "path"
"strconv"
"sort"
"time" "time"
)
// Write metadata from Upload struct to file
func metadataWrite(filename string, upload *Upload) error {
// Write metadata, overwriting if necessary
"bitbucket.org/taruti/mimemagic"
"github.com/dchest/uniuri"
)
file, err := os.Create(path.Join(Config.metaDir, upload.Filename))
if err != nil {
return err
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
} }
defer file.Close()
w := bufio.NewWriter(file)
type Metadata struct {
DeleteKey string
Sha256sum string
Mimetype string
Size int64
Expiry time.Time
ArchiveFiles []string
}
fmt.Fprintln(w, upload.Expiry.Unix())
fmt.Fprintln(w, upload.DeleteKey)
var NotFoundErr = errors.New("File not found.")
var BadMetadata = errors.New("Corrupted metadata.")
return w.Flush()
func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
file, err := os.Open(path.Join(Config.filesDir, fName))
fileInfo, err := os.Stat(path.Join(Config.filesDir, fName))
if err != nil {
return
} }
defer file.Close()
// Return list of strings from a filename's metadata source
func metadataRead(filename string) ([]string, error) {
file, err := os.Open(path.Join(Config.metaDir, filename))
m.Size = fileInfo.Size()
m.Expiry = exp
if err != nil {
return nil, err
if delKey == "" {
m.DeleteKey = uniuri.NewLen(30)
} else {
m.DeleteKey = delKey
} }
defer file.Close()
// Get first 512 bytes for mimetype detection
header := make([]byte, 512)
file.Read(header)
var lines []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
m.Mimetype = mimemagic.Match("", header)
// Compute the sha256sum
hasher := sha256.New()
file.Seek(0, 0)
_, err = io.Copy(hasher, file)
if err == nil {
m.Sha256sum = hex.EncodeToString(hasher.Sum(nil))
}
file.Seek(0, 0)
// If archive, grab list of filenames
if m.Mimetype == "application/x-tar" {
tReadr := tar.NewReader(file)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
} else if m.Mimetype == "application/x-gzip" {
gzf, err := gzip.NewReader(file)
if err == nil {
tReadr := tar.NewReader(gzf)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
}
} else if m.Mimetype == "application/x-bzip" {
bzf := bzip2.NewReader(file)
tReadr := tar.NewReader(bzf)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
} else if m.Mimetype == "application/zip" {
zf, err := zip.NewReader(file, m.Size)
if err == nil {
for _, f := range zf.File {
m.ArchiveFiles = append(m.ArchiveFiles, f.Name)
}
}
sort.Strings(m.ArchiveFiles)
} }
return lines, scanner.Err()
return
} }
func metadataGetExpiry(filename string) (expiry time.Time, err error) {
metadata, err := metadataRead(filename)
func metadataWrite(filename string, metadata *Metadata) error {
file, err := os.Create(path.Join(Config.metaDir, filename))
if err != nil {
return err
}
defer file.Close()
// XXX in this case it's up to the caller to determine proper behavior
// for a nonexistant metadata file or broken file
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
byt, err := json.Marshal(mjson)
if err != nil { if err != nil {
return
return err
} }
if len(metadata) < 1 {
err = errors.New("ERR: Metadata file does not contain expiry")
return
_, err = file.Write(byt)
if err != nil {
return err
} }
expirySecs, err := strconv.ParseInt(metadata[0], 10, 64)
expiry = time.Unix(expirySecs, 0)
return
return nil
} }
func metadataGetDeleteKey(filename string) (string, error) {
metadata, err := metadataRead(filename)
if len(metadata) < 2 {
err := errors.New("ERR: Metadata file does not contain deletion key")
return "", err
func metadataRead(filename string) (metadata Metadata, err error) {
b, err := ioutil.ReadFile(path.Join(Config.metaDir, filename))
if err != nil {
// Metadata does not exist, generate one
newMData, err := generateMetadata(filename, neverExpire, "")
if err != nil {
return metadata, err
}
metadataWrite(filename, &newMData)
b, err = ioutil.ReadFile(path.Join(Config.metaDir, filename))
if err != nil {
return metadata, BadMetadata
} }
}
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil { if err != nil {
return "", err
} else {
return metadata[1], err
return metadata, BadMetadata
} }
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
return
} }

6
torrent.go

@ -80,9 +80,13 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"] fileName := c.URLParams["name"]
filePath := path.Join(Config.filesDir, fileName) filePath := path.Join(Config.filesDir, fileName)
if !fileExistsAndNotExpired(fileName) {
err := checkFile(fileName)
if err == NotFoundErr {
notFoundHandler(c, w, r) notFoundHandler(c, w, r)
return return
} else if err == BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
} }
encoded, err := createTorrent(fileName, filePath) encoded, err := createTorrent(fileName, filePath)

41
upload.go

@ -41,9 +41,7 @@ type UploadRequest struct {
// Metadata associated with a file as it would actually be stored // Metadata associated with a file as it would actually be stored
type Upload struct { type Upload struct {
Filename string // Final filename on disk Filename string // Final filename on disk
Size int64
Expiry time.Time // Unix timestamp of expiry, 0=never
DeleteKey string // Deletion key, one generated if not provided
Metadata Metadata
} }
func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@ -246,34 +244,35 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
defer dst.Close() defer dst.Close()
// Get the rest of the metadata needed for storage // Get the rest of the metadata needed for storage
var expiry time.Time
if upReq.expiry == 0 { if upReq.expiry == 0 {
upload.Expiry = neverExpire
expiry = neverExpire
} else { } else {
upload.Expiry = time.Now().Add(upReq.expiry)
expiry = time.Now().Add(upReq.expiry)
} }
// If no delete key specified, pick a random one.
if upReq.deletionKey == "" {
upload.DeleteKey = uniuri.NewLen(30)
} else {
upload.DeleteKey = upReq.deletionKey
}
metadataWrite(upload.Filename, &upload)
bytes, err := io.Copy(dst, io.MultiReader(bytes.NewReader(header), upReq.src)) bytes, err := io.Copy(dst, io.MultiReader(bytes.NewReader(header), upReq.src))
if bytes == 0 { if bytes == 0 {
os.Remove(path.Join(Config.filesDir, upload.Filename)) os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
return upload, errors.New("Empty file") return upload, errors.New("Empty file")
} else if err != nil { } else if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename)) os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
return return
} }
upload.Size = bytes
upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
return
}
err = metadataWrite(upload.Filename, &upload.Metadata)
if err != nil {
os.Remove(path.Join(Config.filesDir, upload.Filename))
os.Remove(path.Join(Config.metaDir, upload.Filename))
return
}
return return
} }
@ -285,9 +284,11 @@ func generateJSONresponse(upload Upload) []byte {
js, _ := json.Marshal(map[string]string{ js, _ := json.Marshal(map[string]string{
"url": Config.siteURL + upload.Filename, "url": Config.siteURL + upload.Filename,
"filename": upload.Filename, "filename": upload.Filename,
"delete_key": upload.DeleteKey,
"expiry": strconv.FormatInt(upload.Expiry.Unix(), 10),
"size": strconv.FormatInt(upload.Size, 10),
"delete_key": upload.Metadata.DeleteKey,
"expiry": strconv.FormatInt(upload.Metadata.Expiry.Unix(), 10),
"size": strconv.FormatInt(upload.Metadata.Size, 10),
"mimetype": upload.Metadata.Mimetype,
"sha256sum": upload.Metadata.Sha256sum,
}) })
return js return js

Loading…
Cancel
Save