Rework storage backends to integrate metadata

backends/localfs/localfs.go

@@ -1,65 +1,151 @@
 package localfs
 
 import (
+	"encoding/json"
 	"errors"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
+	"time"
 
 	"github.com/andreimarcu/linx-server/backends"
+	"github.com/andreimarcu/linx-server/helpers"
 	"github.com/andreimarcu/linx-server/torrent"
 )
 
 type LocalfsBackend struct {
-	basePath string
+	metaPath  string
+	filesPath string
 }
 
-func (b LocalfsBackend) Delete(key string) error {
-	return os.Remove(path.Join(b.basePath, key))
+type MetadataJSON struct {
+	DeleteKey    string   `json:"delete_key"`
+	Sha256sum    string   `json:"sha256sum"`
+	Mimetype     string   `json:"mimetype"`
+	Size         int64    `json:"size"`
+	Expiry       int64    `json:"expiry"`
+	ArchiveFiles []string `json:"archive_files,omitempty"`
+}
+
+func (b LocalfsBackend) Delete(key string) (err error) {
+	err = os.Remove(path.Join(b.filesPath, key))
+	if err != nil {
+		return
+	}
+	err = os.Remove(path.Join(b.metaPath, key))
+	return
 }
 
 func (b LocalfsBackend) Exists(key string) (bool, error) {
-	_, err := os.Stat(path.Join(b.basePath, key))
+	_, err := os.Stat(path.Join(b.filesPath, key))
 	return err == nil, err
 }
 
-func (b LocalfsBackend) Get(key string) ([]byte, error) {
-	return ioutil.ReadFile(path.Join(b.basePath, key))
+func (b LocalfsBackend) Head(key string) (metadata backends.Metadata, err error) {
+	f, err := os.Open(path.Join(b.metaPath, key))
+	if os.IsNotExist(err) {
+		return metadata, backends.NotFoundErr
+	} else if err != nil {
+		return metadata, backends.BadMetadata
+	}
+	defer f.Close()
+
+	decoder := json.NewDecoder(f)
+
+	mjson := MetadataJSON{}
+	if err := decoder.Decode(&mjson); err != nil {
+		return metadata, backends.BadMetadata
+	}
+
+	metadata.DeleteKey = mjson.DeleteKey
+	metadata.Mimetype = mjson.Mimetype
+	metadata.ArchiveFiles = mjson.ArchiveFiles
+	metadata.Sha256sum = mjson.Sha256sum
+	metadata.Expiry = time.Unix(mjson.Expiry, 0)
+	metadata.Size = mjson.Size
+
+	return
 }
 
-func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) {
-	dst, err := os.Create(path.Join(b.basePath, key))
+func (b LocalfsBackend) Get(key string) (metadata backends.Metadata, f io.ReadCloser, err error) {
+	metadata, err = b.Head(key)
 	if err != nil {
-		return 0, err
+		return
+	}
+
+	f, err = os.Open(path.Join(b.filesPath, key))
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) error {
+	metaPath := path.Join(b.metaPath, key)
+
+	mjson := MetadataJSON{
+		DeleteKey: metadata.DeleteKey,
+		Mimetype: metadata.Mimetype,
+		ArchiveFiles: metadata.ArchiveFiles,
+		Sha256sum: metadata.Sha256sum,
+		Expiry: metadata.Expiry.Unix(),
+		Size: metadata.Size,
+	}
+
+	dst, err := os.Create(metaPath)
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+
+	encoder := json.NewEncoder(dst)
+	err = encoder.Encode(mjson)
+	if err != nil {
+		os.Remove(metaPath)
+		return err
+	}
+
+	return nil
+}
+
+func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) {
+	filePath := path.Join(b.filesPath, key)
+
+	dst, err := os.Create(filePath)
+	if err != nil {
+		return
 	}
 	defer dst.Close()
 
 	bytes, err := io.Copy(dst, r)
 	if bytes == 0 {
-		b.Delete(key)
-		return bytes, errors.New("Empty file")
+		os.Remove(filePath)
+		return m, errors.New("Empty file")
 	} else if err != nil {
-		b.Delete(key)
-		return bytes, err
+		os.Remove(filePath)
+		return m, err
 	}
 
-	return bytes, err
-}
+	m.Expiry = expiry
+	m.DeleteKey = deleteKey
+	m.Size = bytes
+	m.Mimetype, _ = helpers.DetectMime(dst)
+	m.Sha256sum, _ = helpers.Sha256sum(dst)
+	m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, dst)
 
-func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) {
-	return os.Open(path.Join(b.basePath, key))
-}
+	err = b.writeMetadata(key, m)
+	if err != nil {
+		os.Remove(filePath)
+		return
+	}
 
-func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) error {
-	filePath := path.Join(b.basePath, key)
-	http.ServeFile(w, r, filePath)
-	return nil
+	return
 }
 
 func (b LocalfsBackend) Size(key string) (int64, error) {
-	fileInfo, err := os.Stat(path.Join(b.basePath, key))
+	fileInfo, err := os.Stat(path.Join(b.filesPath, key))
 	if err != nil {
 		return 0, err
 	}
@@ -79,7 +165,7 @@ func (b LocalfsBackend) GetTorrent(fileName string, url string) (t torrent.Torre
 		UrlList: []string{url},
 	}
 
-	f, err := b.Open(fileName)
+	_, f, err := b.Get(fileName)
 	if err != nil {
 		return
 	}
@@ -103,7 +189,7 @@ func (b LocalfsBackend) GetTorrent(fileName string, url string) (t torrent.Torre
 func (b LocalfsBackend) List() ([]string, error) {
 	var output []string
 
-	files, err := ioutil.ReadDir(b.basePath)
+	files, err := ioutil.ReadDir(b.filesPath)
 	if err != nil {
 		return nil, err
 	}
@@ -115,6 +201,9 @@ func (b LocalfsBackend) List() ([]string, error) {
 	return output, nil
 }
 
-func NewLocalfsBackend(basePath string) LocalfsBackend {
-	return LocalfsBackend{basePath: basePath}
+func NewLocalfsBackend(metaPath string, filesPath string) LocalfsBackend {
+	return LocalfsBackend{
+		metaPath:  metaPath,
+		filesPath: filesPath,
+	}
 }

backends/meta.go

@@ -5,11 +5,6 @@ import (
 	"time"
 )
 
-type MetaBackend interface {
-	Get(key string) (Metadata, error)
-	Put(key string, metadata *Metadata) error
-}
-
 type Metadata struct {
 	DeleteKey    string
 	Sha256sum    string

backends/metajson/metajson.go

@@ -1,70 +0,0 @@
-package metajson
-
-import (
-	"bytes"
-	"encoding/json"
-	"time"
-
-	"github.com/andreimarcu/linx-server/backends"
-)
-
-type MetadataJSON struct {
-	DeleteKey    string   `json:"delete_key"`
-	Sha256sum    string   `json:"sha256sum"`
-	Mimetype     string   `json:"mimetype"`
-	Size         int64    `json:"size"`
-	Expiry       int64    `json:"expiry"`
-	ArchiveFiles []string `json:"archive_files,omitempty"`
-}
-
-type MetaJSONBackend struct {
-	storage backends.MetaStorageBackend
-}
-
-func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error {
-	mjson := MetadataJSON{}
-	mjson.DeleteKey = metadata.DeleteKey
-	mjson.Mimetype = metadata.Mimetype
-	mjson.ArchiveFiles = metadata.ArchiveFiles
-	mjson.Sha256sum = metadata.Sha256sum
-	mjson.Expiry = metadata.Expiry.Unix()
-	mjson.Size = metadata.Size
-
-	byt, err := json.Marshal(mjson)
-	if err != nil {
-		return err
-	}
-
-	if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) {
-	b, err := m.storage.Get(key)
-	if err != nil {
-		return metadata, backends.BadMetadata
-	}
-
-	mjson := MetadataJSON{}
-
-	err = json.Unmarshal(b, &mjson)
-	if err != nil {
-		return metadata, backends.BadMetadata
-	}
-
-	metadata.DeleteKey = mjson.DeleteKey
-	metadata.Mimetype = mjson.Mimetype
-	metadata.ArchiveFiles = mjson.ArchiveFiles
-	metadata.Sha256sum = mjson.Sha256sum
-	metadata.Expiry = time.Unix(mjson.Expiry, 0)
-	metadata.Size = mjson.Size
-
-	return
-}
-
-func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend {
-	return MetaJSONBackend{storage: storage}
-}

backends/s3/s3.go

@@ -1,14 +1,16 @@
-package localfs
+package s3
 
 import (
 	"errors"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
+	"strconv"
+	"time"
 
 	"github.com/andreimarcu/linx-server/backends"
+	"github.com/andreimarcu/linx-server/helpers"
 	"github.com/andreimarcu/linx-server/torrent"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -19,7 +21,7 @@ import (
 
 type S3Backend struct {
 	bucket string
-	svc *S3
+	svc *s3.S3
 }
 
 func (b S3Backend) Delete(key string) error {
@@ -28,6 +30,9 @@ func (b S3Backend) Delete(key string) error {
 		Key: aws.String(key),
 	}
 	_, err := b.svc.DeleteObject(input)
+	if err != nil {
+		return err
+	}
 	return os.Remove(path.Join(b.bucket, key))
 }
 
@@ -40,61 +45,99 @@ func (b S3Backend) Exists(key string) (bool, error) {
 	return err == nil, err
 }
 
-func (b S3Backend) Get(key string) ([]byte, error) {
-	input := &s3.GetObjectInput{
+func (b S3Backend) Head(key string) (metadata backends.Metadata, err error) {
+	input := &s3.HeadObjectInput{
 		Bucket: aws.String(b.bucket),
 		Key: aws.String(key),
 	}
-	result, err := b.svc.GetObject(input)
+	result, err := b.svc.HeadObject(input)
 	if err != nil {
-		return []byte{}, err
+		return
 	}
-	defer result.Body.Close()
 
-	return ioutil.ReadAll(result.Body)
+	metadata, err = unmapMetadata(result.Metadata)
+	return
 }
 
-func (b S3Backend) Put(key string, r io.Reader) (int64, error) {
-	uploader := s3manager.NewUploaderWithClient(b.svc)
-	input := &s3manager.UploadInput{
+func (b S3Backend) Get(key string) (metadata backends.Metadata, r io.ReadCloser, err error) {
+	input := &s3.GetObjectInput{
 		Bucket: aws.String(b.bucket),
 		Key: aws.String(key),
-		Body: r,
 	}
-	result, err := uploader.Upload(input)
+	result, err := b.svc.GetObject(input)
 	if err != nil {
-		return 0, err
+		return
 	}
 
-	return -1, nil
+	metadata, err = unmapMetadata(result.Metadata)
+	r = result.Body
+	return
 }
 
-func (b S3Backend) Open(key string) (backends.ReadSeekCloser, error) {
-	input := &s3.GetObjectInput{
-		Bucket: aws.String(b.bucket),
-		Key: aws.String(key),
+func mapMetadata(m backends.Metadata) map[string]*string {
+	return map[string]*string{
+		"expiry": aws.String(strconv.FormatInt(m.Expiry.Unix(), 10)),
+		"delete_key": aws.String(m.DeleteKey),
+		"size": aws.String(strconv.FormatInt(m.Size, 10)),
+		"mimetype": aws.String(m.Mimetype),
+		"sha256sum": aws.String(m.Sha256sum),
 	}
-	result, err := b.svc.GetObject(input)
+}
+
+func unmapMetadata(input map[string]*string) (m backends.Metadata, err error) {
+	expiry, err := strconv.ParseInt(*input["expiry"], 10, 64)
 	if err != nil {
-		return nil, err
+		return
+	}
+	m.Expiry = time.Unix(expiry, 0)
+
+	m.Size, err = strconv.ParseInt(*input["size"], 10, 64)
+	if err != nil {
+		return
 	}
 
-	return result.Body, nil
+	m.DeleteKey = *input["delete_key"]
+	m.Mimetype = *input["mimetype"]
+	m.Sha256sum = *input["sha256sum"]
+	return
 }
 
-func (b S3Backend) ServeFile(key string, w http.ResponseWriter, r *http.Request) {
-	input := &s3.GetObjectInput{
+func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) {
+	tmpDst, err := ioutil.TempFile("", "linx-server-upload")
+	if err != nil {
+		return
+	}
+	defer tmpDst.Close()
+	defer os.Remove(tmpDst.Name())
+
+	bytes, err := io.Copy(tmpDst, r)
+	if bytes == 0 {
+		return m, errors.New("Empty file")
+	} else if err != nil {
+		return m, err
+	}
+
+	m.Expiry = expiry
+	m.DeleteKey = deleteKey
+	m.Size = bytes
+	m.Mimetype, _ = helpers.DetectMime(tmpDst)
+	m.Sha256sum, _ = helpers.Sha256sum(tmpDst)
+	// XXX: we may not be able to write this to AWS easily
+	//m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, tmpDst)
+
+	uploader := s3manager.NewUploaderWithClient(b.svc)
+	input := &s3manager.UploadInput{
 		Bucket: aws.String(b.bucket),
 		Key: aws.String(key),
+		Body: tmpDst,
+		Metadata: mapMetadata(m),
 	}
-	result, err := b.svc.GetObject(input)
+	_, err = uploader.Upload(input)
 	if err != nil {
-		return err
+		return
 	}
-	defer result.Body.Close()
 
-	http.ServeContent(w, r, key, *result.LastModified, result.Body)
-	return nil
+	return
 }
 
 func (b S3Backend) Size(key string) (int64, error) {
@@ -139,7 +182,7 @@ func (b S3Backend) GetTorrent(fileName string, url string) (t torrent.Torrent, e
 func (b S3Backend) List() ([]string, error) {
 	var output []string
 	input := &s3.ListObjectsInput{
-		bucket: aws.String(b.bucket),
+		Bucket: aws.String(b.bucket),
 	}
 
 	results, err := b.svc.ListObjects(input)

backends/storage.go

@@ -1,26 +1,19 @@
 package backends
 
 import (
+	"errors"
 	"io"
-	"net/http"
+	"time"
 
 	"github.com/andreimarcu/linx-server/torrent"
 )
 
-type ReadSeekCloser interface {
-	io.Reader
-	io.Closer
-	io.Seeker
-	io.ReaderAt
-}
-
 type StorageBackend interface {
 	Delete(key string) error
 	Exists(key string) (bool, error)
-	Get(key string) ([]byte, error)
-	Put(key string, r io.Reader) (int64, error)
-	Open(key string) (ReadSeekCloser, error)
-	ServeFile(key string, w http.ResponseWriter, r *http.Request) error
+	Head(key string) (Metadata, error)
+	Get(key string) (Metadata, io.ReadCloser, error)
+	Put(key string, r io.Reader, expiry time.Time, deleteKey string) (Metadata, error)
 	Size(key string) (int64, error)
 	GetTorrent(fileName string, url string) (torrent.Torrent, error)
 }
@@ -29,3 +22,5 @@ type MetaStorageBackend interface {
 	StorageBackend
 	List() ([]string, error)
 }
+
+var NotFoundErr = errors.New("File not found.")

delete.go

@@ -3,8 +3,8 @@ package main
 import (
 	"fmt"
 	"net/http"
-	"os"
 
+	"github.com/andreimarcu/linx-server/backends"
 	"github.com/zenazn/goji/web"
 )
 
@@ -13,24 +13,19 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 
 	filename := c.URLParams["name"]
 
-	// Ensure requested file actually exists
-	if _, readErr := fileBackend.Exists(filename); os.IsNotExist(readErr) {
+	// Ensure that file exists and delete key is correct
+	metadata, err := storageBackend.Head(filename)
+	if err == backends.NotFoundErr {
 		notFoundHandler(c, w, r) // 404 - file doesn't exist
 		return
-	}
-
-	// Ensure delete key is correct
-	metadata, err := metadataRead(filename)
-	if err != nil {
+	} else if err != nil {
 		unauthorizedHandler(c, w, r) // 401 - no metadata available
 		return
 	}
 
 	if metadata.DeleteKey == requestKey {
-		fileDelErr := fileBackend.Delete(filename)
-		metaDelErr := metaStorageBackend.Delete(filename)
-
-		if (fileDelErr != nil) || (metaDelErr != nil) {
+		err := storageBackend.Delete(filename)
+		if err != nil {
 			oopsHandler(c, w, r, RespPLAIN, "Could not delete")
 			return
 		}

display.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"io/ioutil"
 	"net/http"
 	"path/filepath"
 	"regexp"
@@ -9,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/andreimarcu/linx-server/backends"
 	"github.com/andreimarcu/linx-server/expiry"
 	"github.com/dustin/go-humanize"
 	"github.com/flosch/pongo2"
@@ -30,12 +32,12 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	fileName := c.URLParams["name"]
 
 	_, err := checkFile(fileName)
-	if err == NotFoundErr {
+	if err == backends.NotFoundErr {
 		notFoundHandler(c, w, r)
 		return
 	}
 
-	metadata, err := metadataRead(fileName)
+	metadata, err := storageBackend.Head(fileName)
 	if err != nil {
 		oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
 		return
@@ -77,8 +79,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 		tpl = Templates["display/pdf.html"]
 
 	} else if extension == "story" {
+		metadata, reader, err := storageBackend.Get(fileName)
+		if err != nil {
+			oopsHandler(c, w, r, RespHTML, err.Error())
+		}
+
 		if metadata.Size < maxDisplayFileSizeBytes {
-			bytes, err := fileBackend.Get(fileName)
+			bytes, err := ioutil.ReadAll(reader)
 			if err == nil {
 				extra["contents"] = string(bytes)
 				lines = strings.Split(extra["contents"], "\n")
@@ -87,8 +94,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 		}
 
 	} else if extension == "md" {
+		metadata, reader, err := storageBackend.Get(fileName)
+		if err != nil {
+			oopsHandler(c, w, r, RespHTML, err.Error())
+		}
+
 		if metadata.Size < maxDisplayFileSizeBytes {
-			bytes, err := fileBackend.Get(fileName)
+			bytes, err := ioutil.ReadAll(reader)
 			if err == nil {
 				unsafe := blackfriday.MarkdownCommon(bytes)
 				html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
@@ -99,8 +111,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 		}
 
 	} else if strings.HasPrefix(metadata.Mimetype, "text/") || supportedBinExtension(extension) {
+		metadata, reader, err := storageBackend.Get(fileName)
+		if err != nil {
+			oopsHandler(c, w, r, RespHTML, err.Error())
+		}
+
 		if metadata.Size < maxDisplayFileSizeBytes {
-			bytes, err := fileBackend.Get(fileName)
+			bytes, err := ioutil.ReadAll(reader)
 			if err == nil {
 				extra["extension"] = extension
 				extra["lang_hl"], extra["lang_ace"] = extensionToHlAndAceLangs(extension)

expiry.go

@@ -24,7 +24,7 @@ type ExpirationTime struct {
 
 // Determine if the given filename is expired
 func isFileExpired(filename string) (bool, error) {
-	metadata, err := metadataRead(filename)
+	metadata, err := storageBackend.Head(filename)
 	if err != nil {
 		return false, err
 	}

fileserve.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"github.com/andreimarcu/linx-server/backends"
@@ -14,7 +16,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	fileName := c.URLParams["name"]
 
 	metadata, err := checkFile(fileName)
-	if err == NotFoundErr {
+	if err == backends.NotFoundErr {
 		notFoundHandler(c, w, r)
 		return
 	} else if err == backends.BadMetadata {
@@ -38,12 +40,26 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy)
 	w.Header().Set("Referrer-Policy", Config.fileReferrerPolicy)
 
+	_, reader, err := storageBackend.Get(fileName)
+	if err != nil {
+		oopsHandler(c, w, r, RespAUTO, err.Error())
+	}
+
+	w.Header().Set("Content-Type", metadata.Mimetype)
+	w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
 	w.Header().Set("Etag", metadata.Sha256sum)
 	w.Header().Set("Cache-Control", "max-age=0")
 
-	err = fileBackend.ServeFile(fileName, w, r)
-	if err != nil {
-		oopsHandler(c, w, r, RespAUTO, err.Error())
+	// TODO: implement If-Match, If-None-Match (ETag)
+	// TODO: implement If-Unmodified-Since, If-Modified-Since (Last-Modified)
+	// TODO: implement range requests?
+
+	if r.Method != "HEAD" {
+		defer reader.Close()
+
+		if _, err = io.CopyN(w, reader, metadata.Size); err != nil {
+			oopsHandler(c, w, r, RespAUTO, err.Error())
+		}
 	}
 }
 
@@ -72,21 +88,14 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 }
 
 func checkFile(filename string) (metadata backends.Metadata, err error) {
-	_, err = fileBackend.Exists(filename)
-	if err != nil {
-		err = NotFoundErr
-		return
-	}
-
-	metadata, err = metadataRead(filename)
+	metadata, err = storageBackend.Head(filename)
 	if err != nil {
 		return
 	}
 
 	if expiry.IsTsExpired(metadata.Expiry) {
-		fileBackend.Delete(filename)
-		metaStorageBackend.Delete(filename)
-		err = NotFoundErr
+		storageBackend.Delete(filename)
+		err = backends.NotFoundErr
 		return
 	}
 

helpers/archive.go

@@ -0,0 +1,70 @@
+package helpers
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"compress/bzip2"
+	"compress/gzip"
+	"io"
+	"sort"
+)
+
+type ReadSeekerAt interface {
+	io.Reader
+	io.Seeker
+	io.ReaderAt
+}
+
+func ListArchiveFiles(mimetype string, size int64, r ReadSeekerAt) (files []string, err error) {
+	if mimetype == "application/x-tar" {
+		tReadr := tar.NewReader(r)
+		for {
+			hdr, err := tReadr.Next()
+			if err == io.EOF || err != nil {
+				break
+			}
+			if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
+				files = append(files, hdr.Name)
+			}
+		}
+		sort.Strings(files)
+	} else if mimetype == "application/x-gzip" {
+		gzf, err := gzip.NewReader(r)
+		if err == nil {
+			tReadr := tar.NewReader(gzf)
+			for {
+				hdr, err := tReadr.Next()
+				if err == io.EOF || err != nil {
+					break
+				}
+				if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
+					files = append(files, hdr.Name)
+				}
+			}
+			sort.Strings(files)
+		}
+	} else if mimetype == "application/x-bzip" {
+		bzf := bzip2.NewReader(r)
+		tReadr := tar.NewReader(bzf)
+		for {
+			hdr, err := tReadr.Next()
+			if err == io.EOF || err != nil {
+				break
+			}
+			if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
+				files = append(files, hdr.Name)
+			}
+		}
+		sort.Strings(files)
+	} else if mimetype == "application/zip" {
+		zf, err := zip.NewReader(r, size)
+		if err == nil {
+			for _, f := range zf.File {
+				files = append(files, f.Name)
+			}
+		}
+		sort.Strings(files)
+	}
+
+	return
+}

helpers/helpers.go

@@ -0,0 +1,67 @@
+package helpers
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"unicode"
+
+	"gopkg.in/h2non/filetype.v1"
+)
+
+func DetectMime(r io.ReadSeeker) (string, error) {
+	// Get first 512 bytes for mimetype detection
+	header := make([]byte, 512)
+
+	r.Seek(0, 0)
+	r.Read(header)
+	r.Seek(0, 0)
+
+	kind, err := filetype.Match(header)
+	if err != nil {
+		return "application/octet-stream", err
+	} else if kind.MIME.Value != "" {
+		return kind.MIME.Value, nil
+	}
+
+	// Check if the file seems anything like text
+	if printable(header) {
+		return "text/plain", nil
+	} else {
+		return "application/octet-stream", nil
+	}
+}
+
+func Sha256sum(r io.ReadSeeker) (string, error) {
+	hasher := sha256.New()
+
+	r.Seek(0, 0)
+	_, err := io.Copy(hasher, r)
+	if err != nil {
+		return "", err
+	}
+
+	r.Seek(0, 0)
+
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
+func printable(data []byte) bool {
+	for i, b := range data {
+		r := rune(b)
+
+		// A null terminator that's not at the beginning of the file
+		if r == 0 && i == 0 {
+			return false
+		} else if r == 0 && i < 0 {
+			continue
+		}
+
+		if r > unicode.MaxASCII {
+			return false
+		}
+
+	}
+
+	return true
+}

meta.go

@@ -1,165 +0,0 @@
-package main
-
-import (
-	"archive/tar"
-	"archive/zip"
-	"compress/bzip2"
-	"compress/gzip"
-	"crypto/sha256"
-	"encoding/hex"
-	"errors"
-	"io"
-	"sort"
-	"time"
-	"unicode"
-
-	"github.com/andreimarcu/linx-server/backends"
-	"github.com/andreimarcu/linx-server/expiry"
-	"github.com/dchest/uniuri"
-	"gopkg.in/h2non/filetype.v1"
-)
-
-var NotFoundErr = errors.New("File not found.")
-
-func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) {
-	file, err := fileBackend.Open(fName)
-	if err != nil {
-		return
-	}
-	defer file.Close()
-
-	m.Size, err = fileBackend.Size(fName)
-	if err != nil {
-		return
-	}
-
-	m.Expiry = exp
-
-	if delKey == "" {
-		m.DeleteKey = uniuri.NewLen(30)
-	} else {
-		m.DeleteKey = delKey
-	}
-
-	// Get first 512 bytes for mimetype detection
-	header := make([]byte, 512)
-	file.Read(header)
-
-	kind, err := filetype.Match(header)
-	if err != nil {
-		m.Mimetype = "application/octet-stream"
-	} else {
-		m.Mimetype = kind.MIME.Value
-	}
-
-	if m.Mimetype == "" {
-		// Check if the file seems anything like text
-		if printable(header) {
-			m.Mimetype = "text/plain"
-		} else {
-			m.Mimetype = "application/octet-stream"
-		}
-	}
-
-	// Compute the sha256sum
-	hasher := sha256.New()
-	file.Seek(0, 0)
-	_, err = io.Copy(hasher, file)
-	if err == nil {
-		m.Sha256sum = hex.EncodeToString(hasher.Sum(nil))
-	}
-	file.Seek(0, 0)
-
-	// If archive, grab list of filenames
-	if m.Mimetype == "application/x-tar" {
-		tReadr := tar.NewReader(file)
-		for {
-			hdr, err := tReadr.Next()
-			if err == io.EOF || err != nil {
-				break
-			}
-			if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
-				m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
-			}
-		}
-		sort.Strings(m.ArchiveFiles)
-	} else if m.Mimetype == "application/x-gzip" {
-		gzf, err := gzip.NewReader(file)
-		if err == nil {
-			tReadr := tar.NewReader(gzf)
-			for {
-				hdr, err := tReadr.Next()
-				if err == io.EOF || err != nil {
-					break
-				}
-				if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
-					m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
-				}
-			}
-			sort.Strings(m.ArchiveFiles)
-		}
-	} else if m.Mimetype == "application/x-bzip" {
-		bzf := bzip2.NewReader(file)
-		tReadr := tar.NewReader(bzf)
-		for {
-			hdr, err := tReadr.Next()
-			if err == io.EOF || err != nil {
-				break
-			}
-			if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
-				m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
-			}
-		}
-		sort.Strings(m.ArchiveFiles)
-	} else if m.Mimetype == "application/zip" {
-		zf, err := zip.NewReader(file, m.Size)
-		if err == nil {
-			for _, f := range zf.File {
-				m.ArchiveFiles = append(m.ArchiveFiles, f.Name)
-			}
-		}
-		sort.Strings(m.ArchiveFiles)
-	}
-
-	return
-}
-
-func metadataWrite(filename string, metadata *backends.Metadata) error {
-	return metaBackend.Put(filename, metadata)
-}
-
-func metadataRead(filename string) (metadata backends.Metadata, err error) {
-	metadata, err = metaBackend.Get(filename)
-	if err != nil {
-		// Metadata does not exist, generate one
-		newMData, err := generateMetadata(filename, expiry.NeverExpire, "")
-		if err != nil {
-			return metadata, err
-		}
-		metadataWrite(filename, &newMData)
-
-		metadata, err = metaBackend.Get(filename)
-	}
-
-	return
-}
-
-func printable(data []byte) bool {
-	for i, b := range data {
-		r := rune(b)
-
-		// A null terminator that's not at the beginning of the file
-		if r == 0 && i == 0 {
-			return false
-		} else if r == 0 && i < 0 {
-			continue
-		}
-
-		if r > unicode.MaxASCII {
-			return false
-		}
-
-	}
-
-	return true
-}

server.go

@@ -16,7 +16,7 @@ import (
 	"github.com/GeertJohan/go.rice"
 	"github.com/andreimarcu/linx-server/backends"
 	"github.com/andreimarcu/linx-server/backends/localfs"
-	"github.com/andreimarcu/linx-server/backends/metajson"
+	"github.com/andreimarcu/linx-server/backends/s3"
 	"github.com/flosch/pongo2"
 	"github.com/vharitonsky/iniflags"
 	"github.com/zenazn/goji/graceful"
@@ -60,6 +60,9 @@ var Config struct {
 	remoteAuthFile            string
 	addHeaders                headerList
 	noDirectAgents            bool
+	s3Endpoint                string
+	s3Region                  string
+	s3Bucket                  string
 }
 
 var Templates = make(map[string]*pongo2.Template)
@@ -69,8 +72,7 @@ var timeStarted time.Time
 var timeStartedStr string
 var remoteAuthKeys []string
 var metaStorageBackend backends.MetaStorageBackend
-var metaBackend backends.MetaBackend
-var fileBackend backends.StorageBackend
+var storageBackend backends.StorageBackend
 
 func setup() *web.Mux {
 	mux := web.New()
@@ -129,9 +131,11 @@ func setup() *web.Mux {
 		Config.sitePath = "/"
 	}
 
-	metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir)
-	metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend)
-	fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
+	if Config.s3Bucket != "" {
+		storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint)
+	} else {
+		storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir)
+	}
 
 	// Template setup
 	p2l, err := NewPongo2TemplatesLoader()
@@ -247,6 +251,12 @@ func main() {
 		"Add an arbitrary header to the response. This option can be used multiple times.")
 	flag.BoolVar(&Config.noDirectAgents, "nodirectagents", false,
 		"disable serving files directly for wget/curl user agents")
+	flag.StringVar(&Config.s3Endpoint, "s3-endpoint", "",
+		"S3 endpoint")
+	flag.StringVar(&Config.s3Region, "s3-region", "",
+		"S3 region")
+	flag.StringVar(&Config.s3Bucket, "s3-bucket", "",
+		"S3 bucket to use for files and metadata")
 
 	iniflags.Parse()
 

torrent.go

@@ -14,7 +14,7 @@ import (
 func createTorrent(fileName string, r *http.Request) ([]byte, error) {
 	url := fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName)
 
-	t, err := fileBackend.GetTorrent(fileName, url)
+	t, err := storageBackend.GetTorrent(fileName, url)
 	if err != nil {
 		return []byte{}, err
 	}
@@ -31,7 +31,7 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
 	fileName := c.URLParams["name"]
 
 	_, err := checkFile(fileName)
-	if err == NotFoundErr {
+	if err == backends.NotFoundErr {
 		notFoundHandler(c, w, r)
 		return
 	} else if err == backends.BadMetadata {

upload.go

@@ -237,11 +237,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
 	upload.Filename = strings.Join([]string{barename, extension}, ".")
 	upload.Filename = strings.Replace(upload.Filename, " ", "", -1)
 
-	fileexists, _ := fileBackend.Exists(upload.Filename)
+	fileexists, _ := storageBackend.Exists(upload.Filename)
 
 	// Check if the delete key matches, in which case overwrite
 	if fileexists {
-		metad, merr := metadataRead(upload.Filename)
+		metad, merr := storageBackend.Head(upload.Filename)
 		if merr == nil {
 			if upReq.deletionKey == metad.DeleteKey {
 				fileexists = false
@@ -258,7 +258,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
 		}
 		upload.Filename = strings.Join([]string{barename, extension}, ".")
 
-		fileexists, err = fileBackend.Exists(upload.Filename)
+		fileexists, err = storageBackend.Exists(upload.Filename)
 	}
 
 	if fileBlacklist[strings.ToLower(upload.Filename)] {
@@ -273,21 +273,15 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
 		fileExpiry = time.Now().Add(upReq.expiry)
 	}
 
-	_, err = fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
-	if err != nil {
-		return upload, err
+	if upReq.deletionKey == "" {
+		upReq.deletionKey = uniuri.NewLen(30)
 	}
 
-	upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey)
-	if err != nil {
-		fileBackend.Delete(upload.Filename)
-		return
-	}
-	err = metadataWrite(upload.Filename, &upload.Metadata)
+	upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deletionKey)
 	if err != nil {
-		fileBackend.Delete(upload.Filename)
-		return
+		return upload, err
 	}
+
 	return
 }