From 7a38b6e0df317c5f8e87b6ca9ed7d96cc7244baf Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Thu, 3 Jan 2019 22:42:12 -0800 Subject: [PATCH] Rework storage backends to integrate metadata --- backends/localfs/localfs.go | 145 ++++++++++++++++++++++++------ backends/meta.go | 5 -- backends/metajson/metajson.go | 70 --------------- backends/s3/s3.go | 105 +++++++++++++++------- backends/storage.go | 19 ++-- delete.go | 19 ++-- display.go | 27 ++++-- expiry.go | 2 +- fileserve.go | 37 +++++--- helpers/archive.go | 70 +++++++++++++++ helpers/helpers.go | 67 ++++++++++++++ meta.go | 165 ---------------------------------- server.go | 22 +++-- torrent.go | 4 +- upload.go | 22 ++--- 15 files changed, 414 insertions(+), 365 deletions(-) delete mode 100644 backends/metajson/metajson.go create mode 100644 helpers/archive.go create mode 100644 helpers/helpers.go delete mode 100644 meta.go diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index 70a8bed..4f8bf1c 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -1,65 +1,151 @@ package localfs import ( + "encoding/json" "errors" "io" "io/ioutil" - "net/http" "os" "path" + "time" "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/helpers" "github.com/andreimarcu/linx-server/torrent" ) type LocalfsBackend struct { - basePath string + metaPath string + filesPath string } -func (b LocalfsBackend) Delete(key string) error { - return os.Remove(path.Join(b.basePath, key)) +type MetadataJSON struct { + DeleteKey string `json:"delete_key"` + Sha256sum string `json:"sha256sum"` + Mimetype string `json:"mimetype"` + Size int64 `json:"size"` + Expiry int64 `json:"expiry"` + ArchiveFiles []string `json:"archive_files,omitempty"` +} + +func (b LocalfsBackend) Delete(key string) (err error) { + err = os.Remove(path.Join(b.filesPath, key)) + if err != nil { + return + } + err = os.Remove(path.Join(b.metaPath, key)) + return } func (b LocalfsBackend) Exists(key string) (bool, error) { - _, err := os.Stat(path.Join(b.basePath, key)) + _, err := os.Stat(path.Join(b.filesPath, key)) return err == nil, err } -func (b LocalfsBackend) Get(key string) ([]byte, error) { - return ioutil.ReadFile(path.Join(b.basePath, key)) +func (b LocalfsBackend) Head(key string) (metadata backends.Metadata, err error) { + f, err := os.Open(path.Join(b.metaPath, key)) + if os.IsNotExist(err) { + return metadata, backends.NotFoundErr + } else if err != nil { + return metadata, backends.BadMetadata + } + defer f.Close() + + decoder := json.NewDecoder(f) + + mjson := MetadataJSON{} + if err := decoder.Decode(&mjson); err != nil { + return metadata, backends.BadMetadata + } + + metadata.DeleteKey = mjson.DeleteKey + metadata.Mimetype = mjson.Mimetype + metadata.ArchiveFiles = mjson.ArchiveFiles + metadata.Sha256sum = mjson.Sha256sum + metadata.Expiry = time.Unix(mjson.Expiry, 0) + metadata.Size = mjson.Size + + return } -func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) { - dst, err := os.Create(path.Join(b.basePath, key)) +func (b LocalfsBackend) Get(key string) (metadata backends.Metadata, f io.ReadCloser, err error) { + metadata, err = b.Head(key) if err != nil { - return 0, err + return + } + + f, err = os.Open(path.Join(b.filesPath, key)) + if err != nil { + return + } + + return +} + +func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) error { + metaPath := path.Join(b.metaPath, key) + + mjson := MetadataJSON{ + DeleteKey: metadata.DeleteKey, + Mimetype: metadata.Mimetype, + ArchiveFiles: metadata.ArchiveFiles, + Sha256sum: metadata.Sha256sum, + Expiry: metadata.Expiry.Unix(), + Size: metadata.Size, + } + + dst, err := os.Create(metaPath) + if err != nil { + return err + } + defer dst.Close() + + encoder := json.NewEncoder(dst) + err = encoder.Encode(mjson) + if err != nil { + os.Remove(metaPath) + return err + } + + return nil +} + +func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) { + filePath := path.Join(b.filesPath, key) + + dst, err := os.Create(filePath) + if err != nil { + return } defer dst.Close() bytes, err := io.Copy(dst, r) if bytes == 0 { - b.Delete(key) - return bytes, errors.New("Empty file") + os.Remove(filePath) + return m, errors.New("Empty file") } else if err != nil { - b.Delete(key) - return bytes, err + os.Remove(filePath) + return m, err } - return bytes, err -} + m.Expiry = expiry + m.DeleteKey = deleteKey + m.Size = bytes + m.Mimetype, _ = helpers.DetectMime(dst) + m.Sha256sum, _ = helpers.Sha256sum(dst) + m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, dst) -func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) { - return os.Open(path.Join(b.basePath, key)) -} + err = b.writeMetadata(key, m) + if err != nil { + os.Remove(filePath) + return + } -func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) error { - filePath := path.Join(b.basePath, key) - http.ServeFile(w, r, filePath) - return nil + return } func (b LocalfsBackend) Size(key string) (int64, error) { - fileInfo, err := os.Stat(path.Join(b.basePath, key)) + fileInfo, err := os.Stat(path.Join(b.filesPath, key)) if err != nil { return 0, err } @@ -79,7 +165,7 @@ func (b LocalfsBackend) GetTorrent(fileName string, url string) (t torrent.Torre UrlList: []string{url}, } - f, err := b.Open(fileName) + _, f, err := b.Get(fileName) if err != nil { return } @@ -103,7 +189,7 @@ func (b LocalfsBackend) GetTorrent(fileName string, url string) (t torrent.Torre func (b LocalfsBackend) List() ([]string, error) { var output []string - files, err := ioutil.ReadDir(b.basePath) + files, err := ioutil.ReadDir(b.filesPath) if err != nil { return nil, err } @@ -115,6 +201,9 @@ func (b LocalfsBackend) List() ([]string, error) { return output, nil } -func NewLocalfsBackend(basePath string) LocalfsBackend { - return LocalfsBackend{basePath: basePath} +func NewLocalfsBackend(metaPath string, filesPath string) LocalfsBackend { + return LocalfsBackend{ + metaPath: metaPath, + filesPath: filesPath, + } } diff --git a/backends/meta.go b/backends/meta.go index 27c3e41..7ba522d 100644 --- a/backends/meta.go +++ b/backends/meta.go @@ -5,11 +5,6 @@ import ( "time" ) -type MetaBackend interface { - Get(key string) (Metadata, error) - Put(key string, metadata *Metadata) error -} - type Metadata struct { DeleteKey string Sha256sum string diff --git a/backends/metajson/metajson.go b/backends/metajson/metajson.go deleted file mode 100644 index 8ec53c4..0000000 --- a/backends/metajson/metajson.go +++ /dev/null @@ -1,70 +0,0 @@ -package metajson - -import ( - "bytes" - "encoding/json" - "time" - - "github.com/andreimarcu/linx-server/backends" -) - -type MetadataJSON struct { - DeleteKey string `json:"delete_key"` - Sha256sum string `json:"sha256sum"` - Mimetype string `json:"mimetype"` - Size int64 `json:"size"` - Expiry int64 `json:"expiry"` - ArchiveFiles []string `json:"archive_files,omitempty"` -} - -type MetaJSONBackend struct { - storage backends.MetaStorageBackend -} - -func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error { - mjson := MetadataJSON{} - mjson.DeleteKey = metadata.DeleteKey - mjson.Mimetype = metadata.Mimetype - mjson.ArchiveFiles = metadata.ArchiveFiles - mjson.Sha256sum = metadata.Sha256sum - mjson.Expiry = metadata.Expiry.Unix() - mjson.Size = metadata.Size - - byt, err := json.Marshal(mjson) - if err != nil { - return err - } - - if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil { - return err - } - - return nil -} - -func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) { - b, err := m.storage.Get(key) - if err != nil { - return metadata, backends.BadMetadata - } - - mjson := MetadataJSON{} - - err = json.Unmarshal(b, &mjson) - if err != nil { - return metadata, backends.BadMetadata - } - - metadata.DeleteKey = mjson.DeleteKey - metadata.Mimetype = mjson.Mimetype - metadata.ArchiveFiles = mjson.ArchiveFiles - metadata.Sha256sum = mjson.Sha256sum - metadata.Expiry = time.Unix(mjson.Expiry, 0) - metadata.Size = mjson.Size - - return -} - -func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend { - return MetaJSONBackend{storage: storage} -} diff --git a/backends/s3/s3.go b/backends/s3/s3.go index 4eeb760..e475d24 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -1,14 +1,16 @@ -package localfs +package s3 import ( "errors" "io" "io/ioutil" - "net/http" "os" "path" + "strconv" + "time" "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/helpers" "github.com/andreimarcu/linx-server/torrent" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -19,7 +21,7 @@ import ( type S3Backend struct { bucket string - svc *S3 + svc *s3.S3 } func (b S3Backend) Delete(key string) error { @@ -28,6 +30,9 @@ func (b S3Backend) Delete(key string) error { Key: aws.String(key), } _, err := b.svc.DeleteObject(input) + if err != nil { + return err + } return os.Remove(path.Join(b.bucket, key)) } @@ -40,61 +45,99 @@ func (b S3Backend) Exists(key string) (bool, error) { return err == nil, err } -func (b S3Backend) Get(key string) ([]byte, error) { - input := &s3.GetObjectInput{ +func (b S3Backend) Head(key string) (metadata backends.Metadata, err error) { + input := &s3.HeadObjectInput{ Bucket: aws.String(b.bucket), Key: aws.String(key), } - result, err := b.svc.GetObject(input) + result, err := b.svc.HeadObject(input) if err != nil { - return []byte{}, err + return } - defer result.Body.Close() - return ioutil.ReadAll(result.Body) + metadata, err = unmapMetadata(result.Metadata) + return } -func (b S3Backend) Put(key string, r io.Reader) (int64, error) { - uploader := s3manager.NewUploaderWithClient(b.svc) - input := &s3manager.UploadInput{ +func (b S3Backend) Get(key string) (metadata backends.Metadata, r io.ReadCloser, err error) { + input := &s3.GetObjectInput{ Bucket: aws.String(b.bucket), Key: aws.String(key), - Body: r, } - result, err := uploader.Upload(input) + result, err := b.svc.GetObject(input) if err != nil { - return 0, err + return } - return -1, nil + metadata, err = unmapMetadata(result.Metadata) + r = result.Body + return } -func (b S3Backend) Open(key string) (backends.ReadSeekCloser, error) { - input := &s3.GetObjectInput{ - Bucket: aws.String(b.bucket), - Key: aws.String(key), +func mapMetadata(m backends.Metadata) map[string]*string { + return map[string]*string{ + "expiry": aws.String(strconv.FormatInt(m.Expiry.Unix(), 10)), + "delete_key": aws.String(m.DeleteKey), + "size": aws.String(strconv.FormatInt(m.Size, 10)), + "mimetype": aws.String(m.Mimetype), + "sha256sum": aws.String(m.Sha256sum), } - result, err := b.svc.GetObject(input) +} + +func unmapMetadata(input map[string]*string) (m backends.Metadata, err error) { + expiry, err := strconv.ParseInt(*input["expiry"], 10, 64) if err != nil { - return nil, err + return + } + m.Expiry = time.Unix(expiry, 0) + + m.Size, err = strconv.ParseInt(*input["size"], 10, 64) + if err != nil { + return } - return result.Body, nil + m.DeleteKey = *input["delete_key"] + m.Mimetype = *input["mimetype"] + m.Sha256sum = *input["sha256sum"] + return } -func (b S3Backend) ServeFile(key string, w http.ResponseWriter, r *http.Request) { - input := &s3.GetObjectInput{ +func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) { + tmpDst, err := ioutil.TempFile("", "linx-server-upload") + if err != nil { + return + } + defer tmpDst.Close() + defer os.Remove(tmpDst.Name()) + + bytes, err := io.Copy(tmpDst, r) + if bytes == 0 { + return m, errors.New("Empty file") + } else if err != nil { + return m, err + } + + m.Expiry = expiry + m.DeleteKey = deleteKey + m.Size = bytes + m.Mimetype, _ = helpers.DetectMime(tmpDst) + m.Sha256sum, _ = helpers.Sha256sum(tmpDst) + // XXX: we may not be able to write this to AWS easily + //m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, tmpDst) + + uploader := s3manager.NewUploaderWithClient(b.svc) + input := &s3manager.UploadInput{ Bucket: aws.String(b.bucket), Key: aws.String(key), + Body: tmpDst, + Metadata: mapMetadata(m), } - result, err := b.svc.GetObject(input) + _, err = uploader.Upload(input) if err != nil { - return err + return } - defer result.Body.Close() - http.ServeContent(w, r, key, *result.LastModified, result.Body) - return nil + return } func (b S3Backend) Size(key string) (int64, error) { @@ -139,7 +182,7 @@ func (b S3Backend) GetTorrent(fileName string, url string) (t torrent.Torrent, e func (b S3Backend) List() ([]string, error) { var output []string input := &s3.ListObjectsInput{ - bucket: aws.String(b.bucket), + Bucket: aws.String(b.bucket), } results, err := b.svc.ListObjects(input) diff --git a/backends/storage.go b/backends/storage.go index 329d301..caff5b7 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -1,26 +1,19 @@ package backends import ( + "errors" "io" - "net/http" + "time" "github.com/andreimarcu/linx-server/torrent" ) -type ReadSeekCloser interface { - io.Reader - io.Closer - io.Seeker - io.ReaderAt -} - type StorageBackend interface { Delete(key string) error Exists(key string) (bool, error) - Get(key string) ([]byte, error) - Put(key string, r io.Reader) (int64, error) - Open(key string) (ReadSeekCloser, error) - ServeFile(key string, w http.ResponseWriter, r *http.Request) error + Head(key string) (Metadata, error) + Get(key string) (Metadata, io.ReadCloser, error) + Put(key string, r io.Reader, expiry time.Time, deleteKey string) (Metadata, error) Size(key string) (int64, error) GetTorrent(fileName string, url string) (torrent.Torrent, error) } @@ -29,3 +22,5 @@ type MetaStorageBackend interface { StorageBackend List() ([]string, error) } + +var NotFoundErr = errors.New("File not found.") diff --git a/delete.go b/delete.go index 61c6fa8..38e36e3 100644 --- a/delete.go +++ b/delete.go @@ -3,8 +3,8 @@ package main import ( "fmt" "net/http" - "os" + "github.com/andreimarcu/linx-server/backends" "github.com/zenazn/goji/web" ) @@ -13,24 +13,19 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) { filename := c.URLParams["name"] - // Ensure requested file actually exists - if _, readErr := fileBackend.Exists(filename); os.IsNotExist(readErr) { + // Ensure that file exists and delete key is correct + metadata, err := storageBackend.Head(filename) + if err == backends.NotFoundErr { notFoundHandler(c, w, r) // 404 - file doesn't exist return - } - - // Ensure delete key is correct - metadata, err := metadataRead(filename) - if err != nil { + } else if err != nil { unauthorizedHandler(c, w, r) // 401 - no metadata available return } if metadata.DeleteKey == requestKey { - fileDelErr := fileBackend.Delete(filename) - metaDelErr := metaStorageBackend.Delete(filename) - - if (fileDelErr != nil) || (metaDelErr != nil) { + err := storageBackend.Delete(filename) + if err != nil { oopsHandler(c, w, r, RespPLAIN, "Could not delete") return } diff --git a/display.go b/display.go index d897e5a..00b77d5 100644 --- a/display.go +++ b/display.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "io/ioutil" "net/http" "path/filepath" "regexp" @@ -9,6 +10,7 @@ import ( "strings" "time" + "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" "github.com/dustin/go-humanize" "github.com/flosch/pongo2" @@ -30,12 +32,12 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] _, err := checkFile(fileName) - if err == NotFoundErr { + if err == backends.NotFoundErr { notFoundHandler(c, w, r) return } - metadata, err := metadataRead(fileName) + metadata, err := storageBackend.Head(fileName) if err != nil { oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") return @@ -77,8 +79,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { tpl = Templates["display/pdf.html"] } else if extension == "story" { + metadata, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespHTML, err.Error()) + } + if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := fileBackend.Get(fileName) + bytes, err := ioutil.ReadAll(reader) if err == nil { extra["contents"] = string(bytes) lines = strings.Split(extra["contents"], "\n") @@ -87,8 +94,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } } else if extension == "md" { + metadata, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespHTML, err.Error()) + } + if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := fileBackend.Get(fileName) + bytes, err := ioutil.ReadAll(reader) if err == nil { unsafe := blackfriday.MarkdownCommon(bytes) html := bluemonday.UGCPolicy().SanitizeBytes(unsafe) @@ -99,8 +111,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } } else if strings.HasPrefix(metadata.Mimetype, "text/") || supportedBinExtension(extension) { + metadata, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespHTML, err.Error()) + } + if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := fileBackend.Get(fileName) + bytes, err := ioutil.ReadAll(reader) if err == nil { extra["extension"] = extension extra["lang_hl"], extra["lang_ace"] = extensionToHlAndAceLangs(extension) diff --git a/expiry.go b/expiry.go index 6d8887d..63b7757 100644 --- a/expiry.go +++ b/expiry.go @@ -24,7 +24,7 @@ type ExpirationTime struct { // Determine if the given filename is expired func isFileExpired(filename string) (bool, error) { - metadata, err := metadataRead(filename) + metadata, err := storageBackend.Head(filename) if err != nil { return false, err } diff --git a/fileserve.go b/fileserve.go index 4b8f95c..30a5de6 100644 --- a/fileserve.go +++ b/fileserve.go @@ -1,8 +1,10 @@ package main import ( + "io" "net/http" "net/url" + "strconv" "strings" "github.com/andreimarcu/linx-server/backends" @@ -14,7 +16,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] metadata, err := checkFile(fileName) - if err == NotFoundErr { + if err == backends.NotFoundErr { notFoundHandler(c, w, r) return } else if err == backends.BadMetadata { @@ -38,12 +40,26 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy) w.Header().Set("Referrer-Policy", Config.fileReferrerPolicy) + _, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespAUTO, err.Error()) + } + + w.Header().Set("Content-Type", metadata.Mimetype) + w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10)) w.Header().Set("Etag", metadata.Sha256sum) w.Header().Set("Cache-Control", "max-age=0") - err = fileBackend.ServeFile(fileName, w, r) - if err != nil { - oopsHandler(c, w, r, RespAUTO, err.Error()) + // TODO: implement If-Match, If-None-Match (ETag) + // TODO: implement If-Unmodified-Since, If-Modified-Since (Last-Modified) + // TODO: implement range requests? + + if r.Method != "HEAD" { + defer reader.Close() + + if _, err = io.CopyN(w, reader, metadata.Size); err != nil { + oopsHandler(c, w, r, RespAUTO, err.Error()) + } } } @@ -72,21 +88,14 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) { } func checkFile(filename string) (metadata backends.Metadata, err error) { - _, err = fileBackend.Exists(filename) - if err != nil { - err = NotFoundErr - return - } - - metadata, err = metadataRead(filename) + metadata, err = storageBackend.Head(filename) if err != nil { return } if expiry.IsTsExpired(metadata.Expiry) { - fileBackend.Delete(filename) - metaStorageBackend.Delete(filename) - err = NotFoundErr + storageBackend.Delete(filename) + err = backends.NotFoundErr return } diff --git a/helpers/archive.go b/helpers/archive.go new file mode 100644 index 0000000..2a4380b --- /dev/null +++ b/helpers/archive.go @@ -0,0 +1,70 @@ +package helpers + +import ( + "archive/tar" + "archive/zip" + "compress/bzip2" + "compress/gzip" + "io" + "sort" +) + +type ReadSeekerAt interface { + io.Reader + io.Seeker + io.ReaderAt +} + +func ListArchiveFiles(mimetype string, size int64, r ReadSeekerAt) (files []string, err error) { + if mimetype == "application/x-tar" { + tReadr := tar.NewReader(r) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + files = append(files, hdr.Name) + } + } + sort.Strings(files) + } else if mimetype == "application/x-gzip" { + gzf, err := gzip.NewReader(r) + if err == nil { + tReadr := tar.NewReader(gzf) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + files = append(files, hdr.Name) + } + } + sort.Strings(files) + } + } else if mimetype == "application/x-bzip" { + bzf := bzip2.NewReader(r) + tReadr := tar.NewReader(bzf) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + files = append(files, hdr.Name) + } + } + sort.Strings(files) + } else if mimetype == "application/zip" { + zf, err := zip.NewReader(r, size) + if err == nil { + for _, f := range zf.File { + files = append(files, f.Name) + } + } + sort.Strings(files) + } + + return +} diff --git a/helpers/helpers.go b/helpers/helpers.go new file mode 100644 index 0000000..d8e65e3 --- /dev/null +++ b/helpers/helpers.go @@ -0,0 +1,67 @@ +package helpers + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "unicode" + + "gopkg.in/h2non/filetype.v1" +) + +func DetectMime(r io.ReadSeeker) (string, error) { + // Get first 512 bytes for mimetype detection + header := make([]byte, 512) + + r.Seek(0, 0) + r.Read(header) + r.Seek(0, 0) + + kind, err := filetype.Match(header) + if err != nil { + return "application/octet-stream", err + } else if kind.MIME.Value != "" { + return kind.MIME.Value, nil + } + + // Check if the file seems anything like text + if printable(header) { + return "text/plain", nil + } else { + return "application/octet-stream", nil + } +} + +func Sha256sum(r io.ReadSeeker) (string, error) { + hasher := sha256.New() + + r.Seek(0, 0) + _, err := io.Copy(hasher, r) + if err != nil { + return "", err + } + + r.Seek(0, 0) + + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func printable(data []byte) bool { + for i, b := range data { + r := rune(b) + + // A null terminator that's not at the beginning of the file + if r == 0 && i == 0 { + return false + } else if r == 0 && i < 0 { + continue + } + + if r > unicode.MaxASCII { + return false + } + + } + + return true +} diff --git a/meta.go b/meta.go deleted file mode 100644 index 10cd4e1..0000000 --- a/meta.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -import ( - "archive/tar" - "archive/zip" - "compress/bzip2" - "compress/gzip" - "crypto/sha256" - "encoding/hex" - "errors" - "io" - "sort" - "time" - "unicode" - - "github.com/andreimarcu/linx-server/backends" - "github.com/andreimarcu/linx-server/expiry" - "github.com/dchest/uniuri" - "gopkg.in/h2non/filetype.v1" -) - -var NotFoundErr = errors.New("File not found.") - -func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) { - file, err := fileBackend.Open(fName) - if err != nil { - return - } - defer file.Close() - - m.Size, err = fileBackend.Size(fName) - if err != nil { - return - } - - m.Expiry = exp - - if delKey == "" { - m.DeleteKey = uniuri.NewLen(30) - } else { - m.DeleteKey = delKey - } - - // Get first 512 bytes for mimetype detection - header := make([]byte, 512) - file.Read(header) - - kind, err := filetype.Match(header) - if err != nil { - m.Mimetype = "application/octet-stream" - } else { - m.Mimetype = kind.MIME.Value - } - - if m.Mimetype == "" { - // Check if the file seems anything like text - if printable(header) { - m.Mimetype = "text/plain" - } else { - m.Mimetype = "application/octet-stream" - } - } - - // Compute the sha256sum - hasher := sha256.New() - file.Seek(0, 0) - _, err = io.Copy(hasher, file) - if err == nil { - m.Sha256sum = hex.EncodeToString(hasher.Sum(nil)) - } - file.Seek(0, 0) - - // If archive, grab list of filenames - if m.Mimetype == "application/x-tar" { - tReadr := tar.NewReader(file) - for { - hdr, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { - m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) - } - } - sort.Strings(m.ArchiveFiles) - } else if m.Mimetype == "application/x-gzip" { - gzf, err := gzip.NewReader(file) - if err == nil { - tReadr := tar.NewReader(gzf) - for { - hdr, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { - m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) - } - } - sort.Strings(m.ArchiveFiles) - } - } else if m.Mimetype == "application/x-bzip" { - bzf := bzip2.NewReader(file) - tReadr := tar.NewReader(bzf) - for { - hdr, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { - m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) - } - } - sort.Strings(m.ArchiveFiles) - } else if m.Mimetype == "application/zip" { - zf, err := zip.NewReader(file, m.Size) - if err == nil { - for _, f := range zf.File { - m.ArchiveFiles = append(m.ArchiveFiles, f.Name) - } - } - sort.Strings(m.ArchiveFiles) - } - - return -} - -func metadataWrite(filename string, metadata *backends.Metadata) error { - return metaBackend.Put(filename, metadata) -} - -func metadataRead(filename string) (metadata backends.Metadata, err error) { - metadata, err = metaBackend.Get(filename) - if err != nil { - // Metadata does not exist, generate one - newMData, err := generateMetadata(filename, expiry.NeverExpire, "") - if err != nil { - return metadata, err - } - metadataWrite(filename, &newMData) - - metadata, err = metaBackend.Get(filename) - } - - return -} - -func printable(data []byte) bool { - for i, b := range data { - r := rune(b) - - // A null terminator that's not at the beginning of the file - if r == 0 && i == 0 { - return false - } else if r == 0 && i < 0 { - continue - } - - if r > unicode.MaxASCII { - return false - } - - } - - return true -} diff --git a/server.go b/server.go index 1dd1f09..4af3ea6 100644 --- a/server.go +++ b/server.go @@ -16,7 +16,7 @@ import ( "github.com/GeertJohan/go.rice" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends/localfs" - "github.com/andreimarcu/linx-server/backends/metajson" + "github.com/andreimarcu/linx-server/backends/s3" "github.com/flosch/pongo2" "github.com/vharitonsky/iniflags" "github.com/zenazn/goji/graceful" @@ -60,6 +60,9 @@ var Config struct { remoteAuthFile string addHeaders headerList noDirectAgents bool + s3Endpoint string + s3Region string + s3Bucket string } var Templates = make(map[string]*pongo2.Template) @@ -69,8 +72,7 @@ var timeStarted time.Time var timeStartedStr string var remoteAuthKeys []string var metaStorageBackend backends.MetaStorageBackend -var metaBackend backends.MetaBackend -var fileBackend backends.StorageBackend +var storageBackend backends.StorageBackend func setup() *web.Mux { mux := web.New() @@ -129,9 +131,11 @@ func setup() *web.Mux { Config.sitePath = "/" } - metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir) - metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend) - fileBackend = localfs.NewLocalfsBackend(Config.filesDir) + if Config.s3Bucket != "" { + storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint) + } else { + storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir) + } // Template setup p2l, err := NewPongo2TemplatesLoader() @@ -247,6 +251,12 @@ func main() { "Add an arbitrary header to the response. This option can be used multiple times.") flag.BoolVar(&Config.noDirectAgents, "nodirectagents", false, "disable serving files directly for wget/curl user agents") + flag.StringVar(&Config.s3Endpoint, "s3-endpoint", "", + "S3 endpoint") + flag.StringVar(&Config.s3Region, "s3-region", "", + "S3 region") + flag.StringVar(&Config.s3Bucket, "s3-bucket", "", + "S3 bucket to use for files and metadata") iniflags.Parse() diff --git a/torrent.go b/torrent.go index 73f415c..50585c1 100644 --- a/torrent.go +++ b/torrent.go @@ -14,7 +14,7 @@ import ( func createTorrent(fileName string, r *http.Request) ([]byte, error) { url := fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName) - t, err := fileBackend.GetTorrent(fileName, url) + t, err := storageBackend.GetTorrent(fileName, url) if err != nil { return []byte{}, err } @@ -31,7 +31,7 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] _, err := checkFile(fileName) - if err == NotFoundErr { + if err == backends.NotFoundErr { notFoundHandler(c, w, r) return } else if err == backends.BadMetadata { diff --git a/upload.go b/upload.go index 986eaa6..b66d8ca 100644 --- a/upload.go +++ b/upload.go @@ -237,11 +237,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { upload.Filename = strings.Join([]string{barename, extension}, ".") upload.Filename = strings.Replace(upload.Filename, " ", "", -1) - fileexists, _ := fileBackend.Exists(upload.Filename) + fileexists, _ := storageBackend.Exists(upload.Filename) // Check if the delete key matches, in which case overwrite if fileexists { - metad, merr := metadataRead(upload.Filename) + metad, merr := storageBackend.Head(upload.Filename) if merr == nil { if upReq.deletionKey == metad.DeleteKey { fileexists = false @@ -258,7 +258,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { } upload.Filename = strings.Join([]string{barename, extension}, ".") - fileexists, err = fileBackend.Exists(upload.Filename) + fileexists, err = storageBackend.Exists(upload.Filename) } if fileBlacklist[strings.ToLower(upload.Filename)] { @@ -273,21 +273,15 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { fileExpiry = time.Now().Add(upReq.expiry) } - _, err = fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src)) - if err != nil { - return upload, err + if upReq.deletionKey == "" { + upReq.deletionKey = uniuri.NewLen(30) } - upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey) - if err != nil { - fileBackend.Delete(upload.Filename) - return - } - err = metadataWrite(upload.Filename, &upload.Metadata) + upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deletionKey) if err != nil { - fileBackend.Delete(upload.Filename) - return + return upload, err } + return }