Docker
/
linx-server
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Rework storage backends to integrate metadata

pull/156/head
mutantmonkey 6 years ago
parent
e24bbd1255
commit
7a38b6e0df
15 changed files with 414 additions and 365 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 145
      backends/localfs/localfs.go
  2. 5
      backends/meta.go
  3. 70
      backends/metajson/metajson.go
  4. 105
      backends/s3/s3.go
  5. 19
      backends/storage.go
  6. 19
      delete.go
  7. 27
      display.go
  8. 2
      expiry.go
  9. 37
      fileserve.go
  10. 70
      helpers/archive.go
  11. 67
      helpers/helpers.go
  12. 165
      meta.go
  13. 22
      server.go
  14. 4
      torrent.go
  15. 22
      upload.go

145
backends/localfs/localfs.go
View File

@ -1,65 +1,151 @@
package localfs
import (
"encoding/json"
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"time"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/helpers"
"github.com/andreimarcu/linx-server/torrent"
)
type LocalfsBackend struct {
basePath string
metaPath string
filesPath string
}
func (b LocalfsBackend) Delete(key string) error {
return os.Remove(path.Join(b.basePath, key))
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
}
func (b LocalfsBackend) Delete(key string) (err error) {
err = os.Remove(path.Join(b.filesPath, key))
if err != nil {
return
}
err = os.Remove(path.Join(b.metaPath, key))
return
}
func (b LocalfsBackend) Exists(key string) (bool, error) {
_, err := os.Stat(path.Join(b.basePath, key))
_, err := os.Stat(path.Join(b.filesPath, key))
return err == nil, err
}
func (b LocalfsBackend) Get(key string) ([]byte, error) {
return ioutil.ReadFile(path.Join(b.basePath, key))
func (b LocalfsBackend) Head(key string) (metadata backends.Metadata, err error) {
f, err := os.Open(path.Join(b.metaPath, key))
if os.IsNotExist(err) {
return metadata, backends.NotFoundErr
} else if err != nil {
return metadata, backends.BadMetadata
}
defer f.Close()
decoder := json.NewDecoder(f)
mjson := MetadataJSON{}
if err := decoder.Decode(&mjson); err != nil {
return metadata, backends.BadMetadata
}
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
return
}
func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) {
dst, err := os.Create(path.Join(b.basePath, key))
func (b LocalfsBackend) Get(key string) (metadata backends.Metadata, f io.ReadCloser, err error) {
metadata, err = b.Head(key)
if err != nil {
return 0, err
return
}
f, err = os.Open(path.Join(b.filesPath, key))
if err != nil {
return
}
return
}
func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) error {
metaPath := path.Join(b.metaPath, key)
mjson := MetadataJSON{
DeleteKey: metadata.DeleteKey,
Mimetype: metadata.Mimetype,
ArchiveFiles: metadata.ArchiveFiles,
Sha256sum: metadata.Sha256sum,
Expiry: metadata.Expiry.Unix(),
Size: metadata.Size,
}
dst, err := os.Create(metaPath)
if err != nil {
return err
}
defer dst.Close()
encoder := json.NewEncoder(dst)
err = encoder.Encode(mjson)
if err != nil {
os.Remove(metaPath)
return err
}
return nil
}
func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) {
filePath := path.Join(b.filesPath, key)
dst, err := os.Create(filePath)
if err != nil {
return
}
defer dst.Close()
bytes, err := io.Copy(dst, r)
if bytes == 0 {
b.Delete(key)
return bytes, errors.New("Empty file")
os.Remove(filePath)
return m, errors.New("Empty file")
} else if err != nil {
b.Delete(key)
return bytes, err
os.Remove(filePath)
return m, err
}
return bytes, err
}
m.Expiry = expiry
m.DeleteKey = deleteKey
m.Size = bytes
m.Mimetype, _ = helpers.DetectMime(dst)
m.Sha256sum, _ = helpers.Sha256sum(dst)
m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, dst)
func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) {
return os.Open(path.Join(b.basePath, key))
}
err = b.writeMetadata(key, m)
if err != nil {
os.Remove(filePath)
return
}
func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) error {
filePath := path.Join(b.basePath, key)
http.ServeFile(w, r, filePath)
return nil
return
}
func (b LocalfsBackend) Size(key string) (int64, error) {
fileInfo, err := os.Stat(path.Join(b.basePath, key))
fileInfo, err := os.Stat(path.Join(b.filesPath, key))
if err != nil {
return 0, err
}
@ -79,7 +165,7 @@ func (b LocalfsBackend) GetTorrent(fileName string, url string) (t torrent.Torre
UrlList: []string{url},
}
f, err := b.Open(fileName)
_, f, err := b.Get(fileName)
if err != nil {
return
}
@ -103,7 +189,7 @@ func (b LocalfsBackend) GetTorrent(fileName string, url string) (t torrent.Torre
func (b LocalfsBackend) List() ([]string, error) {
var output []string
files, err := ioutil.ReadDir(b.basePath)
files, err := ioutil.ReadDir(b.filesPath)
if err != nil {
return nil, err
}
@ -115,6 +201,9 @@ func (b LocalfsBackend) List() ([]string, error) {
return output, nil
}
func NewLocalfsBackend(basePath string) LocalfsBackend {
return LocalfsBackend{basePath: basePath}
func NewLocalfsBackend(metaPath string, filesPath string) LocalfsBackend {
return LocalfsBackend{
metaPath: metaPath,
filesPath: filesPath,
}
}

5
backends/meta.go
View File

@ -5,11 +5,6 @@ import (
"time"
)
type MetaBackend interface {
Get(key string) (Metadata, error)
Put(key string, metadata *Metadata) error
}
type Metadata struct {
DeleteKey string
Sha256sum string

70
backends/metajson/metajson.go
View File

@ -1,70 +0,0 @@
package metajson
import (
"bytes"
"encoding/json"
"time"
"github.com/andreimarcu/linx-server/backends"
)
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
}
type MetaJSONBackend struct {
storage backends.MetaStorageBackend
}
func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error {
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
byt, err := json.Marshal(mjson)
if err != nil {
return err
}
if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil {
return err
}
return nil
}
func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) {
b, err := m.storage.Get(key)
if err != nil {
return metadata, backends.BadMetadata
}
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil {
return metadata, backends.BadMetadata
}
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
return
}
func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend {
return MetaJSONBackend{storage: storage}
}

105
backends/s3/s3.go
View File

@ -1,14 +1,16 @@
package localfs
package s3
import (
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"strconv"
"time"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/helpers"
"github.com/andreimarcu/linx-server/torrent"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
@ -19,7 +21,7 @@ import (
type S3Backend struct {
bucket string
svc *S3
svc *s3.S3
}
func (b S3Backend) Delete(key string) error {
@ -28,6 +30,9 @@ func (b S3Backend) Delete(key string) error {
Key: aws.String(key),
}
_, err := b.svc.DeleteObject(input)
if err != nil {
return err
}
return os.Remove(path.Join(b.bucket, key))
}
@ -40,61 +45,99 @@ func (b S3Backend) Exists(key string) (bool, error) {
return err == nil, err
}
func (b S3Backend) Get(key string) ([]byte, error) {
input := &s3.GetObjectInput{
func (b S3Backend) Head(key string) (metadata backends.Metadata, err error) {
input := &s3.HeadObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
}
result, err := b.svc.GetObject(input)
result, err := b.svc.HeadObject(input)
if err != nil {
return []byte{}, err
return
}
defer result.Body.Close()
return ioutil.ReadAll(result.Body)
metadata, err = unmapMetadata(result.Metadata)
return
}
func (b S3Backend) Put(key string, r io.Reader) (int64, error) {
uploader := s3manager.NewUploaderWithClient(b.svc)
input := &s3manager.UploadInput{
func (b S3Backend) Get(key string) (metadata backends.Metadata, r io.ReadCloser, err error) {
input := &s3.GetObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
Body: r,
}
result, err := uploader.Upload(input)
result, err := b.svc.GetObject(input)
if err != nil {
return 0, err
return
}
return -1, nil
metadata, err = unmapMetadata(result.Metadata)
r = result.Body
return
}
func (b S3Backend) Open(key string) (backends.ReadSeekCloser, error) {
input := &s3.GetObjectInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
func mapMetadata(m backends.Metadata) map[string]*string {
return map[string]*string{
"expiry": aws.String(strconv.FormatInt(m.Expiry.Unix(), 10)),
"delete_key": aws.String(m.DeleteKey),
"size": aws.String(strconv.FormatInt(m.Size, 10)),
"mimetype": aws.String(m.Mimetype),
"sha256sum": aws.String(m.Sha256sum),
}
result, err := b.svc.GetObject(input)
}
func unmapMetadata(input map[string]*string) (m backends.Metadata, err error) {
expiry, err := strconv.ParseInt(*input["expiry"], 10, 64)
if err != nil {
return nil, err
return
}
m.Expiry = time.Unix(expiry, 0)
m.Size, err = strconv.ParseInt(*input["size"], 10, 64)
if err != nil {
return
}
return result.Body, nil
m.DeleteKey = *input["delete_key"]
m.Mimetype = *input["mimetype"]
m.Sha256sum = *input["sha256sum"]
return
}
func (b S3Backend) ServeFile(key string, w http.ResponseWriter, r *http.Request) {
input := &s3.GetObjectInput{
func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) {
tmpDst, err := ioutil.TempFile("", "linx-server-upload")
if err != nil {
return
}
defer tmpDst.Close()
defer os.Remove(tmpDst.Name())
bytes, err := io.Copy(tmpDst, r)
if bytes == 0 {
return m, errors.New("Empty file")
} else if err != nil {
return m, err
}
m.Expiry = expiry
m.DeleteKey = deleteKey
m.Size = bytes
m.Mimetype, _ = helpers.DetectMime(tmpDst)
m.Sha256sum, _ = helpers.Sha256sum(tmpDst)
// XXX: we may not be able to write this to AWS easily
//m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, tmpDst)
uploader := s3manager.NewUploaderWithClient(b.svc)
input := &s3manager.UploadInput{
Bucket: aws.String(b.bucket),
Key: aws.String(key),
Body: tmpDst,
Metadata: mapMetadata(m),
}
result, err := b.svc.GetObject(input)
_, err = uploader.Upload(input)
if err != nil {
return err
return
}
defer result.Body.Close()
http.ServeContent(w, r, key, *result.LastModified, result.Body)
return nil
return
}
func (b S3Backend) Size(key string) (int64, error) {
@ -139,7 +182,7 @@ func (b S3Backend) GetTorrent(fileName string, url string) (t torrent.Torrent, e
func (b S3Backend) List() ([]string, error) {
var output []string
input := &s3.ListObjectsInput{
bucket: aws.String(b.bucket),
Bucket: aws.String(b.bucket),
}
results, err := b.svc.ListObjects(input)

19
backends/storage.go
View File

@ -1,26 +1,19 @@
package backends
import (
"errors"
"io"
"net/http"
"time"
"github.com/andreimarcu/linx-server/torrent"
)
type ReadSeekCloser interface {
io.Reader
io.Closer
io.Seeker
io.ReaderAt
}
type StorageBackend interface {
Delete(key string) error
Exists(key string) (bool, error)
Get(key string) ([]byte, error)
Put(key string, r io.Reader) (int64, error)
Open(key string) (ReadSeekCloser, error)
ServeFile(key string, w http.ResponseWriter, r *http.Request) error
Head(key string) (Metadata, error)
Get(key string) (Metadata, io.ReadCloser, error)
Put(key string, r io.Reader, expiry time.Time, deleteKey string) (Metadata, error)
Size(key string) (int64, error)
GetTorrent(fileName string, url string) (torrent.Torrent, error)
}
@ -29,3 +22,5 @@ type MetaStorageBackend interface {
StorageBackend
List() ([]string, error)
}
var NotFoundErr = errors.New("File not found.")

19
delete.go
View File

@ -3,8 +3,8 @@ package main
import (
"fmt"
"net/http"
"os"
"github.com/andreimarcu/linx-server/backends"
"github.com/zenazn/goji/web"
)
@ -13,24 +13,19 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
filename := c.URLParams["name"]
// Ensure requested file actually exists
if _, readErr := fileBackend.Exists(filename); os.IsNotExist(readErr) {
// Ensure that file exists and delete key is correct
metadata, err := storageBackend.Head(filename)
if err == backends.NotFoundErr {
notFoundHandler(c, w, r) // 404 - file doesn't exist
return
}
// Ensure delete key is correct
metadata, err := metadataRead(filename)
if err != nil {
} else if err != nil {
unauthorizedHandler(c, w, r) // 401 - no metadata available
return
}
if metadata.DeleteKey == requestKey {
fileDelErr := fileBackend.Delete(filename)
metaDelErr := metaStorageBackend.Delete(filename)
if (fileDelErr != nil) || (metaDelErr != nil) {
err := storageBackend.Delete(filename)
if err != nil {
oopsHandler(c, w, r, RespPLAIN, "Could not delete")
return
}

27
display.go
View File

@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"io/ioutil"
"net/http"
"path/filepath"
"regexp"
@ -9,6 +10,7 @@ import (
"strings"
"time"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dustin/go-humanize"
"github.com/flosch/pongo2"
@ -30,12 +32,12 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
_, err := checkFile(fileName)
if err == NotFoundErr {
if err == backends.NotFoundErr {
notFoundHandler(c, w, r)
return
}
metadata, err := metadataRead(fileName)
metadata, err := storageBackend.Head(fileName)
if err != nil {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
@ -77,8 +79,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
tpl = Templates["display/pdf.html"]
} else if extension == "story" {
metadata, reader, err := storageBackend.Get(fileName)
if err != nil {
oopsHandler(c, w, r, RespHTML, err.Error())
}
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := fileBackend.Get(fileName)
bytes, err := ioutil.ReadAll(reader)
if err == nil {
extra["contents"] = string(bytes)
lines = strings.Split(extra["contents"], "\n")
@ -87,8 +94,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
} else if extension == "md" {
metadata, reader, err := storageBackend.Get(fileName)
if err != nil {
oopsHandler(c, w, r, RespHTML, err.Error())
}
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := fileBackend.Get(fileName)
bytes, err := ioutil.ReadAll(reader)
if err == nil {
unsafe := blackfriday.MarkdownCommon(bytes)
html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
@ -99,8 +111,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
} else if strings.HasPrefix(metadata.Mimetype, "text/") || supportedBinExtension(extension) {
metadata, reader, err := storageBackend.Get(fileName)
if err != nil {
oopsHandler(c, w, r, RespHTML, err.Error())
}
if metadata.Size < maxDisplayFileSizeBytes {
bytes, err := fileBackend.Get(fileName)
bytes, err := ioutil.ReadAll(reader)
if err == nil {
extra["extension"] = extension
extra["lang_hl"], extra["lang_ace"] = extensionToHlAndAceLangs(extension)

2
expiry.go
View File

@ -24,7 +24,7 @@ type ExpirationTime struct {
// Determine if the given filename is expired
func isFileExpired(filename string) (bool, error) {
metadata, err := metadataRead(filename)
metadata, err := storageBackend.Head(filename)
if err != nil {
return false, err
}

37
fileserve.go
View File

@ -1,8 +1,10 @@
package main
import (
"io"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/andreimarcu/linx-server/backends"
@ -14,7 +16,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
metadata, err := checkFile(fileName)
if err == NotFoundErr {
if err == backends.NotFoundErr {
notFoundHandler(c, w, r)
return
} else if err == backends.BadMetadata {
@ -38,12 +40,26 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy)
w.Header().Set("Referrer-Policy", Config.fileReferrerPolicy)
_, reader, err := storageBackend.Get(fileName)
if err != nil {
oopsHandler(c, w, r, RespAUTO, err.Error())
}
w.Header().Set("Content-Type", metadata.Mimetype)
w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
w.Header().Set("Etag", metadata.Sha256sum)
w.Header().Set("Cache-Control", "max-age=0")
err = fileBackend.ServeFile(fileName, w, r)
if err != nil {
oopsHandler(c, w, r, RespAUTO, err.Error())
// TODO: implement If-Match, If-None-Match (ETag)
// TODO: implement If-Unmodified-Since, If-Modified-Since (Last-Modified)
// TODO: implement range requests?
if r.Method != "HEAD" {
defer reader.Close()
if _, err = io.CopyN(w, reader, metadata.Size); err != nil {
oopsHandler(c, w, r, RespAUTO, err.Error())
}
}
}
@ -72,21 +88,14 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) {
}
func checkFile(filename string) (metadata backends.Metadata, err error) {
_, err = fileBackend.Exists(filename)
if err != nil {
err = NotFoundErr
return
}
metadata, err = metadataRead(filename)
metadata, err = storageBackend.Head(filename)
if err != nil {
return
}
if expiry.IsTsExpired(metadata.Expiry) {
fileBackend.Delete(filename)
metaStorageBackend.Delete(filename)
err = NotFoundErr
storageBackend.Delete(filename)
err = backends.NotFoundErr
return
}

70
helpers/archive.go
View File

@ -0,0 +1,70 @@
package helpers
import (
"archive/tar"
"archive/zip"
"compress/bzip2"
"compress/gzip"
"io"
"sort"
)
type ReadSeekerAt interface {
io.Reader
io.Seeker
io.ReaderAt
}
func ListArchiveFiles(mimetype string, size int64, r ReadSeekerAt) (files []string, err error) {
if mimetype == "application/x-tar" {
tReadr := tar.NewReader(r)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
files = append(files, hdr.Name)
}
}
sort.Strings(files)
} else if mimetype == "application/x-gzip" {
gzf, err := gzip.NewReader(r)
if err == nil {
tReadr := tar.NewReader(gzf)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
files = append(files, hdr.Name)
}
}
sort.Strings(files)
}
} else if mimetype == "application/x-bzip" {
bzf := bzip2.NewReader(r)
tReadr := tar.NewReader(bzf)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
files = append(files, hdr.Name)
}
}
sort.Strings(files)
} else if mimetype == "application/zip" {
zf, err := zip.NewReader(r, size)
if err == nil {
for _, f := range zf.File {
files = append(files, f.Name)
}
}
sort.Strings(files)
}
return
}

67
helpers/helpers.go
View File

@ -0,0 +1,67 @@
package helpers
import (
"crypto/sha256"
"encoding/hex"
"io"
"unicode"
"gopkg.in/h2non/filetype.v1"
)
func DetectMime(r io.ReadSeeker) (string, error) {
// Get first 512 bytes for mimetype detection
header := make([]byte, 512)
r.Seek(0, 0)
r.Read(header)
r.Seek(0, 0)
kind, err := filetype.Match(header)
if err != nil {
return "application/octet-stream", err
} else if kind.MIME.Value != "" {
return kind.MIME.Value, nil
}
// Check if the file seems anything like text
if printable(header) {
return "text/plain", nil
} else {
return "application/octet-stream", nil
}
}
func Sha256sum(r io.ReadSeeker) (string, error) {
hasher := sha256.New()
r.Seek(0, 0)
_, err := io.Copy(hasher, r)
if err != nil {
return "", err
}
r.Seek(0, 0)
return hex.EncodeToString(hasher.Sum(nil)), nil
}
func printable(data []byte) bool {
for i, b := range data {
r := rune(b)
// A null terminator that's not at the beginning of the file
if r == 0 && i == 0 {
return false
} else if r == 0 && i < 0 {
continue
}
if r > unicode.MaxASCII {
return false
}
}
return true
}

165
meta.go
View File

@ -1,165 +0,0 @@
package main
import (
"archive/tar"
"archive/zip"
"compress/bzip2"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"sort"
"time"
"unicode"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dchest/uniuri"
"gopkg.in/h2non/filetype.v1"
)
var NotFoundErr = errors.New("File not found.")
func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) {
file, err := fileBackend.Open(fName)
if err != nil {
return
}
defer file.Close()
m.Size, err = fileBackend.Size(fName)
if err != nil {
return
}
m.Expiry = exp
if delKey == "" {
m.DeleteKey = uniuri.NewLen(30)
} else {
m.DeleteKey = delKey
}
// Get first 512 bytes for mimetype detection
header := make([]byte, 512)
file.Read(header)
kind, err := filetype.Match(header)
if err != nil {
m.Mimetype = "application/octet-stream"
} else {
m.Mimetype = kind.MIME.Value
}
if m.Mimetype == "" {
// Check if the file seems anything like text
if printable(header) {
m.Mimetype = "text/plain"
} else {
m.Mimetype = "application/octet-stream"
}
}
// Compute the sha256sum
hasher := sha256.New()
file.Seek(0, 0)
_, err = io.Copy(hasher, file)
if err == nil {
m.Sha256sum = hex.EncodeToString(hasher.Sum(nil))
}
file.Seek(0, 0)
// If archive, grab list of filenames
if m.Mimetype == "application/x-tar" {
tReadr := tar.NewReader(file)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
} else if m.Mimetype == "application/x-gzip" {
gzf, err := gzip.NewReader(file)
if err == nil {
tReadr := tar.NewReader(gzf)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
}
} else if m.Mimetype == "application/x-bzip" {
bzf := bzip2.NewReader(file)
tReadr := tar.NewReader(bzf)
for {
hdr, err := tReadr.Next()
if err == io.EOF || err != nil {
break
}
if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
}
}
sort.Strings(m.ArchiveFiles)
} else if m.Mimetype == "application/zip" {
zf, err := zip.NewReader(file, m.Size)
if err == nil {
for _, f := range zf.File {
m.ArchiveFiles = append(m.ArchiveFiles, f.Name)
}
}
sort.Strings(m.ArchiveFiles)
}
return
}
func metadataWrite(filename string, metadata *backends.Metadata) error {
return metaBackend.Put(filename, metadata)
}
func metadataRead(filename string) (metadata backends.Metadata, err error) {
metadata, err = metaBackend.Get(filename)
if err != nil {
// Metadata does not exist, generate one
newMData, err := generateMetadata(filename, expiry.NeverExpire, "")
if err != nil {
return metadata, err
}
metadataWrite(filename, &newMData)
metadata, err = metaBackend.Get(filename)
}
return
}
func printable(data []byte) bool {
for i, b := range data {
r := rune(b)
// A null terminator that's not at the beginning of the file
if r == 0 && i == 0 {
return false
} else if r == 0 && i < 0 {
continue
}
if r > unicode.MaxASCII {
return false
}
}
return true
}

22
server.go
View File

@ -16,7 +16,7 @@ import (
"github.com/GeertJohan/go.rice"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/backends/metajson"
"github.com/andreimarcu/linx-server/backends/s3"
"github.com/flosch/pongo2"
"github.com/vharitonsky/iniflags"
"github.com/zenazn/goji/graceful"
@ -60,6 +60,9 @@ var Config struct {
remoteAuthFile string
addHeaders headerList
noDirectAgents bool
s3Endpoint string
s3Region string
s3Bucket string
}
var Templates = make(map[string]*pongo2.Template)
@ -69,8 +72,7 @@ var timeStarted time.Time
var timeStartedStr string
var remoteAuthKeys []string
var metaStorageBackend backends.MetaStorageBackend
var metaBackend backends.MetaBackend
var fileBackend backends.StorageBackend
var storageBackend backends.StorageBackend
func setup() *web.Mux {
mux := web.New()
@ -129,9 +131,11 @@ func setup() *web.Mux {
Config.sitePath = "/"
}
metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir)
metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend)
fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
if Config.s3Bucket != "" {
storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint)
} else {
storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir)
}
// Template setup
p2l, err := NewPongo2TemplatesLoader()
@ -247,6 +251,12 @@ func main() {
"Add an arbitrary header to the response. This option can be used multiple times.")
flag.BoolVar(&Config.noDirectAgents, "nodirectagents", false,
"disable serving files directly for wget/curl user agents")
flag.StringVar(&Config.s3Endpoint, "s3-endpoint", "",
"S3 endpoint")
flag.StringVar(&Config.s3Region, "s3-region", "",
"S3 region")
flag.StringVar(&Config.s3Bucket, "s3-bucket", "",
"S3 bucket to use for files and metadata")
iniflags.Parse()

4
torrent.go
View File

@ -14,7 +14,7 @@ import (
func createTorrent(fileName string, r *http.Request) ([]byte, error) {
url := fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName)
t, err := fileBackend.GetTorrent(fileName, url)
t, err := storageBackend.GetTorrent(fileName, url)
if err != nil {
return []byte{}, err
}
@ -31,7 +31,7 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
fileName := c.URLParams["name"]
_, err := checkFile(fileName)
if err == NotFoundErr {
if err == backends.NotFoundErr {
notFoundHandler(c, w, r)
return
} else if err == backends.BadMetadata {

22
upload.go
View File

@ -237,11 +237,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
upload.Filename = strings.Join([]string{barename, extension}, ".")
upload.Filename = strings.Replace(upload.Filename, " ", "", -1)
fileexists, _ := fileBackend.Exists(upload.Filename)
fileexists, _ := storageBackend.Exists(upload.Filename)
// Check if the delete key matches, in which case overwrite
if fileexists {
metad, merr := metadataRead(upload.Filename)
metad, merr := storageBackend.Head(upload.Filename)
if merr == nil {
if upReq.deletionKey == metad.DeleteKey {
fileexists = false
@ -258,7 +258,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
}
upload.Filename = strings.Join([]string{barename, extension}, ".")
fileexists, err = fileBackend.Exists(upload.Filename)
fileexists, err = storageBackend.Exists(upload.Filename)
}
if fileBlacklist[strings.ToLower(upload.Filename)] {
@ -273,21 +273,15 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
fileExpiry = time.Now().Add(upReq.expiry)
}
_, err = fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
if err != nil {
return upload, err
if upReq.deletionKey == "" {
upReq.deletionKey = uniuri.NewLen(30)
}
upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey)
if err != nil {
fileBackend.Delete(upload.Filename)
return
}
err = metadataWrite(upload.Filename, &upload.Metadata)
upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deletionKey)
if err != nil {
fileBackend.Delete(upload.Filename)
return
return upload, err
}
return
}

Write Preview
Loading…
Cancel
Save