From fcd18eceec5c637d9a102dc63698e270fee35103 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Mon, 6 Jun 2016 23:37:42 -0700 Subject: [PATCH] use abstracted storage for flexibility I moved the storage functionality into the StorageBackend interface, which is currently only implemented by LocalfsBackend. --- backends/backends.go | 23 ++++++++++++ backends/localfs/localfs.go | 70 +++++++++++++++++++++++++++++++++++++ delete.go | 9 ++--- display.go | 12 +++---- fileserve.go | 13 +++---- meta.go | 27 ++++++-------- server.go | 7 ++++ torrent.go | 21 +++++------ torrent_test.go | 17 +++++++-- upload.go | 32 +++++------------ 10 files changed, 154 insertions(+), 77 deletions(-) create mode 100644 backends/backends.go create mode 100644 backends/localfs/localfs.go diff --git a/backends/backends.go b/backends/backends.go new file mode 100644 index 0000000..42a33f0 --- /dev/null +++ b/backends/backends.go @@ -0,0 +1,23 @@ +package backends + +import ( + "io" + "net/http" +) + +type ReadSeekCloser interface { + io.Reader + io.Closer + io.Seeker + io.ReaderAt +} + +type StorageBackend interface { + Delete(key string) error + Exists(key string) (bool, error) + Get(key string) ([]byte, error) + Put(key string, r io.Reader) (int64, error) + Open(key string) (ReadSeekCloser, error) + ServeFile(key string, w http.ResponseWriter, r *http.Request) + Size(key string) (int64, error) +} diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go new file mode 100644 index 0000000..148cf2e --- /dev/null +++ b/backends/localfs/localfs.go @@ -0,0 +1,70 @@ +package localfs + +import ( + "errors" + "io" + "io/ioutil" + "net/http" + "os" + "path" + + "github.com/andreimarcu/linx-server/backends" +) + +type LocalfsBackend struct { + basePath string +} + +func (b LocalfsBackend) Delete(key string) error { + return os.Remove(path.Join(b.basePath, key)) +} + +func (b LocalfsBackend) Exists(key string) (bool, error) { + _, err := os.Stat(path.Join(b.basePath, key)) + return err == nil, err +} + +func (b LocalfsBackend) Get(key string) ([]byte, error) { + return ioutil.ReadFile(path.Join(b.basePath, key)) +} + +func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) { + dst, err := os.Create(path.Join(b.basePath, key)) + if err != nil { + return 0, err + } + defer dst.Close() + + bytes, err := io.Copy(dst, r) + if bytes == 0 { + b.Delete(key) + return bytes, errors.New("Empty file") + } else if err != nil { + b.Delete(key) + return bytes, err + } + + return bytes, err +} + +func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) { + return os.Open(path.Join(b.basePath, key)) +} + +func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) { + filePath := path.Join(b.basePath, key) + http.ServeFile(w, r, filePath) +} + +func (b LocalfsBackend) Size(key string) (int64, error) { + fileInfo, err := os.Stat(path.Join(b.basePath, key)) + if err != nil { + return 0, err + } + + return fileInfo.Size(), nil +} + +func NewLocalfsBackend(basePath string) LocalfsBackend { + return LocalfsBackend{basePath: basePath} +} diff --git a/delete.go b/delete.go index f727f60..e42e623 100644 --- a/delete.go +++ b/delete.go @@ -4,7 +4,6 @@ import ( "fmt" "net/http" "os" - "path" "github.com/zenazn/goji/web" ) @@ -13,11 +12,9 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) { requestKey := r.Header.Get("Linx-Delete-Key") filename := c.URLParams["name"] - filePath := path.Join(Config.filesDir, filename) - metaPath := path.Join(Config.metaDir, filename) // Ensure requested file actually exists - if _, readErr := os.Stat(filePath); os.IsNotExist(readErr) { + if _, readErr := fileBackend.Exists(filename); os.IsNotExist(readErr) { notFoundHandler(c, w, r) // 404 - file doesn't exist return } @@ -30,8 +27,8 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) { } if metadata.DeleteKey == requestKey { - fileDelErr := os.Remove(filePath) - metaDelErr := os.Remove(metaPath) + fileDelErr := fileBackend.Delete(filename) + metaDelErr := metaBackend.Delete(filename) if (fileDelErr != nil) || (metaDelErr != nil) { oopsHandler(c, w, r, RespPLAIN, "Could not delete") diff --git a/display.go b/display.go index 45727c8..9105119 100644 --- a/display.go +++ b/display.go @@ -2,10 +2,7 @@ package main import ( "encoding/json" - "io/ioutil" "net/http" - "os" - "path" "path/filepath" "strconv" "strings" @@ -22,7 +19,6 @@ const maxDisplayFileSizeBytes = 1024 * 512 func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - filePath := path.Join(Config.filesDir, fileName) err := checkFile(fileName) if err == NotFoundErr { @@ -43,7 +39,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { extra := make(map[string]string) lines := []string{} - file, _ := os.Open(filePath) + file, _ := fileBackend.Open(fileName) defer file.Close() header := make([]byte, 512) @@ -79,7 +75,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } else if extension == "story" { if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := ioutil.ReadFile(filePath) + bytes, err := fileBackend.Get(fileName) if err == nil { extra["contents"] = string(bytes) lines = strings.Split(extra["contents"], "\n") @@ -89,7 +85,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } else if extension == "md" { if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := ioutil.ReadFile(filePath) + bytes, err := fileBackend.Get(fileName) if err == nil { unsafe := blackfriday.MarkdownCommon(bytes) html := bluemonday.UGCPolicy().SanitizeBytes(unsafe) @@ -101,7 +97,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } else if strings.HasPrefix(metadata.Mimetype, "text/") || supportedBinExtension(extension) { if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := ioutil.ReadFile(filePath) + bytes, err := fileBackend.Get(fileName) if err == nil { extra["extension"] = extension extra["lang_hl"], extra["lang_ace"] = extensionToHlAndAceLangs(extension) diff --git a/fileserve.go b/fileserve.go index c7eac21..1a48a74 100644 --- a/fileserve.go +++ b/fileserve.go @@ -3,8 +3,6 @@ package main import ( "net/http" "net/url" - "os" - "path" "strings" "github.com/zenazn/goji/web" @@ -12,7 +10,6 @@ import ( func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - filePath := path.Join(Config.filesDir, fileName) err := checkFile(fileName) if err == NotFoundErr { @@ -35,7 +32,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy) - http.ServeFile(w, r, filePath) + fileBackend.ServeFile(fileName, w, r) } func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) { @@ -63,9 +60,7 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) { } func checkFile(filename string) error { - filePath := path.Join(Config.filesDir, filename) - - _, err := os.Stat(filePath) + _, err := fileBackend.Exists(filename) if err != nil { return NotFoundErr } @@ -76,8 +71,8 @@ func checkFile(filename string) error { } if expired { - os.Remove(path.Join(Config.filesDir, filename)) - os.Remove(path.Join(Config.metaDir, filename)) + fileBackend.Delete(filename) + metaBackend.Delete(filename) return NotFoundErr } diff --git a/meta.go b/meta.go index 0c6928d..7487176 100644 --- a/meta.go +++ b/meta.go @@ -3,6 +3,7 @@ package main import ( "archive/tar" "archive/zip" + "bytes" "compress/bzip2" "compress/gzip" "crypto/sha256" @@ -10,9 +11,6 @@ import ( "encoding/json" "errors" "io" - "io/ioutil" - "os" - "path" "sort" "time" "unicode" @@ -43,14 +41,17 @@ var NotFoundErr = errors.New("File not found.") var BadMetadata = errors.New("Corrupted metadata.") func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) { - file, err := os.Open(path.Join(Config.filesDir, fName)) - fileInfo, err := os.Stat(path.Join(Config.filesDir, fName)) + file, err := fileBackend.Open(fName) if err != nil { return } defer file.Close() - m.Size = fileInfo.Size() + m.Size, err = fileBackend.Size(fName) + if err != nil { + return + } + m.Expiry = exp if delKey == "" { @@ -138,12 +139,6 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e } func metadataWrite(filename string, metadata *Metadata) error { - file, err := os.Create(path.Join(Config.metaDir, filename)) - if err != nil { - return err - } - defer file.Close() - mjson := MetadataJSON{} mjson.DeleteKey = metadata.DeleteKey mjson.Mimetype = metadata.Mimetype @@ -157,8 +152,7 @@ func metadataWrite(filename string, metadata *Metadata) error { return err } - _, err = file.Write(byt) - if err != nil { + if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil { return err } @@ -166,7 +160,7 @@ func metadataWrite(filename string, metadata *Metadata) error { } func metadataRead(filename string) (metadata Metadata, err error) { - b, err := ioutil.ReadFile(path.Join(Config.metaDir, filename)) + b, err := metaBackend.Get(filename) if err != nil { // Metadata does not exist, generate one newMData, err := generateMetadata(filename, neverExpire, "") @@ -174,7 +168,8 @@ func metadataRead(filename string) (metadata Metadata, err error) { return metadata, err } metadataWrite(filename, &newMData) - b, err = ioutil.ReadFile(path.Join(Config.metaDir, filename)) + + b, err = metaBackend.Get(filename) if err != nil { return metadata, BadMetadata } diff --git a/server.go b/server.go index e7eee43..6e0b279 100644 --- a/server.go +++ b/server.go @@ -14,6 +14,8 @@ import ( "time" "github.com/GeertJohan/go.rice" + "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/backends/localfs" "github.com/flosch/pongo2" "github.com/vharitonsky/iniflags" "github.com/zenazn/goji/graceful" @@ -61,6 +63,8 @@ var staticBox *rice.Box var timeStarted time.Time var timeStartedStr string var remoteAuthKeys []string +var metaBackend backends.StorageBackend +var fileBackend backends.StorageBackend func setup() *web.Mux { mux := web.New() @@ -118,6 +122,9 @@ func setup() *web.Mux { Config.sitePath = "/" } + metaBackend = localfs.NewLocalfsBackend(Config.metaDir) + fileBackend = localfs.NewLocalfsBackend(Config.filesDir) + // Template setup p2l, err := NewPongo2TemplatesLoader() if err != nil { diff --git a/torrent.go b/torrent.go index 2e82598..f0f4d48 100644 --- a/torrent.go +++ b/torrent.go @@ -6,8 +6,6 @@ import ( "fmt" "io" "net/http" - "os" - "path" "time" "github.com/zeebo/bencode" @@ -37,7 +35,7 @@ func hashPiece(piece []byte) []byte { return h.Sum(nil) } -func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, error) { +func createTorrent(fileName string, f io.ReadCloser, r *http.Request) ([]byte, error) { chunk := make([]byte, TORRENT_PIECE_LENGTH) torrent := Torrent{ @@ -49,11 +47,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e UrlList: []string{fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName)}, } - f, err := os.Open(filePath) - if err != nil { - return []byte{}, err - } - for { n, err := f.Read(chunk) if err == io.EOF { @@ -66,8 +59,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e torrent.Info.Pieces += string(hashPiece(chunk[:n])) } - f.Close() - data, err := bencode.EncodeBytes(&torrent) if err != nil { return []byte{}, err @@ -78,7 +69,6 @@ func createTorrent(fileName string, filePath string, r *http.Request) ([]byte, e func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - filePath := path.Join(Config.filesDir, fileName) err := checkFile(fileName) if err == NotFoundErr { @@ -89,7 +79,14 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { return } - encoded, err := createTorrent(fileName, filePath, r) + f, err := fileBackend.Open(fileName) + if err != nil { + oopsHandler(c, w, r, RespHTML, "Could not create torrent.") + return + } + defer f.Close() + + encoded, err := createTorrent(fileName, f, r) if err != nil { oopsHandler(c, w, r, RespHTML, "Could not create torrent.") return diff --git a/torrent_test.go b/torrent_test.go index 03a51da..38132f2 100644 --- a/torrent_test.go +++ b/torrent_test.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "testing" "github.com/zeebo/bencode" @@ -11,7 +12,13 @@ func TestCreateTorrent(t *testing.T) { fileName := "server.go" var decoded Torrent - encoded, err := createTorrent(fileName, fileName, nil) + f, err := os.Open("server.go") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + encoded, err := createTorrent(fileName, f, nil) if err != nil { t.Fatal(err) } @@ -47,7 +54,13 @@ func TestCreateTorrent(t *testing.T) { func TestCreateTorrentWithImage(t *testing.T) { var decoded Torrent - encoded, err := createTorrent("test.jpg", "static/images/404.jpg", nil) + f, err := os.Open("static/images/404.jpg") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + encoded, err := createTorrent("test.jpg", f, nil) if err != nil { t.Fatal(err) } diff --git a/upload.go b/upload.go index 33d845d..b8b395b 100644 --- a/upload.go +++ b/upload.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "net/url" - "os" "path" "path/filepath" "regexp" @@ -232,9 +231,8 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { upload.Filename = strings.Join([]string{barename, extension}, ".") - _, err = os.Stat(path.Join(Config.filesDir, upload.Filename)) + fileexists, _ := fileBackend.Exists(upload.Filename) - fileexists := err == nil // Check if the delete key matches, in which case overwrite if fileexists { metad, merr := metadataRead(upload.Filename) @@ -254,20 +252,13 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { } upload.Filename = strings.Join([]string{barename, extension}, ".") - _, err = os.Stat(path.Join(Config.filesDir, upload.Filename)) - fileexists = err == nil + fileexists, err = fileBackend.Exists(upload.Filename) } if fileBlacklist[strings.ToLower(upload.Filename)] { return upload, errors.New("Prohibited filename") } - dst, err := os.Create(path.Join(Config.filesDir, upload.Filename)) - if err != nil { - return - } - defer dst.Close() - // Get the rest of the metadata needed for storage var expiry time.Time if upReq.expiry == 0 { @@ -276,29 +267,22 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { expiry = time.Now().Add(upReq.expiry) } - bytes, err := io.Copy(dst, io.MultiReader(bytes.NewReader(header), upReq.src)) - if bytes == 0 { - os.Remove(path.Join(Config.filesDir, upload.Filename)) - return upload, errors.New("Empty file") - - } else if err != nil { - os.Remove(path.Join(Config.filesDir, upload.Filename)) - return + bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src)) + if err != nil { + return upload, err } else if bytes > Config.maxSize { - os.Remove(path.Join(Config.filesDir, upload.Filename)) + fileBackend.Delete(upload.Filename) return upload, errors.New("File too large") } upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey) if err != nil { - os.Remove(path.Join(Config.filesDir, upload.Filename)) - os.Remove(path.Join(Config.metaDir, upload.Filename)) + fileBackend.Delete(upload.Filename) return } err = metadataWrite(upload.Filename, &upload.Metadata) if err != nil { - os.Remove(path.Join(Config.filesDir, upload.Filename)) - os.Remove(path.Join(Config.metaDir, upload.Filename)) + fileBackend.Delete(upload.Filename) return } return