From 97c5237e979b3730f48a6299860608ee5343cb14 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Wed, 23 Jan 2019 22:34:55 -0800 Subject: [PATCH] Move torrent creation out of backend Although S3 offers a GetObjectTorrent API call to generate a torrent file on their end, it doesn't look like any similar systems with S3 compatible APIs actually support it. Notably, Minio and Wasabi do not. In order to remain compatible with these, it's better to not rely on the storage backend to handle creation. --- backends/localfs/localfs.go | 34 ----------------- backends/s3/s3.go | 28 -------------- backends/storage.go | 3 -- torrent.go | 43 +++++++++++++++++---- torrent_test.go | 74 +++++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 72 deletions(-) create mode 100644 torrent_test.go diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index ec1c27b..3f6f5ad 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -10,7 +10,6 @@ import ( "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/helpers" - "github.com/andreimarcu/linx-server/torrent" ) type LocalfsBackend struct { @@ -152,39 +151,6 @@ func (b LocalfsBackend) Size(key string) (int64, error) { return fileInfo.Size(), nil } -func (b LocalfsBackend) GetTorrent(fileName string, url string) (t torrent.Torrent, err error) { - chunk := make([]byte, torrent.TORRENT_PIECE_LENGTH) - - t = torrent.Torrent{ - Encoding: "UTF-8", - Info: torrent.TorrentInfo{ - PieceLength: torrent.TORRENT_PIECE_LENGTH, - Name: fileName, - }, - UrlList: []string{url}, - } - - _, f, err := b.Get(fileName) - if err != nil { - return - } - defer f.Close() - - for { - n, err := f.Read(chunk) - if err == io.EOF { - break - } else if err != nil { - return t, err - } - - t.Info.Length += n - t.Info.Pieces += string(torrent.HashPiece(chunk[:n])) - } - - return -} - func (b LocalfsBackend) List() ([]string, error) { var output []string diff --git a/backends/s3/s3.go b/backends/s3/s3.go index b453747..515f997 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -9,12 +9,10 @@ import ( "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/helpers" - "github.com/andreimarcu/linx-server/torrent" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/zeebo/bencode" ) type S3Backend struct { @@ -151,32 +149,6 @@ func (b S3Backend) Size(key string) (int64, error) { return *result.ContentLength, nil } -func (b S3Backend) GetTorrent(fileName string, url string) (t torrent.Torrent, err error) { - input := &s3.GetObjectTorrentInput{ - Bucket: aws.String(b.bucket), - Key: aws.String(fileName), - } - result, err := b.svc.GetObjectTorrent(input) - if err != nil { - return - } - defer result.Body.Close() - - data, err := ioutil.ReadAll(result.Body) - if err != nil { - return - } - - err = bencode.DecodeBytes(data, &t) - if err != nil { - return - } - - t.Info.Name = fileName - t.UrlList = []string{url} - return -} - func (b S3Backend) List() ([]string, error) { var output []string input := &s3.ListObjectsInput{ diff --git a/backends/storage.go b/backends/storage.go index 5dbdc13..d40a2b9 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -4,8 +4,6 @@ import ( "errors" "io" "time" - - "github.com/andreimarcu/linx-server/torrent" ) type StorageBackend interface { @@ -15,7 +13,6 @@ type StorageBackend interface { Get(key string) (Metadata, io.ReadCloser, error) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (Metadata, error) Size(key string) (int64, error) - GetTorrent(fileName string, url string) (torrent.Torrent, error) } type MetaStorageBackend interface { diff --git a/torrent.go b/torrent.go index 50585c1..f6a5505 100644 --- a/torrent.go +++ b/torrent.go @@ -3,20 +3,40 @@ package main import ( "bytes" "fmt" + "io" "net/http" "time" "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/expiry" + "github.com/andreimarcu/linx-server/torrent" "github.com/zeebo/bencode" "github.com/zenazn/goji/web" ) -func createTorrent(fileName string, r *http.Request) ([]byte, error) { - url := fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName) +func createTorrent(fileName string, f io.Reader, r *http.Request) ([]byte, error) { + url := getSiteURL(r) + Config.selifPath + fileName + chunk := make([]byte, torrent.TORRENT_PIECE_LENGTH) - t, err := storageBackend.GetTorrent(fileName, url) - if err != nil { - return []byte{}, err + t := torrent.Torrent{ + Encoding: "UTF-8", + Info: torrent.TorrentInfo{ + PieceLength: torrent.TORRENT_PIECE_LENGTH, + Name: fileName, + }, + UrlList: []string{url}, + } + + for { + n, err := f.Read(chunk) + if err == io.EOF { + break + } else if err != nil { + return []byte{}, err + } + + t.Info.Length += n + t.Info.Pieces += string(torrent.HashPiece(chunk[:n])) } data, err := bencode.EncodeBytes(&t) @@ -30,16 +50,25 @@ func createTorrent(fileName string, r *http.Request) ([]byte, error) { func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - _, err := checkFile(fileName) + metadata, f, err := storageBackend.Get(fileName) if err == backends.NotFoundErr { notFoundHandler(c, w, r) return } else if err == backends.BadMetadata { oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") return + } else if err != nil { + oopsHandler(c, w, r, RespAUTO, "Could not create torrent.") + } + defer f.Close() + + if expiry.IsTsExpired(metadata.Expiry) { + storageBackend.Delete(fileName) + notFoundHandler(c, w, r) + return } - encoded, err := createTorrent(fileName, r) + encoded, err := createTorrent(fileName, f, r) if err != nil { oopsHandler(c, w, r, RespHTML, "Could not create torrent.") return diff --git a/torrent_test.go b/torrent_test.go new file mode 100644 index 0000000..1d227fd --- /dev/null +++ b/torrent_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/andreimarcu/linx-server/torrent" + "github.com/zeebo/bencode" +) + +func TestCreateTorrent(t *testing.T) { + fileName := "server.go" + var decoded torrent.Torrent + + f, err := os.Open("server.go") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + encoded, err := createTorrent(fileName, f, nil) + if err != nil { + t.Fatal(err) + } + + bencode.DecodeBytes(encoded, &decoded) + + if decoded.Encoding != "UTF-8" { + t.Fatalf("Encoding was %s, expected UTF-8", decoded.Encoding) + } + + if decoded.Info.Name != "server.go" { + t.Fatalf("Name was %s, expected server.go", decoded.Info.Name) + } + + if decoded.Info.PieceLength <= 0 { + t.Fatal("Expected a piece length, got none") + } + + if len(decoded.Info.Pieces) <= 0 { + t.Fatal("Expected at least one piece, got none") + } + + if decoded.Info.Length <= 0 { + t.Fatal("Length was less than or equal to 0, expected more") + } + + tracker := fmt.Sprintf("%s%s%s", Config.siteURL, Config.selifPath, fileName) + if decoded.UrlList[0] != tracker { + t.Fatalf("First entry in URL list was %s, expected %s", decoded.UrlList[0], tracker) + } +} + +func TestCreateTorrentWithImage(t *testing.T) { + var decoded torrent.Torrent + + f, err := os.Open("static/images/404.jpg") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + encoded, err := createTorrent("test.jpg", f, nil) + if err != nil { + t.Fatal(err) + } + + bencode.DecodeBytes(encoded, &decoded) + + if decoded.Info.Pieces != "r\x01\x80j\x99\x84\n\xd3dZ;1NX\xec;\x9d$+f" { + t.Fatal("Torrent pieces did not match expected pieces for image") + } +}