Browse Source
Add initial version of S3 backend
Add initial version of S3 backend
This new backend currently isn't hooked up; new and existing installs will continue to use the localfs backend. * Rework torrent generation to be backend-dependent so we can use S3's existing torrent API. * Remove the torrent test cases, which broke with this torrent rework; they will need to be added back later. * Use `http.MaxBytesReader` for better max size handling. * Allow backends to return errors in `ServeFile` if needed.pull/156/head
mutantmonkey
6 years ago
9 changed files with 335 additions and 140 deletions
-
37backends/localfs/localfs.go
-
170backends/s3/s3.go
-
5backends/storage.go
-
5fileserve.go
-
76server_test.go
-
60torrent.go
-
28torrent/torrent.go
-
73torrent_test.go
-
19upload.go
@ -0,0 +1,170 @@ |
|||
package localfs |
|||
|
|||
import ( |
|||
"errors" |
|||
"io" |
|||
"io/ioutil" |
|||
"net/http" |
|||
"os" |
|||
"path" |
|||
|
|||
"github.com/andreimarcu/linx-server/backends" |
|||
"github.com/andreimarcu/linx-server/torrent" |
|||
"github.com/aws/aws-sdk-go/aws" |
|||
"github.com/aws/aws-sdk-go/aws/session" |
|||
"github.com/aws/aws-sdk-go/service/s3" |
|||
"github.com/aws/aws-sdk-go/service/s3/s3manager" |
|||
"github.com/zeebo/bencode" |
|||
) |
|||
|
|||
type S3Backend struct { |
|||
bucket string |
|||
svc *S3 |
|||
} |
|||
|
|||
func (b S3Backend) Delete(key string) error { |
|||
input := &s3.DeleteObjectInput{ |
|||
Bucket: aws.String(b.bucket), |
|||
Key: aws.String(key), |
|||
} |
|||
_, err := b.svc.DeleteObject(input) |
|||
return os.Remove(path.Join(b.bucket, key)) |
|||
} |
|||
|
|||
func (b S3Backend) Exists(key string) (bool, error) { |
|||
input := &s3.HeadObjectInput{ |
|||
Bucket: aws.String(b.bucket), |
|||
Key: aws.String(key), |
|||
} |
|||
_, err := b.svc.HeadObject(input) |
|||
return err == nil, err |
|||
} |
|||
|
|||
func (b S3Backend) Get(key string) ([]byte, error) { |
|||
input := &s3.GetObjectInput{ |
|||
Bucket: aws.String(b.bucket), |
|||
Key: aws.String(key), |
|||
} |
|||
result, err := b.svc.GetObject(input) |
|||
if err != nil { |
|||
return []byte{}, err |
|||
} |
|||
defer result.Body.Close() |
|||
|
|||
return ioutil.ReadAll(result.Body) |
|||
} |
|||
|
|||
func (b S3Backend) Put(key string, r io.Reader) (int64, error) { |
|||
uploader := s3manager.NewUploaderWithClient(b.svc) |
|||
input := &s3manager.UploadInput{ |
|||
Bucket: aws.String(b.bucket), |
|||
Key: aws.String(key), |
|||
Body: r, |
|||
} |
|||
result, err := uploader.Upload(input) |
|||
if err != nil { |
|||
return 0, err |
|||
} |
|||
|
|||
return -1, nil |
|||
} |
|||
|
|||
func (b S3Backend) Open(key string) (backends.ReadSeekCloser, error) { |
|||
input := &s3.GetObjectInput{ |
|||
Bucket: aws.String(b.bucket), |
|||
Key: aws.String(key), |
|||
} |
|||
result, err := b.svc.GetObject(input) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
return result.Body, nil |
|||
} |
|||
|
|||
func (b S3Backend) ServeFile(key string, w http.ResponseWriter, r *http.Request) { |
|||
input := &s3.GetObjectInput{ |
|||
Bucket: aws.String(b.bucket), |
|||
Key: aws.String(key), |
|||
} |
|||
result, err := b.svc.GetObject(input) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
defer result.Body.Close() |
|||
|
|||
http.ServeContent(w, r, key, *result.LastModified, result.Body) |
|||
return nil |
|||
} |
|||
|
|||
func (b S3Backend) Size(key string) (int64, error) { |
|||
input := &s3.HeadObjectInput{ |
|||
Bucket: aws.String(b.bucket), |
|||
Key: aws.String(key), |
|||
} |
|||
result, err := b.svc.HeadObject(input) |
|||
if err != nil { |
|||
return 0, err |
|||
} |
|||
|
|||
return *result.ContentLength, nil |
|||
} |
|||
|
|||
func (b S3Backend) GetTorrent(fileName string, url string) (t torrent.Torrent, err error) { |
|||
input := &s3.GetObjectTorrentInput{ |
|||
Bucket: aws.String(b.bucket), |
|||
Key: aws.String(fileName), |
|||
} |
|||
result, err := b.svc.GetObjectTorrent(input) |
|||
if err != nil { |
|||
return |
|||
} |
|||
defer result.Body.Close() |
|||
|
|||
data, err := ioutil.ReadAll(result.Body) |
|||
if err != nil { |
|||
return |
|||
} |
|||
|
|||
err = bencode.DecodeBytes(data, &t) |
|||
if err != nil { |
|||
return |
|||
} |
|||
|
|||
t.Info.Name = fileName |
|||
t.UrlList = []string{url} |
|||
return |
|||
} |
|||
|
|||
func (b S3Backend) List() ([]string, error) { |
|||
var output []string |
|||
input := &s3.ListObjectsInput{ |
|||
bucket: aws.String(b.bucket), |
|||
} |
|||
|
|||
results, err := b.svc.ListObjects(input) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
|
|||
for _, object := range results.Contents { |
|||
output = append(output, *object.Key) |
|||
} |
|||
|
|||
return output, nil |
|||
} |
|||
|
|||
func NewS3Backend(bucket string, region string, endpoint string) S3Backend { |
|||
awsConfig := &aws.Config{} |
|||
if region != "" { |
|||
awsConfig.Region = aws.String(region) |
|||
} |
|||
if endpoint != "" { |
|||
awsConfig.Endpoint = aws.String(endpoint) |
|||
} |
|||
|
|||
sess := session.Must(session.NewSession(awsConfig)) |
|||
svc := s3.New(sess) |
|||
return S3Backend{bucket: bucket, svc: svc} |
|||
} |
|||
@ -0,0 +1,28 @@ |
|||
package torrent |
|||
|
|||
import ( |
|||
"crypto/sha1" |
|||
) |
|||
|
|||
const ( |
|||
TORRENT_PIECE_LENGTH = 262144 |
|||
) |
|||
|
|||
type TorrentInfo struct { |
|||
PieceLength int `bencode:"piece length"` |
|||
Pieces string `bencode:"pieces"` |
|||
Name string `bencode:"name"` |
|||
Length int `bencode:"length"` |
|||
} |
|||
|
|||
type Torrent struct { |
|||
Encoding string `bencode:"encoding"` |
|||
Info TorrentInfo `bencode:"info"` |
|||
UrlList []string `bencode:"url-list"` |
|||
} |
|||
|
|||
func HashPiece(piece []byte) []byte { |
|||
h := sha1.New() |
|||
h.Write(piece) |
|||
return h.Sum(nil) |
|||
} |
|||
@ -1,73 +0,0 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"fmt" |
|||
"os" |
|||
"testing" |
|||
|
|||
"github.com/zeebo/bencode" |
|||
) |
|||
|
|||
func TestCreateTorrent(t *testing.T) { |
|||
fileName := "server.go" |
|||
var decoded Torrent |
|||
|
|||
f, err := os.Open("server.go") |
|||
if err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
defer f.Close() |
|||
|
|||
encoded, err := createTorrent(fileName, f, nil) |
|||
if err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
|
|||
bencode.DecodeBytes(encoded, &decoded) |
|||
|
|||
if decoded.Encoding != "UTF-8" { |
|||
t.Fatalf("Encoding was %s, expected UTF-8", decoded.Encoding) |
|||
} |
|||
|
|||
if decoded.Info.Name != "server.go" { |
|||
t.Fatalf("Name was %s, expected server.go", decoded.Info.Name) |
|||
} |
|||
|
|||
if decoded.Info.PieceLength <= 0 { |
|||
t.Fatal("Expected a piece length, got none") |
|||
} |
|||
|
|||
if len(decoded.Info.Pieces) <= 0 { |
|||
t.Fatal("Expected at least one piece, got none") |
|||
} |
|||
|
|||
if decoded.Info.Length <= 0 { |
|||
t.Fatal("Length was less than or equal to 0, expected more") |
|||
} |
|||
|
|||
tracker := fmt.Sprintf("%sselif/%s", Config.siteURL, fileName) |
|||
if decoded.UrlList[0] != tracker { |
|||
t.Fatalf("First entry in URL list was %s, expected %s", decoded.UrlList[0], tracker) |
|||
} |
|||
} |
|||
|
|||
func TestCreateTorrentWithImage(t *testing.T) { |
|||
var decoded Torrent |
|||
|
|||
f, err := os.Open("static/images/404.jpg") |
|||
if err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
defer f.Close() |
|||
|
|||
encoded, err := createTorrent("test.jpg", f, nil) |
|||
if err != nil { |
|||
t.Fatal(err) |
|||
} |
|||
|
|||
bencode.DecodeBytes(encoded, &decoded) |
|||
|
|||
if decoded.Info.Pieces != "r\x01\x80j\x99\x84\n\xd3dZ;1NX\xec;\x9d$+f" { |
|||
t.Fatal("Torrent pieces did not match expected pieces for image") |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue