From 091225b9e4fae2c6a467f564686853718570ca96 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Mon, 28 Sep 2015 22:58:14 -0700 Subject: [PATCH 1/5] add torrent support This change adds an option to download files with BitTorrent. A webseed is provided in the torrent file to bootstrap the swarm. --- server.go | 2 + server_test.go | 18 +++++++ templates/display/base.html | 1 + torrent.go | 96 +++++++++++++++++++++++++++++++++++++ torrent_test.go | 43 +++++++++++++++++ 5 files changed, 160 insertions(+) create mode 100644 torrent.go create mode 100644 torrent_test.go diff --git a/server.go b/server.go index efd4c83..44f6deb 100644 --- a/server.go +++ b/server.go @@ -68,6 +68,7 @@ func setup() { nameRe := regexp.MustCompile(`^/(?P[a-z0-9-\.]+)$`) selifRe := regexp.MustCompile(`^/selif/(?P[a-z0-9-\.]+)$`) selifIndexRe := regexp.MustCompile(`^/selif/$`) + torrentRe := regexp.MustCompile(`^/(?P[a-z0-9-\.]+)/torrent$`) goji.Get("/", indexHandler) @@ -83,6 +84,7 @@ func setup() { goji.Get(nameRe, fileDisplayHandler) goji.Get(selifRe, fileServeHandler) goji.Get(selifIndexRe, unauthorizedHandler) + goji.Get(torrentRe, fileTorrentHandler) goji.NotFound(notFoundHandler) } diff --git a/server_test.go b/server_test.go index fc8b7f8..f029e17 100644 --- a/server_test.go +++ b/server_test.go @@ -371,6 +371,15 @@ func TestPutAndDelete(t *testing.T) { if w.Code != 404 { t.Fatal("Status code was not 404, but " + strconv.Itoa(w.Code)) } + + // Make sure torrent is also gone + w = httptest.NewRecorder() + req, err = http.NewRequest("GET", "/"+myjson.Filename+"/torrent", nil) + goji.DefaultMux.ServeHTTP(w, req) + + if w.Code != 404 { + t.Fatal("Status code was not 404, but " + strconv.Itoa(w.Code)) + } } func TestPutAndSpecificDelete(t *testing.T) { @@ -418,6 +427,15 @@ func TestPutAndSpecificDelete(t *testing.T) { if w.Code != 404 { t.Fatal("Status code was not 404, but " + strconv.Itoa(w.Code)) } + + // Make sure torrent is gone too + w = httptest.NewRecorder() + req, err = http.NewRequest("GET", "/"+myjson.Filename+"/torrent", nil) + goji.DefaultMux.ServeHTTP(w, req) + + if w.Code != 404 { + t.Fatal("Status code was not 404, but " + strconv.Itoa(w.Code)) + } } func TestShutdown(t *testing.T) { diff --git a/templates/display/base.html b/templates/display/base.html index f250f82..f919f84 100644 --- a/templates/display/base.html +++ b/templates/display/base.html @@ -13,6 +13,7 @@ {% block infoleft %}{% endblock %}
+ torrent | get
diff --git a/torrent.go b/torrent.go new file mode 100644 index 0000000..19094f2 --- /dev/null +++ b/torrent.go @@ -0,0 +1,96 @@ +package main + +import ( + "bytes" + "crypto/sha1" + "fmt" + "io" + "net/http" + "os" + "path" + "time" + + "github.com/zeebo/bencode" + "github.com/zenazn/goji/web" +) + +const ( + TORRENT_PIECE_LENGTH = 262144 +) + +func check(e error) { + if e != nil { + panic(e) + } +} + +type TorrentInfo struct { + PieceLength int `bencode:"piece length"` + Pieces []byte `bencode:"pieces"` + Name string `bencode:"name"` + Length int `bencode:"length"` +} + +type Torrent struct { + Encoding string `bencode:"encoding"` + Info TorrentInfo `bencode:"info"` + UrlList []string `bencode:"url-list"` +} + +func CreateTorrent(fileName string, filePath string) []byte { + chunk := make([]byte, TORRENT_PIECE_LENGTH) + var pieces []byte + length := 0 + + f, err := os.Open(filePath) + check(err) + + for { + n, err := f.Read(chunk) + if err == io.EOF { + break + } + check(err) + + length += n + + h := sha1.New() + h.Write(chunk) + pieces = append(pieces, h.Sum(nil)...) + } + + f.Close() + + torrent := &Torrent{ + Encoding: "UTF-8", + Info: TorrentInfo{ + PieceLength: TORRENT_PIECE_LENGTH, + Pieces: pieces, + Name: fileName, + Length: length, + }, + UrlList: []string{fmt.Sprintf("%sselif/%s", Config.siteURL, fileName)}, + } + + data, err := bencode.EncodeBytes(torrent) + check(err) + + return data +} + +func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { + fileName := c.URLParams["name"] + filePath := path.Join(Config.filesDir, fileName) + + if !fileExistsAndNotExpired(fileName) { + notFoundHandler(c, w, r) + return + } + + encoded := CreateTorrent(fileName, filePath) + + w.Header().Set(`Content-Disposition`, fmt.Sprintf(`attachment; filename="%s.torrent"`, fileName)) + http.ServeContent(w, r, "", time.Now(), bytes.NewReader(encoded)) +} + +// vim:set ts=8 sw=8 noet: diff --git a/torrent_test.go b/torrent_test.go new file mode 100644 index 0000000..dff738b --- /dev/null +++ b/torrent_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + "testing" + + "github.com/zeebo/bencode" +) + +func TestCreateTorrent(t *testing.T) { + fileName := "server.go" + encoded := CreateTorrent(fileName, fileName) + var decoded Torrent + + bencode.DecodeBytes(encoded, &decoded) + + if decoded.Encoding != "UTF-8" { + t.Fatalf("Encoding was %s, expected UTF-8", decoded.Encoding) + } + + if decoded.Info.Name != "server.go" { + t.Fatalf("Name was %s, expected server.go", decoded.Info.Name) + } + + if decoded.Info.PieceLength <= 0 { + t.Fatal("Expected a piece length, got none") + } + + if len(decoded.Info.Pieces) <= 0 { + t.Fatal("Expected at least one piece, got none") + } + + if decoded.Info.Length <= 0 { + t.Fatal("Length was less than or equal to 0, expected more") + } + + tracker := fmt.Sprintf("%sselif/%s", Config.siteURL, fileName) + if decoded.UrlList[0] != tracker { + t.Fatal("First entry in URL list was %s, expected %s", decoded.UrlList[0], tracker) + } +} + +// vim:set ts=8 sw=8 noet: From b81477c1d36ed02fb7b483505f979281601fd437 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Mon, 28 Sep 2015 23:03:40 -0700 Subject: [PATCH 2/5] fix go vet complaint in torrent_test.go --- torrent_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torrent_test.go b/torrent_test.go index dff738b..9b96e22 100644 --- a/torrent_test.go +++ b/torrent_test.go @@ -36,7 +36,7 @@ func TestCreateTorrent(t *testing.T) { tracker := fmt.Sprintf("%sselif/%s", Config.siteURL, fileName) if decoded.UrlList[0] != tracker { - t.Fatal("First entry in URL list was %s, expected %s", decoded.UrlList[0], tracker) + t.Fatalf("First entry in URL list was %s, expected %s", decoded.UrlList[0], tracker) } } From df09b005defd4e4f2836e3acd02231da64c830ea Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Tue, 29 Sep 2015 08:41:42 -0700 Subject: [PATCH 3/5] use oopsHandler instead of panicking on error --- torrent.go | 27 +++++++++++++++------------ torrent_test.go | 6 +++++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/torrent.go b/torrent.go index 19094f2..33714cb 100644 --- a/torrent.go +++ b/torrent.go @@ -18,12 +18,6 @@ const ( TORRENT_PIECE_LENGTH = 262144 ) -func check(e error) { - if e != nil { - panic(e) - } -} - type TorrentInfo struct { PieceLength int `bencode:"piece length"` Pieces []byte `bencode:"pieces"` @@ -37,20 +31,23 @@ type Torrent struct { UrlList []string `bencode:"url-list"` } -func CreateTorrent(fileName string, filePath string) []byte { +func CreateTorrent(fileName string, filePath string) ([]byte, error) { chunk := make([]byte, TORRENT_PIECE_LENGTH) var pieces []byte length := 0 f, err := os.Open(filePath) - check(err) + if err != nil { + return []byte{}, err + } for { n, err := f.Read(chunk) if err == io.EOF { break + } else if err != nil { + return []byte{}, err } - check(err) length += n @@ -73,9 +70,11 @@ func CreateTorrent(fileName string, filePath string) []byte { } data, err := bencode.EncodeBytes(torrent) - check(err) + if err != nil { + return []byte{}, err + } - return data + return data, nil } func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { @@ -87,7 +86,11 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { return } - encoded := CreateTorrent(fileName, filePath) + encoded, err := CreateTorrent(fileName, filePath) + if err != nil { + oopsHandler(c, w, r) // 500 - creating torrent failed + return + } w.Header().Set(`Content-Disposition`, fmt.Sprintf(`attachment; filename="%s.torrent"`, fileName)) http.ServeContent(w, r, "", time.Now(), bytes.NewReader(encoded)) diff --git a/torrent_test.go b/torrent_test.go index 9b96e22..6d9afbb 100644 --- a/torrent_test.go +++ b/torrent_test.go @@ -9,9 +9,13 @@ import ( func TestCreateTorrent(t *testing.T) { fileName := "server.go" - encoded := CreateTorrent(fileName, fileName) var decoded Torrent + encoded, err := CreateTorrent(fileName, fileName) + if err != nil { + t.Fatal(err) + } + bencode.DecodeBytes(encoded, &decoded) if decoded.Encoding != "UTF-8" { From baca561f06b55b4ed985addb16bb4c2578399f4a Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Tue, 29 Sep 2015 20:12:50 -0700 Subject: [PATCH 4/5] fix torrent creation for binary data and refactor --- torrent.go | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/torrent.go b/torrent.go index 33714cb..5235c35 100644 --- a/torrent.go +++ b/torrent.go @@ -20,7 +20,7 @@ const ( type TorrentInfo struct { PieceLength int `bencode:"piece length"` - Pieces []byte `bencode:"pieces"` + Pieces string `bencode:"pieces"` Name string `bencode:"name"` Length int `bencode:"length"` } @@ -31,10 +31,23 @@ type Torrent struct { UrlList []string `bencode:"url-list"` } +func hashPiece(piece []byte) []byte { + h := sha1.New() + h.Write(piece) + return h.Sum(nil) +} + func CreateTorrent(fileName string, filePath string) ([]byte, error) { chunk := make([]byte, TORRENT_PIECE_LENGTH) - var pieces []byte - length := 0 + + torrent := Torrent{ + Encoding: "UTF-8", + Info: TorrentInfo{ + PieceLength: TORRENT_PIECE_LENGTH, + Name: fileName, + }, + UrlList: []string{fmt.Sprintf("%sselif/%s", Config.siteURL, fileName)}, + } f, err := os.Open(filePath) if err != nil { @@ -49,27 +62,13 @@ func CreateTorrent(fileName string, filePath string) ([]byte, error) { return []byte{}, err } - length += n - - h := sha1.New() - h.Write(chunk) - pieces = append(pieces, h.Sum(nil)...) + torrent.Info.Length += n + torrent.Info.Pieces += string(hashPiece(chunk[:n])) } f.Close() - torrent := &Torrent{ - Encoding: "UTF-8", - Info: TorrentInfo{ - PieceLength: TORRENT_PIECE_LENGTH, - Pieces: pieces, - Name: fileName, - Length: length, - }, - UrlList: []string{fmt.Sprintf("%sselif/%s", Config.siteURL, fileName)}, - } - - data, err := bencode.EncodeBytes(torrent) + data, err := bencode.EncodeBytes(&torrent) if err != nil { return []byte{}, err } From 31aa0d666b80c8ce6a9d3e601b1a2cd06f2edc7f Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Tue, 29 Sep 2015 20:13:14 -0700 Subject: [PATCH 5/5] add torrent test for binary data --- torrent_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/torrent_test.go b/torrent_test.go index 6d9afbb..5a1d34a 100644 --- a/torrent_test.go +++ b/torrent_test.go @@ -44,4 +44,19 @@ func TestCreateTorrent(t *testing.T) { } } +func TestCreateTorrentWithImage(t *testing.T) { + var decoded Torrent + + encoded, err := CreateTorrent("test.jpg", "static/images/404.jpg") + if err != nil { + t.Fatal(err) + } + + bencode.DecodeBytes(encoded, &decoded) + + if decoded.Info.Pieces != "r\x01\x80j\x99\x84\n\xd3dZ;1NX\xec;\x9d$+f" { + t.Fatal("Torrent pieces did not match expected pieces for image") + } +} + // vim:set ts=8 sw=8 noet: