From 5d8a0ef605e535f838000ce21f0de581f287a133 Mon Sep 17 00:00:00 2001 From: Thor77 Date: Wed, 7 Nov 2018 19:13:27 +0100 Subject: [PATCH 01/30] Serve file directly for curl and wget user agents (#145) * Serve file directly for curl and wget user agents Fix #127 * Add test for get with wget user agent * Add -nodirectagents flag to disable serving files directly for wget/curl user agents * Fix TestPutAndGetCLI failing for Go 1.5 It failed because it doesn't include the Content-Type header for every response. --- display.go | 8 ++++++++ server.go | 3 +++ server_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/display.go b/display.go index c6d8470..4220c76 100644 --- a/display.go +++ b/display.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -18,7 +19,14 @@ import ( const maxDisplayFileSizeBytes = 1024 * 512 +var cliUserAgentRe = regexp.MustCompile("(?i)(lib)?curl|wget") + func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { + if !Config.noDirectAgents && cliUserAgentRe.MatchString(r.Header.Get("User-Agent")) { + fileServeHandler(c, w, r) + return + } + fileName := c.URLParams["name"] err := checkFile(fileName) diff --git a/server.go b/server.go index daaf89e..ba0efb8 100644 --- a/server.go +++ b/server.go @@ -58,6 +58,7 @@ var Config struct { remoteAuthFile string addHeaders headerList googleShorterAPIKey string + noDirectAgents bool } var Templates = make(map[string]*pongo2.Template) @@ -243,6 +244,8 @@ func main() { "Add an arbitrary header to the response. This option can be used multiple times.") flag.StringVar(&Config.googleShorterAPIKey, "googleapikey", "", "API Key for Google's URL Shortener.") + flag.BoolVar(&Config.noDirectAgents, "nodirectagents", false, + "disable serving files directly for wget/curl user agents") iniflags.Parse() diff --git a/server_test.go b/server_test.go index 6fe363c..7727f0c 100644 --- a/server_test.go +++ b/server_test.go @@ -1121,3 +1121,50 @@ func TestShutdown(t *testing.T) { os.RemoveAll(Config.filesDir) os.RemoveAll(Config.metaDir) } + +func TestPutAndGetCLI(t *testing.T) { + var myjson RespOkJSON + mux := setup() + + // upload file + w := httptest.NewRecorder() + req, err := http.NewRequest("PUT", "/upload", strings.NewReader("File content")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Accept", "application/json") + mux.ServeHTTP(w, req) + + err = json.Unmarshal([]byte(w.Body.String()), &myjson) + if err != nil { + t.Fatal(err) + } + + // request file without wget user agent + w = httptest.NewRecorder() + req, err = http.NewRequest("GET", myjson.Url, nil) + if err != nil { + t.Fatal(err) + } + mux.ServeHTTP(w, req) + + contentType := w.Header().Get("Content-Type") + if strings.HasPrefix(contentType, "text/plain") { + t.Fatalf("Didn't receive file display page but %s", contentType) + } + + // request file with wget user agent + w = httptest.NewRecorder() + req, err = http.NewRequest("GET", myjson.Url, nil) + req.Header.Set("User-Agent", "wget") + if err != nil { + t.Fatal(err) + } + mux.ServeHTTP(w, req) + + contentType = w.Header().Get("Content-Type") + if !strings.HasPrefix(contentType, "text/plain") { + t.Fatalf("Didn't receive file directly but %s", contentType) + } + +} From f19247a79059590b76c98274c24002917efef017 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Tue, 8 Jan 2019 11:18:57 -0800 Subject: [PATCH 02/30] Update Travis to 1.10 and 1.11 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index eaf499f..cd71721 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - 1.5 - - 1.6 + - 1.10 + - 1.11 before_script: - go vet ./... From 19a95e36a6549313c0d3fbb44112d0417d327b45 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Tue, 8 Jan 2019 11:28:10 -0800 Subject: [PATCH 03/30] Fix Travis parsing of 1.10 as 1.1 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd71721..7baaebc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - 1.10 - - 1.11 + - "1.10" + - "1.11" before_script: - go vet ./... From bad7d2666e053cb9bb1381e3d4f6fdf6107cf3f8 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Tue, 8 Jan 2019 19:56:09 +0000 Subject: [PATCH 04/30] Switch to Referrer-Policy header (#149) Use of the Content-Security-Policy header to specify a referrer policy was deprecated in favor of a [new header](https://github.com/w3c/webappsec-referrer-policy/commit/fc55d917bee0d5636f52a19a5aefa65f8995c766). This change changes the existing referrer policy directives to use this header and adds corresponding config options/command line flags. --- README.md | 6 ++++-- csp.go | 11 +++++++++-- csp_test.go | 11 +++++++---- fileserve.go | 1 + server.go | 17 +++++++++++++---- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7fa6b68..e1a5c8a 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,10 @@ allowhotlink = true - ```-maxsize 4294967296``` -- maximum upload file size in bytes (default 4GB) - ```-maxexpiry 86400``` -- maximum expiration time in seconds (default is 0, which is no expiry) - ```-allowhotlink``` -- Allow file hotlinking -- ```-contentsecuritypolicy "..."``` -- Content-Security-Policy header for pages (default is "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-ancestors 'self'; referrer origin;") -- ```-filecontentsecuritypolicy "..."``` -- Content-Security-Policy header for files (default is "default-src 'none'; img-src 'self'; object-src 'self'; media-src 'self'; style-src 'self' 'unsafe-inline'; frame-ancestors 'self'; referrer origin;") +- ```-contentsecuritypolicy "..."``` -- Content-Security-Policy header for pages (default is "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-ancestors 'self';") +- ```-filecontentsecuritypolicy "..."``` -- Content-Security-Policy header for files (default is "default-src 'none'; img-src 'self'; object-src 'self'; media-src 'self'; style-src 'self' 'unsafe-inline'; frame-ancestors 'self';") +- ```-refererpolicy "..."``` -- Referrer-Policy header for pages (default is "same-origin") +- ```-filereferrerpolicy "..."``` -- Referrer-Policy header for files (default is "same-origin") - ```-xframeoptions "..." ``` -- X-Frame-Options header (default is "SAMEORIGIN") - ```-remoteuploads``` -- (optionally) enable remote uploads (/upload?url=https://...) - ```-nologs``` -- (optionally) disable request logs in stdout diff --git a/csp.go b/csp.go index 098e271..34b73b4 100644 --- a/csp.go +++ b/csp.go @@ -6,6 +6,7 @@ import ( const ( cspHeader = "Content-Security-Policy" + rpHeader = "Referrer-Policy" frameOptionsHeader = "X-Frame-Options" ) @@ -15,8 +16,9 @@ type csp struct { } type CSPOptions struct { - policy string - frame string + policy string + referrerPolicy string + frame string } func (c csp) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -25,6 +27,11 @@ func (c csp) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Add(cspHeader, c.opts.policy) } + // only add a Referrer Policy if one is not already set + if existing := w.Header().Get(rpHeader); existing == "" { + w.Header().Add(rpHeader, c.opts.referrerPolicy) + } + w.Header().Set(frameOptionsHeader, c.opts.frame) c.h.ServeHTTP(w, r) diff --git a/csp_test.go b/csp_test.go index 190c65d..3d1d499 100644 --- a/csp_test.go +++ b/csp_test.go @@ -12,6 +12,7 @@ import ( var testCSPHeaders = map[string]string{ "Content-Security-Policy": "default-src 'none'; style-src 'self';", + "Referrer-Policy": "strict-origin-when-cross-origin", "X-Frame-Options": "SAMEORIGIN", } @@ -22,8 +23,9 @@ func TestContentSecurityPolicy(t *testing.T) { Config.maxSize = 1024 * 1024 * 1024 Config.noLogs = true Config.siteName = "linx" - Config.contentSecurityPolicy = "default-src 'none'; style-src 'self';" - Config.xFrameOptions = "SAMEORIGIN" + Config.contentSecurityPolicy = testCSPHeaders["Content-Security-Policy"] + Config.referrerPolicy = testCSPHeaders["Referrer-Policy"] + Config.xFrameOptions = testCSPHeaders["X-Frame-Options"] mux := setup() w := httptest.NewRecorder() @@ -34,8 +36,9 @@ func TestContentSecurityPolicy(t *testing.T) { } goji.Use(ContentSecurityPolicy(CSPOptions{ - policy: testCSPHeaders["Content-Security-Policy"], - frame: testCSPHeaders["X-Frame-Options"], + policy: testCSPHeaders["Content-Security-Policy"], + referrerPolicy: testCSPHeaders["Referrer-Policy"], + frame: testCSPHeaders["X-Frame-Options"], })) mux.ServeHTTP(w, req) diff --git a/fileserve.go b/fileserve.go index f7f87a6..951bea2 100644 --- a/fileserve.go +++ b/fileserve.go @@ -32,6 +32,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy) + w.Header().Set("Referrer-Policy", Config.fileReferrerPolicy) fileBackend.ServeFile(fileName, w, r) } diff --git a/server.go b/server.go index ba0efb8..5280503 100644 --- a/server.go +++ b/server.go @@ -46,6 +46,8 @@ var Config struct { keyFile string contentSecurityPolicy string fileContentSecurityPolicy string + referrerPolicy string + fileReferrerPolicy string xFrameOptions string maxSize int64 maxExpiry uint64 @@ -88,8 +90,9 @@ func setup() *web.Mux { mux.Use(middleware.Recoverer) mux.Use(middleware.AutomaticOptions) mux.Use(ContentSecurityPolicy(CSPOptions{ - policy: Config.contentSecurityPolicy, - frame: Config.xFrameOptions, + policy: Config.contentSecurityPolicy, + referrerPolicy: Config.referrerPolicy, + frame: Config.xFrameOptions, })) mux.Use(AddHeaders(Config.addHeaders)) @@ -233,11 +236,17 @@ func main() { flag.StringVar(&Config.remoteAuthFile, "remoteauthfile", "", "path to a file containing newline-separated scrypted auth keys for remote uploads") flag.StringVar(&Config.contentSecurityPolicy, "contentsecuritypolicy", - "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-ancestors 'self'; referrer origin;", + "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-ancestors 'self';", "value of default Content-Security-Policy header") flag.StringVar(&Config.fileContentSecurityPolicy, "filecontentsecuritypolicy", - "default-src 'none'; img-src 'self'; object-src 'self'; media-src 'self'; style-src 'self' 'unsafe-inline'; frame-ancestors 'self'; referrer origin;", + "default-src 'none'; img-src 'self'; object-src 'self'; media-src 'self'; style-src 'self' 'unsafe-inline'; frame-ancestors 'self';", "value of Content-Security-Policy header for file access") + flag.StringVar(&Config.referrerPolicy, "referrerpolicy", + "same-origin", + "value of default Referrer-Policy header") + flag.StringVar(&Config.fileReferrerPolicy, "filereferrerpolicy", + "same-origin", + "value of Referrer-Policy header for file access") flag.StringVar(&Config.xFrameOptions, "xframeoptions", "SAMEORIGIN", "value of X-Frame-Options header") flag.Var(&Config.addHeaders, "addheader", From 10938a3e0b72ab6f0d66381ca2e4fa7ff339140f Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Tue, 8 Jan 2019 19:56:32 +0000 Subject: [PATCH 05/30] Remove Google URL shortener (fix #146) (#150) --- README.md | 1 - backends/meta.go | 1 - backends/metajson/metajson.go | 3 -- display.go | 16 +++---- server.go | 8 ---- shorturl.go | 89 ----------------------------------- static/js/shorturl.js | 39 --------------- templates/display/base.html | 13 ----- 8 files changed, 7 insertions(+), 163 deletions(-) delete mode 100644 shorturl.go delete mode 100644 static/js/shorturl.js diff --git a/README.md b/README.md index e1a5c8a..df13b18 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,6 @@ allowhotlink = true - ```-xframeoptions "..." ``` -- X-Frame-Options header (default is "SAMEORIGIN") - ```-remoteuploads``` -- (optionally) enable remote uploads (/upload?url=https://...) - ```-nologs``` -- (optionally) disable request logs in stdout -- ```-googleapikey``` -- (optionally) API Key for Google's URL Shortener. ([How to create one](https://developers.google.com/url-shortener/v1/getting_started#APIKey)) #### SSL with built-in server - ```-certfile path/to/your.crt``` -- Path to the ssl certificate (required if you want to use the https server) diff --git a/backends/meta.go b/backends/meta.go index eb17d5e..27c3e41 100644 --- a/backends/meta.go +++ b/backends/meta.go @@ -17,7 +17,6 @@ type Metadata struct { Size int64 Expiry time.Time ArchiveFiles []string - ShortURL string } var BadMetadata = errors.New("Corrupted metadata.") diff --git a/backends/metajson/metajson.go b/backends/metajson/metajson.go index 6e76dd4..8ec53c4 100644 --- a/backends/metajson/metajson.go +++ b/backends/metajson/metajson.go @@ -15,7 +15,6 @@ type MetadataJSON struct { Size int64 `json:"size"` Expiry int64 `json:"expiry"` ArchiveFiles []string `json:"archive_files,omitempty"` - ShortURL string `json:"short_url"` } type MetaJSONBackend struct { @@ -30,7 +29,6 @@ func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error { mjson.Sha256sum = metadata.Sha256sum mjson.Expiry = metadata.Expiry.Unix() mjson.Size = metadata.Size - mjson.ShortURL = metadata.ShortURL byt, err := json.Marshal(mjson) if err != nil { @@ -63,7 +61,6 @@ func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) metadata.Sha256sum = mjson.Sha256sum metadata.Expiry = time.Unix(mjson.Expiry, 0) metadata.Size = mjson.Size - metadata.ShortURL = mjson.ShortURL return } diff --git a/display.go b/display.go index 4220c76..c17fea6 100644 --- a/display.go +++ b/display.go @@ -116,15 +116,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } err = renderTemplate(tpl, pongo2.Context{ - "mime": metadata.Mimetype, - "filename": fileName, - "size": sizeHuman, - "expiry": expiryHuman, - "extra": extra, - "lines": lines, - "files": metadata.ArchiveFiles, - "shorturlEnabled": Config.googleShorterAPIKey != "", - "shorturl": metadata.ShortURL, + "mime": metadata.Mimetype, + "filename": fileName, + "size": sizeHuman, + "expiry": expiryHuman, + "extra": extra, + "lines": lines, + "files": metadata.ArchiveFiles, }, r, w) if err != nil { diff --git a/server.go b/server.go index 5280503..1dd1f09 100644 --- a/server.go +++ b/server.go @@ -59,7 +59,6 @@ var Config struct { authFile string remoteAuthFile string addHeaders headerList - googleShorterAPIKey string noDirectAgents bool } @@ -154,7 +153,6 @@ func setup() *web.Mux { selifRe := regexp.MustCompile("^" + Config.sitePath + `selif/(?P[a-z0-9-\.]+)$`) selifIndexRe := regexp.MustCompile("^" + Config.sitePath + `selif/$`) torrentRe := regexp.MustCompile("^" + Config.sitePath + `(?P[a-z0-9-\.]+)/torrent$`) - shortRe := regexp.MustCompile("^" + Config.sitePath + `(?P[a-z0-9-\.]+)/short$`) if Config.authFile == "" { mux.Get(Config.sitePath, indexHandler) @@ -193,10 +191,6 @@ func setup() *web.Mux { mux.Get(selifIndexRe, unauthorizedHandler) mux.Get(torrentRe, fileTorrentHandler) - if Config.googleShorterAPIKey != "" { - mux.Get(shortRe, shortURLHandler) - } - mux.NotFound(notFoundHandler) return mux @@ -251,8 +245,6 @@ func main() { "value of X-Frame-Options header") flag.Var(&Config.addHeaders, "addheader", "Add an arbitrary header to the response. This option can be used multiple times.") - flag.StringVar(&Config.googleShorterAPIKey, "googleapikey", "", - "API Key for Google's URL Shortener.") flag.BoolVar(&Config.noDirectAgents, "nodirectagents", false, "disable serving files directly for wget/curl user agents") diff --git a/shorturl.go b/shorturl.go deleted file mode 100644 index afdaf00..0000000 --- a/shorturl.go +++ /dev/null @@ -1,89 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "errors" - "net/http" - - "github.com/zenazn/goji/web" -) - -type shortenerRequest struct { - LongURL string `json:"longUrl"` -} - -type shortenerResponse struct { - Kind string `json:"kind"` - ID string `json:"id"` - LongURL string `json:"longUrl"` - Error struct { - Code int `json:"code"` - Message string `json:"message"` - } `json:"error"` -} - -func shortURLHandler(c web.C, w http.ResponseWriter, r *http.Request) { - fileName := c.URLParams["name"] - - err := checkFile(fileName) - if err == NotFoundErr { - notFoundHandler(c, w, r) - return - } - - metadata, err := metadataRead(fileName) - if err != nil { - oopsHandler(c, w, r, RespJSON, "Corrupt metadata.") - return - } - - if metadata.ShortURL == "" { - url, err := shortenURL(getSiteURL(r) + fileName) - if err != nil { - oopsHandler(c, w, r, RespJSON, err.Error()) - return - } - - metadata.ShortURL = url - - err = metadataWrite(fileName, &metadata) - if err != nil { - oopsHandler(c, w, r, RespJSON, "Corrupt metadata.") - return - } - } - - js, _ := json.Marshal(map[string]string{ - "shortUrl": metadata.ShortURL, - }) - w.Write(js) - return -} - -func shortenURL(url string) (string, error) { - apiURL := "https://www.googleapis.com/urlshortener/v1/url?key=" + Config.googleShorterAPIKey - jsonStr, _ := json.Marshal(shortenerRequest{LongURL: url}) - - req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(jsonStr)) - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - shortenerResponse := new(shortenerResponse) - err = json.NewDecoder(resp.Body).Decode(shortenerResponse) - if err != nil { - return "", err - } - - if shortenerResponse.Error.Message != "" { - return "", errors.New(shortenerResponse.Error.Message) - } - - return shortenerResponse.ID, nil -} diff --git a/static/js/shorturl.js b/static/js/shorturl.js deleted file mode 100644 index 26e0c77..0000000 --- a/static/js/shorturl.js +++ /dev/null @@ -1,39 +0,0 @@ -document.getElementById('shorturl').addEventListener('click', function (e) { - e.preventDefault(); - - if (e.target.href !== "") return; - - xhr = new XMLHttpRequest(); - xhr.open("GET", e.target.dataset.url, true); - xhr.setRequestHeader('Accept', 'application/json'); - xhr.onreadystatechange = function () { - if (xhr.readyState === 4) { - var resp = JSON.parse(xhr.responseText); - - if (xhr.status === 200 && resp.error == null) { - e.target.innerText = resp.shortUrl; - e.target.href = resp.shortUrl; - e.target.setAttribute('aria-label', 'Click to copy into clipboard') - } else { - e.target.setAttribute('aria-label', resp.error) - } - } - }; - xhr.send(); -}); - -var clipboard = new Clipboard("#shorturl", { - text: function (trigger) { - if (trigger.href == null) return; - - return trigger.href; - } -}); - -clipboard.on('success', function (e) { - e.trigger.setAttribute('aria-label', 'Successfully copied') -}); - -clipboard.on('error', function (e) { - e.trigger.setAttribute('aria-label', 'Your browser does not support coping to clipboard') -}); diff --git a/templates/display/base.html b/templates/display/base.html index 587e76f..8f33b46 100644 --- a/templates/display/base.html +++ b/templates/display/base.html @@ -17,15 +17,6 @@ {% endif %} {% block infomore %}{% endblock %} {{ size }} | - {% if shorturlEnabled %} - {% if shorturl %} - {{shorturl}} | - {% else %} - short url | - {% endif %} - {% endif %} torrent | get @@ -43,8 +34,4 @@ - - {% if shorturlEnabled %} - - {% endif %} {% endblock %} From 5f4f16e08be7935eba4d4c465ae97eb10aa1af53 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Wed, 9 Jan 2019 04:28:01 +0000 Subject: [PATCH 06/30] Add file ETag support (fix #138) (#152) --- display.go | 2 +- fileserve.go | 27 ++++++++++++++++++--------- torrent.go | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/display.go b/display.go index c17fea6..d897e5a 100644 --- a/display.go +++ b/display.go @@ -29,7 +29,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - err := checkFile(fileName) + _, err := checkFile(fileName) if err == NotFoundErr { notFoundHandler(c, w, r) return diff --git a/fileserve.go b/fileserve.go index 951bea2..3b20c4c 100644 --- a/fileserve.go +++ b/fileserve.go @@ -6,19 +6,23 @@ import ( "strings" "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/expiry" "github.com/zenazn/goji/web" ) func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - err := checkFile(fileName) + metadata, err := checkFile(fileName) if err == NotFoundErr { notFoundHandler(c, w, r) return } else if err == backends.BadMetadata { oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") return + } else if err != nil { + oopsHandler(c, w, r, RespAUTO, err.Error()) + return } if !Config.allowHotlink { @@ -34,6 +38,9 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy) w.Header().Set("Referrer-Policy", Config.fileReferrerPolicy) + w.Header().Set("Etag", metadata.Sha256sum) + w.Header().Set("Cache-Control", "max-age=0") + fileBackend.ServeFile(fileName, w, r) } @@ -61,22 +68,24 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) { } } -func checkFile(filename string) error { - _, err := fileBackend.Exists(filename) +func checkFile(filename string) (metadata backends.Metadata, err error) { + _, err = fileBackend.Exists(filename) if err != nil { - return NotFoundErr + err = NotFoundErr + return } - expired, err := isFileExpired(filename) + metadata, err = metadataRead(filename) if err != nil { - return err + return } - if expired { + if expiry.IsTsExpired(metadata.Expiry) { fileBackend.Delete(filename) metaStorageBackend.Delete(filename) - return NotFoundErr + err = NotFoundErr + return } - return nil + return } diff --git a/torrent.go b/torrent.go index 23361b5..77cd2fd 100644 --- a/torrent.go +++ b/torrent.go @@ -71,7 +71,7 @@ func createTorrent(fileName string, f io.ReadCloser, r *http.Request) ([]byte, e func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - err := checkFile(fileName) + _, err := checkFile(fileName) if err == NotFoundErr { notFoundHandler(c, w, r) return From 6290f408ff7e995e24b39094ad1fec9449c8da2f Mon Sep 17 00:00:00 2001 From: Benjamin Neff Date: Fri, 11 Jan 2019 18:09:54 +0100 Subject: [PATCH 07/30] Allow to paste images (#153) dropzone.js doesn't support pasting itself yet, so adding it externally and calling `.addFile()` to upload the pasted image. Fixes #130 --- static/js/upload.js | 12 +++++++++++- templates/index.html | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/static/js/upload.js b/static/js/upload.js index 5e54d62..159bad2 100644 --- a/static/js/upload.js +++ b/static/js/upload.js @@ -102,8 +102,18 @@ Dropzone.options.dropzone = { previewsContainer: "#uploads", parallelUploads: 5, headers: {"Accept": "application/json"}, - dictDefaultMessage: "Click or Drop file(s)", + dictDefaultMessage: "Click or Drop file(s) or Paste image", dictFallbackMessage: "" }; +document.onpaste = function(event) { + var items = (event.clipboardData || event.originalEvent.clipboardData).items; + for (index in items) { + var item = items[index]; + if (item.kind === "file") { + Dropzone.forElement("#dropzone").addFile(item.getAsFile()); + } + } +}; + // @end-license diff --git a/templates/index.html b/templates/index.html index 5e95d01..87d821d 100644 --- a/templates/index.html +++ b/templates/index.html @@ -13,7 +13,7 @@
- Click or Drop file(s) + Click or Drop file(s) or Paste image
From c746f70c10e1406a1d59c20086ef962e9c398e4f Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Mon, 14 Jan 2019 14:51:02 -0800 Subject: [PATCH 08/30] Allow changing the "selif" path name --- README.md | 1 + csp_test.go | 1 + server.go | 12 ++++++++++-- server_test.go | 4 ++-- templates.go | 1 + templates/display/audio.html | 4 ++-- templates/display/base.html | 2 +- templates/display/file.html | 2 +- templates/display/image.html | 4 ++-- templates/display/pdf.html | 4 ++-- templates/display/video.html | 4 ++-- torrent.go | 2 +- torrent_test.go | 2 +- 13 files changed, 27 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index df13b18..8d4ef11 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ allowhotlink = true - ```-bind 127.0.0.1:8080``` -- what to bind to (default is 127.0.0.1:8080) - ```-sitename myLinx``` -- the site name displayed on top (default is inferred from Host header) - ```-siteurl "http://mylinx.example.org/"``` -- the site url (default is inferred from execution context) +- ```-selifpath "selif"``` -- path relative to site base url (the "selif" in https://mylinx.example.org/selif/image.jpg) where files are accessed directly (default: selif) - ```-filespath files/``` -- Path to store uploads (default is files/) - ```-metapath meta/``` -- Path to store information about uploads (default is meta/) - ```-maxsize 4294967296``` -- maximum upload file size in bytes (default 4GB) diff --git a/csp_test.go b/csp_test.go index 3d1d499..e3dbbdd 100644 --- a/csp_test.go +++ b/csp_test.go @@ -23,6 +23,7 @@ func TestContentSecurityPolicy(t *testing.T) { Config.maxSize = 1024 * 1024 * 1024 Config.noLogs = true Config.siteName = "linx" + Config.selifPath = "selif" Config.contentSecurityPolicy = testCSPHeaders["Content-Security-Policy"] Config.referrerPolicy = testCSPHeaders["Referrer-Policy"] Config.xFrameOptions = testCSPHeaders["X-Frame-Options"] diff --git a/server.go b/server.go index 1dd1f09..c7b4342 100644 --- a/server.go +++ b/server.go @@ -42,6 +42,7 @@ var Config struct { siteName string siteURL string sitePath string + selifPath string certFile string keyFile string contentSecurityPolicy string @@ -129,6 +130,11 @@ func setup() *web.Mux { Config.sitePath = "/" } + Config.selifPath = strings.TrimLeft(Config.selifPath, "/") + if lastChar := Config.selifPath[len(Config.selifPath)-1:]; lastChar != "/" { + Config.selifPath = Config.selifPath + "/" + } + metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir) metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend) fileBackend = localfs.NewLocalfsBackend(Config.filesDir) @@ -150,8 +156,8 @@ func setup() *web.Mux { // Routing setup nameRe := regexp.MustCompile("^" + Config.sitePath + `(?P[a-z0-9-\.]+)$`) - selifRe := regexp.MustCompile("^" + Config.sitePath + `selif/(?P[a-z0-9-\.]+)$`) - selifIndexRe := regexp.MustCompile("^" + Config.sitePath + `selif/$`) + selifRe := regexp.MustCompile("^" + Config.sitePath + Config.selifPath + `(?P[a-z0-9-\.]+)$`) + selifIndexRe := regexp.MustCompile("^" + Config.sitePath + Config.selifPath + `$`) torrentRe := regexp.MustCompile("^" + Config.sitePath + `(?P[a-z0-9-\.]+)/torrent$`) if Config.authFile == "" { @@ -211,6 +217,8 @@ func main() { "name of the site") flag.StringVar(&Config.siteURL, "siteurl", "", "site base url (including trailing slash)") + flag.StringVar(&Config.selifPath, "selifpath", "selif", + "path relative to site base url where files are accessed directly") flag.Int64Var(&Config.maxSize, "maxsize", 4*1024*1024*1024, "maximum upload file size in bytes (default 4GB)") flag.Uint64Var(&Config.maxExpiry, "maxexpiry", 0, diff --git a/server_test.go b/server_test.go index 7727f0c..d9374ab 100644 --- a/server_test.go +++ b/server_test.go @@ -173,7 +173,7 @@ func TestFileNotFound(t *testing.T) { filename := generateBarename() - req, err := http.NewRequest("GET", "/selif/"+filename, nil) + req, err := http.NewRequest("GET", "/"+Config.selifPath+filename, nil) if err != nil { t.Fatal(err) } @@ -941,7 +941,7 @@ func TestPutAndOverwrite(t *testing.T) { // Make sure it's the new file w = httptest.NewRecorder() - req, err = http.NewRequest("GET", "/selif/"+myjson.Filename, nil) + req, err = http.NewRequest("GET", "/"+Config.selifPath+myjson.Filename, nil) mux.ServeHTTP(w, req) if w.Code == 404 { diff --git a/templates.go b/templates.go index 0687bce..79c90ce 100644 --- a/templates.go +++ b/templates.go @@ -83,6 +83,7 @@ func renderTemplate(tpl *pongo2.Template, context pongo2.Context, r *http.Reques } context["sitepath"] = Config.sitePath + context["selifpath"] = Config.selifPath context["using_auth"] = Config.authFile != "" return tpl.ExecuteWriter(context, writer) diff --git a/templates/display/audio.html b/templates/display/audio.html index 689bab3..b5ae1e3 100644 --- a/templates/display/audio.html +++ b/templates/display/audio.html @@ -2,8 +2,8 @@ {% block main %} {% endblock %} diff --git a/templates/display/base.html b/templates/display/base.html index 8f33b46..172e17a 100644 --- a/templates/display/base.html +++ b/templates/display/base.html @@ -18,7 +18,7 @@ {% block infomore %}{% endblock %} {{ size }} | torrent | - get + get
{% block infoleft %}{% endblock %} diff --git a/templates/display/file.html b/templates/display/file.html index 5eeb424..670651e 100644 --- a/templates/display/file.html +++ b/templates/display/file.html @@ -2,7 +2,7 @@ {% block main %}
-

You are requesting {{ filename }}, click here to download.

+

You are requesting {{ filename }}, click here to download.

{% if files|length > 0 %}

Contents of the archive:

diff --git a/templates/display/image.html b/templates/display/image.html index b1ea7dd..807b7ad 100644 --- a/templates/display/image.html +++ b/templates/display/image.html @@ -1,7 +1,7 @@ {% extends "base.html" %} {% block main %} - - + + {% endblock %} diff --git a/templates/display/pdf.html b/templates/display/pdf.html index 7ad20f1..69501f7 100644 --- a/templates/display/pdf.html +++ b/templates/display/pdf.html @@ -1,10 +1,10 @@ {% extends "base.html" %} {% block main %} - +

It appears your Web browser is not configured to display PDF files. -No worries, just click here to download the PDF file.

+No worries, just click here to download the PDF file.

 {% endblock %} diff --git a/templates/display/video.html b/templates/display/video.html index 9fc90d5..317664b 100644 --- a/templates/display/video.html +++ b/templates/display/video.html @@ -2,7 +2,7 @@ {% block main %} {% endblock %} diff --git a/torrent.go b/torrent.go index 77cd2fd..4155872 100644 --- a/torrent.go +++ b/torrent.go @@ -45,7 +45,7 @@ func createTorrent(fileName string, f io.ReadCloser, r *http.Request) ([]byte, e PieceLength: TORRENT_PIECE_LENGTH, Name: fileName, }, - UrlList: []string{fmt.Sprintf("%sselif/%s", getSiteURL(r), fileName)}, + UrlList: []string{fmt.Sprintf("%s%s%s", getSiteURL(r), Config.selifPath, fileName)}, } for { diff --git a/torrent_test.go b/torrent_test.go index 38132f2..b553231 100644 --- a/torrent_test.go +++ b/torrent_test.go @@ -45,7 +45,7 @@ func TestCreateTorrent(t *testing.T) { t.Fatal("Length was less than or equal to 0, expected more") } - tracker := fmt.Sprintf("%sselif/%s", Config.siteURL, fileName) + tracker := fmt.Sprintf("%s%s%s", Config.siteURL, Config.selifPath, fileName) if decoded.UrlList[0] != tracker { t.Fatalf("First entry in URL list was %s, expected %s", decoded.UrlList[0], tracker) } From 1fb92ffce30270ba46f7d7ba6aa0620b518edda6 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Mon, 14 Jan 2019 15:23:37 -0800 Subject: [PATCH 09/30] Fix bug where using curl with json headers would return the file instead --- display.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/display.go b/display.go index d897e5a..70d8dee 100644 --- a/display.go +++ b/display.go @@ -22,7 +22,7 @@ const maxDisplayFileSizeBytes = 1024 * 512 var cliUserAgentRe = regexp.MustCompile("(?i)(lib)?curl|wget") func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { - if !Config.noDirectAgents && cliUserAgentRe.MatchString(r.Header.Get("User-Agent")) { + if !Config.noDirectAgents && cliUserAgentRe.MatchString(r.Header.Get("User-Agent")) && !strings.EqualFold("application/json", r.Header.Get("Accept")) { fileServeHandler(c, w, r) return } From e506304b844d67f3860fd127605beb8ba3bc2244 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Mon, 14 Jan 2019 15:23:56 -0800 Subject: [PATCH 10/30] Return direct URL in json responses --- display.go | 11 ++++++----- upload.go | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/display.go b/display.go index 70d8dee..ca5bc6f 100644 --- a/display.go +++ b/display.go @@ -52,11 +52,12 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { if strings.EqualFold("application/json", r.Header.Get("Accept")) { js, _ := json.Marshal(map[string]string{ - "filename": fileName, - "expiry": strconv.FormatInt(metadata.Expiry.Unix(), 10), - "size": strconv.FormatInt(metadata.Size, 10), - "mimetype": metadata.Mimetype, - "sha256sum": metadata.Sha256sum, + "filename": fileName, + "direct_url": getSiteURL(r) + Config.selifPath + fileName, + "expiry": strconv.FormatInt(metadata.Expiry.Unix(), 10), + "size": strconv.FormatInt(metadata.Size, 10), + "mimetype": metadata.Mimetype, + "sha256sum": metadata.Sha256sum, }) w.Write(js) return diff --git a/upload.go b/upload.go index b7195c3..acdd204 100644 --- a/upload.go +++ b/upload.go @@ -295,6 +295,7 @@ func generateBarename() string { func generateJSONresponse(upload Upload, r *http.Request) []byte { js, _ := json.Marshal(map[string]string{ "url": getSiteURL(r) + upload.Filename, + "direct_url": getSiteURL(r) + Config.selifPath + upload.Filename, "filename": upload.Filename, "delete_key": upload.Metadata.DeleteKey, "expiry": strconv.FormatInt(upload.Metadata.Expiry.Unix(), 10), From 9d7f698c70be785a46dbe061ef649805c724c219 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Mon, 14 Jan 2019 16:16:15 -0800 Subject: [PATCH 11/30] Add direct_url info to API page --- templates/API.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/API.html b/templates/API.html index 045f4c2..64404b6 100644 --- a/templates/API.html +++ b/templates/API.html @@ -41,6 +41,7 @@

“url”: the publicly available upload url
+ “direct_url”: the url to access the file directly
“filename”: the (optionally generated) filename
“delete_key”: the (optionally generated) deletion key,
“expiry”: the unix timestamp at which the file will expire (0 if never)
@@ -121,6 +122,7 @@ DELETED

“url”: the publicly available upload url
+ “direct_url”: the url to access the file directly
“filename”: the (optionally generated) filename
“expiry”: the unix timestamp at which the file will expire (0 if never)
“size”: the size in bytes of the file
From 5340f23f4d32af329509c6e2fceb6600f85610d3 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Tue, 15 Jan 2019 06:41:03 +0000 Subject: [PATCH 12/30] Add new multi-stage slim Dockerfile (#154) --- Dockerfile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8f02dcc..addad3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,26 @@ -FROM golang:alpine +FROM golang:alpine3.8 AS build + +COPY . /go/src/github.com/andreimarcu/linx-server +WORKDIR /go/src/github.com/andreimarcu/linx-server RUN set -ex \ && apk add --no-cache --virtual .build-deps git \ - && go get github.com/andreimarcu/linx-server \ + && go get -v . \ && apk del .build-deps +FROM alpine:3.8 + +COPY --from=build /go/bin/linx-server /usr/local/bin/linx-server + +ENV GOPATH /go +COPY static /go/src/github.com/andreimarcu/linx-server/static/ +COPY templates /go/src/github.com/andreimarcu/linx-server/templates/ + RUN mkdir -p /data/files && mkdir -p /data/meta && chown -R 65534:65534 /data VOLUME ["/data/files", "/data/meta"] EXPOSE 8080 USER nobody -ENTRYPOINT ["/go/bin/linx-server", "-bind=0.0.0.0:8080", "-filespath=/data/files/", "-metapath=/data/meta/"] +ENTRYPOINT ["/usr/local/bin/linx-server", "-bind=0.0.0.0:8080", "-filespath=/data/files/", "-metapath=/data/meta/"] CMD ["-sitename=linx", "-allowhotlink"] From fd0f3d9e46af8729e6c00d88d0726b7ed8159c79 Mon Sep 17 00:00:00 2001 From: Simon Alfassa Date: Sat, 9 Jun 2018 19:31:44 +0200 Subject: [PATCH 13/30] Make the web page mobile friendly --- static/css/dropzone.css | 9 +++++- static/css/github-markdown.css | 3 +- static/css/linx.css | 58 +++++++++++++++++++++------------- static/js/bin.js | 2 +- templates/404.html | 4 ++- templates/base.html | 1 + templates/display/base.html | 5 ++- templates/index.html | 4 +-- templates/paste.html | 10 +++--- 9 files changed, 60 insertions(+), 36 deletions(-) diff --git a/static/css/dropzone.css b/static/css/dropzone.css index 18472a6..f3c4cb2 100644 --- a/static/css/dropzone.css +++ b/static/css/dropzone.css @@ -31,11 +31,18 @@ border: 2px solid #FAFBFC; } -#dropzone { width: 400px; +#dropzone { + width: 400px; margin-left: auto; margin-right: auto; } +@media(max-width: 450px) { + #dropzone { + width: auto; + } +} + #uploads { margin-top: 20px; } diff --git a/static/css/github-markdown.css b/static/css/github-markdown.css index 8072b54..6823d9c 100644 --- a/static/css/github-markdown.css +++ b/static/css/github-markdown.css @@ -8,7 +8,8 @@ font-size: 12px; line-height: 1.6; word-wrap: break-word; - width: 680px; + width: 80vw; + max-width: 680px; padding: 10px; } diff --git a/static/css/linx.css b/static/css/linx.css index 193cbec..a759dc6 100644 --- a/static/css/linx.css +++ b/static/css/linx.css @@ -71,32 +71,26 @@ body { -webkit-box-shadow: 1px 1px 1px 1px #ccc; box-shadow: 1px 1px 1px 1px #ccc; margin-bottom: 15px; +} +.dinfo #filename { + margin: 2px 15px 0 0; } #info { - text-align: left; - + display: flex; + flex-wrap: wrap; + justify-content: space-between; background-color: white; padding: 5px 5px 5px 5px; } -#info #filename, -#editform #filename { - width: 232px; -} - #info #extension, #editform #extension { width: 40px; } -#info .float-left { - margin-top: 2px; - margin-right: 20px; -} - -#info .right { +#info .text-right { font-size: 13px; } @@ -115,6 +109,11 @@ body { color: #556A7F; } +#info input[type=checkbox] { + margin: 0; + vertical-align: bottom; +} + #footer { color: gray; text-align: right; @@ -158,7 +157,8 @@ body { } .fixed { - width: 800px; + width: 80vw; + max-width: 800px; } .needs-border { @@ -245,19 +245,28 @@ body { } #choices { - float: left; - width: 100%; - text-align: left; - vertical-align: bottom; - margin-top: 5px; + display: flex; + align-items: center; + flex-wrap: wrap; + justify-content: space-between; + width: 100%; + margin-top: 5px; font-size:13px; } +#choices label:first-child { + margin-right: 15px; +} + #expiry { - float: right; padding-top: 1px; } +#randomize { + vertical-align: bottom; + margin: 0; +} + .oopscontent { width: 400px; } @@ -267,8 +276,13 @@ body { border: 0; } +.error-404 img { + max-width: 90vw; +} + .editor { - width: 705px; + width: 90vw; + max-width: 705px; height: 450px; border-color: #cccccc; font-family: monospace; @@ -287,7 +301,7 @@ body { /* Content display {{{ */ .display-audio, .display-file { - width: 500px; + width: 100%; } .display-image { diff --git a/static/js/bin.js b/static/js/bin.js index 6e1bbcc..bcaf4f1 100644 --- a/static/js/bin.js +++ b/static/js/bin.js @@ -1,6 +1,6 @@ // @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later -var navlist = document.getElementById("info").getElementsByClassName("right")[0]; +var navlist = document.getElementById("info").getElementsByClassName("text-right")[0]; init(); diff --git a/templates/404.html b/templates/404.html index c1728e5..3b3d64e 100644 --- a/templates/404.html +++ b/templates/404.html @@ -1,5 +1,7 @@ {% extends "base.html" %} {% block content %} - +

+ +
{% endblock %} diff --git a/templates/base.html b/templates/base.html index d750cb4..5392f8d 100644 --- a/templates/base.html +++ b/templates/base.html @@ -3,6 +3,7 @@ {% block title %}{{ sitename }}{% endblock %} + diff --git a/templates/display/base.html b/templates/display/base.html index 172e17a..29d21b8 100644 --- a/templates/display/base.html +++ b/templates/display/base.html @@ -7,11 +7,11 @@ {% block content %}
-
+
{{ filename }}
-
+
{% if expiry %} file expires in {{ expiry }} | {% endif %} @@ -22,7 +22,6 @@
{% block infoleft %}{% endblock %} -
diff --git a/templates/index.html b/templates/index.html index 87d821d..d423879 100644 --- a/templates/index.html +++ b/templates/index.html @@ -17,8 +17,9 @@
+
-
-
diff --git a/templates/paste.html b/templates/paste.html index 9178ba4..c4e88e9 100644 --- a/templates/paste.html +++ b/templates/paste.html @@ -4,17 +4,17 @@
- . - -
+
+ . +
+
+ - -
From b731e17c1ef56373f93eb760c45c0c1cf647cf8b Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Wed, 16 Jan 2019 01:15:24 -0800 Subject: [PATCH 14/30] Cosmetic tweaks & fixes --- display.go | 1 + static/css/dropzone.css | 1 + static/css/linx.css | 176 +++++++++++++++++++---------------- static/js/bin.js | 4 +- templates/display/base.html | 2 +- templates/display/bin.html | 20 ++-- templates/display/story.html | 20 ++-- templates/paste.html | 10 +- 8 files changed, 120 insertions(+), 114 deletions(-) diff --git a/display.go b/display.go index ca5bc6f..63ba765 100644 --- a/display.go +++ b/display.go @@ -121,6 +121,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { "filename": fileName, "size": sizeHuman, "expiry": expiryHuman, + "expirylist": listExpirationTimes(), "extra": extra, "lines": lines, "files": metadata.ArchiveFiles, diff --git a/static/css/dropzone.css b/static/css/dropzone.css index f3c4cb2..e19794f 100644 --- a/static/css/dropzone.css +++ b/static/css/dropzone.css @@ -49,6 +49,7 @@ div.dz-default { border: 2px dashed #C9C9C9; + border-radius: 5px; color: #C9C9C9; font: 14px "helvetica neue",helvetica,arial,sans-serif; background-color: #FAFBFC; diff --git a/static/css/linx.css b/static/css/linx.css index a759dc6..82e256a 100644 --- a/static/css/linx.css +++ b/static/css/linx.css @@ -1,56 +1,56 @@ body { - background-color: #E8ECF0; - color: #556A7F; + background-color: #E8ECF0; + color: #556A7F; - font-family: Arial, Helvetica, sans-serif; - font-size: 14px; + font-family: Arial, Helvetica, sans-serif; + font-size: 14px; } #container_container { - display: table; - table-layout: fixed; - margin-left: auto; - margin-right: auto; + display: table; + table-layout: fixed; + margin-left: auto; + margin-right: auto; } #container { - display: table-cell; - min-width: 200px; + display: table-cell; + min-width: 200px; } #header a { - text-decoration: none; - color: #556A7F; + text-decoration: none; + color: #556A7F; } #navigation { - margin-top: 4px; + margin-top: 4px; } #navigation a { - text-decoration: none; - border-bottom: 1px dotted #556A7F; - color: #556A7F; + text-decoration: none; + border-bottom: 1px dotted #556A7F; + color: #556A7F; } #navigation a:hover { - background-color: #C7D1EB; + background-color: #C7D1EB; } #main { - background-color: white; + background-color: white; - padding: 6px 5px 8px 5px; + padding: 6px 5px 8px 5px; - -moz-box-shadow: 1px 1px 1px 1px #ccc; - -webkit-box-shadow: 1px 1px 1px 1px #ccc; - box-shadow: 1px 1px 1px 1px #ccc; + -moz-box-shadow: 1px 1px 1px 1px #ccc; + -webkit-box-shadow: 1px 1px 1px 1px #ccc; + box-shadow: 1px 1px 1px 1px #ccc; - text-align: center; + text-align: center; } #main a { - color: #556A7F; + color: #556A7F; } #normal-content { @@ -62,10 +62,6 @@ body { margin-bottom: 0; } -.ninfo { - margin-bottom: 5px; -} - .dinfo { -moz-box-shadow: 1px 1px 1px 1px #ccc; -webkit-box-shadow: 1px 1px 1px 1px #ccc; @@ -73,16 +69,12 @@ body { margin-bottom: 15px; } -.dinfo #filename { - margin: 2px 15px 0 0; -} - #info { - display: flex; - flex-wrap: wrap; - justify-content: space-between; + display: flex; + flex-wrap: wrap; + justify-content: space-between; background-color: white; - padding: 5px 5px 5px 5px; + padding: 5px 5px 5px 5px; } #info #extension, @@ -91,7 +83,7 @@ body { } #info .text-right { - font-size: 13px; + font-size: 13px; } #info a { @@ -104,56 +96,50 @@ body { background-color: #E8ECF0; } -#info input[type=text] { - border: 0; - color: #556A7F; -} - #info input[type=checkbox] { - margin: 0; - vertical-align: bottom; + margin: 0; + vertical-align: bottom; } #footer { - color: gray; - text-align: right; - margin-top: 30px; - margin-bottom: 10px; - font-size: 11px; + color: gray; + text-align: right; + margin-top: 30px; + margin-bottom: 10px; + font-size: 11px; } #footer a { - color: gray; - text-decoration: none; + color: gray; + text-decoration: none; } - .normal { - text-align: left; - font-size: 13px; + text-align: left; + font-size: 13px; } .normal a { - text-decoration: none; - border-bottom: 1px dotted gray; + text-decoration: none; + border-bottom: 1px dotted gray; } .normal a:hover { - color: black; - background-color: #E8ECF0; + color: black; + background-color: #E8ECF0; } .normal ul { - padding-left: 15px; + padding-left: 15px; } .normal li { - margin-bottom: 3px; - list-style: none; + margin-bottom: 3px; + list-style: none; } .normal li a { - font-weight: bold; + font-weight: bold; } .fixed { @@ -161,37 +147,46 @@ body { max-width: 800px; } +.paste { + width: 70vw; + max-width: 700px; +} + .needs-border { - border-top: 1px solid rgb(214, 214, 214); + border-top: 1px solid rgb(214, 214, 214); } .left { - text-align: left; + text-align: left; } .float-left { - float: left; + float: left; +} + +.pad-left { + padding-left: 10px; } .pad-right { - padding-right: 10px; + padding-right: 10px; } .text-right { - text-align: right; + text-align: right; } .center { - text-align: center; + text-align: center; } .float-right, .right { - float: right; + float: right; } .clear { - clear: both; + clear: both; } #upload_header { @@ -255,7 +250,7 @@ body { } #choices label:first-child { - margin-right: 15px; + margin-right: 15px; } #expiry { @@ -280,14 +275,34 @@ body { max-width: 90vw; } +.padme { + padding-left: 5px; + padding-right: 5px; +} + .editor { - width: 90vw; - max-width: 705px; - height: 450px; - border-color: #cccccc; - font-family: monospace; - resize: none; - overflow: auto; + width: 100%; + height: 450px; + border: 1px solid #eaeaea; + font-family: monospace; + resize: none; + overflow: auto; + border-radius: 2px; + padding: 2px; + box-sizing: border-box; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + } + + + +#info input[type=text] { + border: 1px solid #eaeaea; + color: #556A7F; + border-radius: 4px 4px 4px 4px; + padding-left: 4px; + padding-right: 4px; + height: 15px; } .storygreen { @@ -329,15 +344,16 @@ body { #editform, #editform .editor { display: none; + width: 100% } #codeb { white-space: pre-wrap; } -#editor { +#inplace-editor { display: none; - width: 794px; + width: 100%; height: 800px; font-size: 13px; } diff --git a/static/js/bin.js b/static/js/bin.js index bcaf4f1..7aed334 100644 --- a/static/js/bin.js +++ b/static/js/bin.js @@ -32,13 +32,13 @@ function edit(ev) { var normalcontent = document.getElementById("normal-content"); normalcontent.removeChild(document.getElementById("normal-code")); - var editordiv = document.getElementById("editor"); + var editordiv = document.getElementById("inplace-editor"); editordiv.style.display = "block"; editordiv.addEventListener('keydown', handleTab); } function paste(ev) { - var editordiv = document.getElementById("editor"); + var editordiv = document.getElementById("inplace-editor"); document.getElementById("newcontent").value = editordiv.value; document.forms["reply"].submit(); } diff --git a/templates/display/base.html b/templates/display/base.html index 29d21b8..011534c 100644 --- a/templates/display/base.html +++ b/templates/display/base.html @@ -11,7 +11,7 @@ {{ filename }}
-
+
{% if expiry %} file expires in {{ expiry }} | {% endif %} diff --git a/templates/display/bin.html b/templates/display/bin.html index 12e49c9..bd029a2 100644 --- a/templates/display/bin.html +++ b/templates/display/bin.html @@ -12,23 +12,17 @@ {% block infoleft %}
-
+
+ - -
- . + .
@@ -41,7 +35,7 @@ {% block main %} 
{{ extra.contents }}
- +
diff --git a/templates/display/story.html b/templates/display/story.html index 763fd4a..20e772c 100644 --- a/templates/display/story.html +++ b/templates/display/story.html @@ -10,23 +10,17 @@ {% block infoleft %}
-
+
+ - -
- . + .
@@ -39,7 +33,7 @@ {% block main %} 
{% for line in lines %}{% if line|make_list|first == ">" %}{{ line }}{% else %}{{ line }}{% endif %}{% endfor %}
- +
diff --git a/templates/paste.html b/templates/paste.html index c4e88e9..7737760 100644 --- a/templates/paste.html +++ b/templates/paste.html @@ -2,10 +2,10 @@ {% block content %}
-
-
+
+
- . + .
@@ -18,8 +18,8 @@
-
- +
+
From 0fb5fa1c517a12ca60d548ccadc45301d5bf6679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Fri, 25 Jan 2019 08:21:49 +0100 Subject: [PATCH 15/30] use sha256-simd (#155) --- meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.go b/meta.go index 10cd4e1..2fc7b81 100644 --- a/meta.go +++ b/meta.go @@ -5,7 +5,6 @@ import ( "archive/zip" "compress/bzip2" "compress/gzip" - "crypto/sha256" "encoding/hex" "errors" "io" @@ -16,6 +15,7 @@ import ( "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" "github.com/dchest/uniuri" + "github.com/minio/sha256-simd" "gopkg.in/h2non/filetype.v1" ) From 5d9a93b1e285fe36b827ec74a2186f8623408b25 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Fri, 25 Jan 2019 07:33:11 +0000 Subject: [PATCH 16/30] Add S3 backend (#156) --- Dockerfile | 2 + backends/localfs/localfs.go | 143 ++++++++++++++++++++----- backends/meta.go | 5 - backends/metajson/metajson.go | 70 ------------- backends/s3/s3.go | 192 ++++++++++++++++++++++++++++++++++ backends/storage.go | 20 ++-- delete.go | 19 ++-- display.go | 46 +++++--- expiry.go | 2 +- fileserve.go | 37 ++++--- helpers/archive.go | 70 +++++++++++++ helpers/helpers.go | 67 ++++++++++++ linx-cleanup/cleanup.go | 10 +- meta.go | 165 ----------------------------- pages.go | 35 +++++-- server.go | 22 ++-- server_test.go | 80 ++++++++++++-- torrent.go | 64 ++++-------- torrent/torrent.go | 28 +++++ torrent_test.go | 5 +- upload.go | 95 +++++++++-------- 21 files changed, 737 insertions(+), 440 deletions(-) delete mode 100644 backends/metajson/metajson.go create mode 100644 backends/s3/s3.go create mode 100644 helpers/archive.go create mode 100644 helpers/helpers.go delete mode 100644 meta.go create mode 100644 torrent/torrent.go diff --git a/Dockerfile b/Dockerfile index addad3f..c1a2f2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,8 @@ FROM alpine:3.8 COPY --from=build /go/bin/linx-server /usr/local/bin/linx-server ENV GOPATH /go +ENV SSL_CERT_FILE /etc/ssl/cert.pem + COPY static /go/src/github.com/andreimarcu/linx-server/static/ COPY templates /go/src/github.com/andreimarcu/linx-server/templates/ diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index b55c986..3f6f5ad 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -1,63 +1,149 @@ package localfs import ( - "errors" + "encoding/json" "io" "io/ioutil" - "net/http" "os" "path" + "time" "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/helpers" ) type LocalfsBackend struct { - basePath string + metaPath string + filesPath string } -func (b LocalfsBackend) Delete(key string) error { - return os.Remove(path.Join(b.basePath, key)) +type MetadataJSON struct { + DeleteKey string `json:"delete_key"` + Sha256sum string `json:"sha256sum"` + Mimetype string `json:"mimetype"` + Size int64 `json:"size"` + Expiry int64 `json:"expiry"` + ArchiveFiles []string `json:"archive_files,omitempty"` +} + +func (b LocalfsBackend) Delete(key string) (err error) { + err = os.Remove(path.Join(b.filesPath, key)) + if err != nil { + return + } + err = os.Remove(path.Join(b.metaPath, key)) + return } func (b LocalfsBackend) Exists(key string) (bool, error) { - _, err := os.Stat(path.Join(b.basePath, key)) + _, err := os.Stat(path.Join(b.filesPath, key)) return err == nil, err } -func (b LocalfsBackend) Get(key string) ([]byte, error) { - return ioutil.ReadFile(path.Join(b.basePath, key)) +func (b LocalfsBackend) Head(key string) (metadata backends.Metadata, err error) { + f, err := os.Open(path.Join(b.metaPath, key)) + if os.IsNotExist(err) { + return metadata, backends.NotFoundErr + } else if err != nil { + return metadata, backends.BadMetadata + } + defer f.Close() + + decoder := json.NewDecoder(f) + + mjson := MetadataJSON{} + if err := decoder.Decode(&mjson); err != nil { + return metadata, backends.BadMetadata + } + + metadata.DeleteKey = mjson.DeleteKey + metadata.Mimetype = mjson.Mimetype + metadata.ArchiveFiles = mjson.ArchiveFiles + metadata.Sha256sum = mjson.Sha256sum + metadata.Expiry = time.Unix(mjson.Expiry, 0) + metadata.Size = mjson.Size + + return } -func (b LocalfsBackend) Put(key string, r io.Reader) (int64, error) { - dst, err := os.Create(path.Join(b.basePath, key)) +func (b LocalfsBackend) Get(key string) (metadata backends.Metadata, f io.ReadCloser, err error) { + metadata, err = b.Head(key) if err != nil { - return 0, err + return + } + + f, err = os.Open(path.Join(b.filesPath, key)) + if err != nil { + return + } + + return +} + +func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) error { + metaPath := path.Join(b.metaPath, key) + + mjson := MetadataJSON{ + DeleteKey: metadata.DeleteKey, + Mimetype: metadata.Mimetype, + ArchiveFiles: metadata.ArchiveFiles, + Sha256sum: metadata.Sha256sum, + Expiry: metadata.Expiry.Unix(), + Size: metadata.Size, + } + + dst, err := os.Create(metaPath) + if err != nil { + return err + } + defer dst.Close() + + encoder := json.NewEncoder(dst) + err = encoder.Encode(mjson) + if err != nil { + os.Remove(metaPath) + return err + } + + return nil +} + +func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) { + filePath := path.Join(b.filesPath, key) + + dst, err := os.Create(filePath) + if err != nil { + return } defer dst.Close() bytes, err := io.Copy(dst, r) if bytes == 0 { - b.Delete(key) - return bytes, errors.New("Empty file") + os.Remove(filePath) + return m, backends.FileEmptyError } else if err != nil { - b.Delete(key) - return bytes, err + os.Remove(filePath) + return m, err } - return bytes, err -} + m.Expiry = expiry + m.DeleteKey = deleteKey + m.Size = bytes + m.Mimetype, _ = helpers.DetectMime(dst) + m.Sha256sum, _ = helpers.Sha256sum(dst) + m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, dst) -func (b LocalfsBackend) Open(key string) (backends.ReadSeekCloser, error) { - return os.Open(path.Join(b.basePath, key)) -} + err = b.writeMetadata(key, m) + if err != nil { + os.Remove(filePath) + return + } -func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) { - filePath := path.Join(b.basePath, key) - http.ServeFile(w, r, filePath) + return } func (b LocalfsBackend) Size(key string) (int64, error) { - fileInfo, err := os.Stat(path.Join(b.basePath, key)) + fileInfo, err := os.Stat(path.Join(b.filesPath, key)) if err != nil { return 0, err } @@ -68,7 +154,7 @@ func (b LocalfsBackend) Size(key string) (int64, error) { func (b LocalfsBackend) List() ([]string, error) { var output []string - files, err := ioutil.ReadDir(b.basePath) + files, err := ioutil.ReadDir(b.filesPath) if err != nil { return nil, err } @@ -80,6 +166,9 @@ func (b LocalfsBackend) List() ([]string, error) { return output, nil } -func NewLocalfsBackend(basePath string) LocalfsBackend { - return LocalfsBackend{basePath: basePath} +func NewLocalfsBackend(metaPath string, filesPath string) LocalfsBackend { + return LocalfsBackend{ + metaPath: metaPath, + filesPath: filesPath, + } } diff --git a/backends/meta.go b/backends/meta.go index 27c3e41..7ba522d 100644 --- a/backends/meta.go +++ b/backends/meta.go @@ -5,11 +5,6 @@ import ( "time" ) -type MetaBackend interface { - Get(key string) (Metadata, error) - Put(key string, metadata *Metadata) error -} - type Metadata struct { DeleteKey string Sha256sum string diff --git a/backends/metajson/metajson.go b/backends/metajson/metajson.go deleted file mode 100644 index 8ec53c4..0000000 --- a/backends/metajson/metajson.go +++ /dev/null @@ -1,70 +0,0 @@ -package metajson - -import ( - "bytes" - "encoding/json" - "time" - - "github.com/andreimarcu/linx-server/backends" -) - -type MetadataJSON struct { - DeleteKey string `json:"delete_key"` - Sha256sum string `json:"sha256sum"` - Mimetype string `json:"mimetype"` - Size int64 `json:"size"` - Expiry int64 `json:"expiry"` - ArchiveFiles []string `json:"archive_files,omitempty"` -} - -type MetaJSONBackend struct { - storage backends.MetaStorageBackend -} - -func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error { - mjson := MetadataJSON{} - mjson.DeleteKey = metadata.DeleteKey - mjson.Mimetype = metadata.Mimetype - mjson.ArchiveFiles = metadata.ArchiveFiles - mjson.Sha256sum = metadata.Sha256sum - mjson.Expiry = metadata.Expiry.Unix() - mjson.Size = metadata.Size - - byt, err := json.Marshal(mjson) - if err != nil { - return err - } - - if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil { - return err - } - - return nil -} - -func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) { - b, err := m.storage.Get(key) - if err != nil { - return metadata, backends.BadMetadata - } - - mjson := MetadataJSON{} - - err = json.Unmarshal(b, &mjson) - if err != nil { - return metadata, backends.BadMetadata - } - - metadata.DeleteKey = mjson.DeleteKey - metadata.Mimetype = mjson.Mimetype - metadata.ArchiveFiles = mjson.ArchiveFiles - metadata.Sha256sum = mjson.Sha256sum - metadata.Expiry = time.Unix(mjson.Expiry, 0) - metadata.Size = mjson.Size - - return -} - -func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend { - return MetaJSONBackend{storage: storage} -} diff --git a/backends/s3/s3.go b/backends/s3/s3.go new file mode 100644 index 0000000..7ae326c --- /dev/null +++ b/backends/s3/s3.go @@ -0,0 +1,192 @@ +package s3 + +import ( + "io" + "io/ioutil" + "os" + "strconv" + "time" + + "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/helpers" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +type S3Backend struct { + bucket string + svc *s3.S3 +} + +func (b S3Backend) Delete(key string) error { + _, err := b.svc.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + }) + if err != nil { + return err + } + return nil +} + +func (b S3Backend) Exists(key string) (bool, error) { + _, err := b.svc.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + }) + return err == nil, err +} + +func (b S3Backend) Head(key string) (metadata backends.Metadata, err error) { + var result *s3.HeadObjectOutput + result, err = b.svc.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + if aerr.Code() == s3.ErrCodeNoSuchKey || aerr.Code() == "NotFound" { + err = backends.NotFoundErr + } + } + return + } + + metadata, err = unmapMetadata(result.Metadata) + return +} + +func (b S3Backend) Get(key string) (metadata backends.Metadata, r io.ReadCloser, err error) { + var result *s3.GetObjectOutput + result, err = b.svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + if aerr.Code() == s3.ErrCodeNoSuchKey || aerr.Code() == "NotFound" { + err = backends.NotFoundErr + } + } + return + } + + metadata, err = unmapMetadata(result.Metadata) + r = result.Body + return +} + +func mapMetadata(m backends.Metadata) map[string]*string { + return map[string]*string{ + "Expiry": aws.String(strconv.FormatInt(m.Expiry.Unix(), 10)), + "Delete_key": aws.String(m.DeleteKey), + "Size": aws.String(strconv.FormatInt(m.Size, 10)), + "Mimetype": aws.String(m.Mimetype), + "Sha256sum": aws.String(m.Sha256sum), + } +} + +func unmapMetadata(input map[string]*string) (m backends.Metadata, err error) { + expiry, err := strconv.ParseInt(aws.StringValue(input["Expiry"]), 10, 64) + if err != nil { + return m, err + } + m.Expiry = time.Unix(expiry, 0) + + m.Size, err = strconv.ParseInt(aws.StringValue(input["Size"]), 10, 64) + if err != nil { + return + } + + m.DeleteKey = aws.StringValue(input["Delete_key"]) + m.Mimetype = aws.StringValue(input["Mimetype"]) + m.Sha256sum = aws.StringValue(input["Sha256sum"]) + return +} + +func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (m backends.Metadata, err error) { + tmpDst, err := ioutil.TempFile("", "linx-server-upload") + if err != nil { + return m, err + } + defer tmpDst.Close() + defer os.Remove(tmpDst.Name()) + + bytes, err := io.Copy(tmpDst, r) + if bytes == 0 { + return m, backends.FileEmptyError + } else if err != nil { + return m, err + } + + m.Expiry = expiry + m.DeleteKey = deleteKey + m.Size = bytes + m.Mimetype, _ = helpers.DetectMime(tmpDst) + m.Sha256sum, _ = helpers.Sha256sum(tmpDst) + // XXX: we may not be able to write this to AWS easily + //m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, tmpDst) + + uploader := s3manager.NewUploaderWithClient(b.svc) + input := &s3manager.UploadInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + Body: tmpDst, + Metadata: mapMetadata(m), + } + _, err = uploader.Upload(input) + if err != nil { + return + } + + return +} + +func (b S3Backend) Size(key string) (int64, error) { + input := &s3.HeadObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + } + result, err := b.svc.HeadObject(input) + if err != nil { + return 0, err + } + + return *result.ContentLength, nil +} + +func (b S3Backend) List() ([]string, error) { + var output []string + input := &s3.ListObjectsInput{ + Bucket: aws.String(b.bucket), + } + + results, err := b.svc.ListObjects(input) + if err != nil { + return nil, err + } + + + for _, object := range results.Contents { + output = append(output, *object.Key) + } + + return output, nil +} + +func NewS3Backend(bucket string, region string, endpoint string) S3Backend { + awsConfig := &aws.Config{} + if region != "" { + awsConfig.Region = aws.String(region) + } + if endpoint != "" { + awsConfig.Endpoint = aws.String(endpoint) + } + + sess := session.Must(session.NewSession(awsConfig)) + svc := s3.New(sess) + return S3Backend{bucket: bucket, svc: svc} +} diff --git a/backends/storage.go b/backends/storage.go index 2b51a2c..d40a2b9 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -1,24 +1,17 @@ package backends import ( + "errors" "io" - "net/http" + "time" ) -type ReadSeekCloser interface { - io.Reader - io.Closer - io.Seeker - io.ReaderAt -} - type StorageBackend interface { Delete(key string) error Exists(key string) (bool, error) - Get(key string) ([]byte, error) - Put(key string, r io.Reader) (int64, error) - Open(key string) (ReadSeekCloser, error) - ServeFile(key string, w http.ResponseWriter, r *http.Request) + Head(key string) (Metadata, error) + Get(key string) (Metadata, io.ReadCloser, error) + Put(key string, r io.Reader, expiry time.Time, deleteKey string) (Metadata, error) Size(key string) (int64, error) } @@ -26,3 +19,6 @@ type MetaStorageBackend interface { StorageBackend List() ([]string, error) } + +var NotFoundErr = errors.New("File not found.") +var FileEmptyError = errors.New("Empty file") diff --git a/delete.go b/delete.go index 61c6fa8..38e36e3 100644 --- a/delete.go +++ b/delete.go @@ -3,8 +3,8 @@ package main import ( "fmt" "net/http" - "os" + "github.com/andreimarcu/linx-server/backends" "github.com/zenazn/goji/web" ) @@ -13,24 +13,19 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) { filename := c.URLParams["name"] - // Ensure requested file actually exists - if _, readErr := fileBackend.Exists(filename); os.IsNotExist(readErr) { + // Ensure that file exists and delete key is correct + metadata, err := storageBackend.Head(filename) + if err == backends.NotFoundErr { notFoundHandler(c, w, r) // 404 - file doesn't exist return - } - - // Ensure delete key is correct - metadata, err := metadataRead(filename) - if err != nil { + } else if err != nil { unauthorizedHandler(c, w, r) // 401 - no metadata available return } if metadata.DeleteKey == requestKey { - fileDelErr := fileBackend.Delete(filename) - metaDelErr := metaStorageBackend.Delete(filename) - - if (fileDelErr != nil) || (metaDelErr != nil) { + err := storageBackend.Delete(filename) + if err != nil { oopsHandler(c, w, r, RespPLAIN, "Could not delete") return } diff --git a/display.go b/display.go index 63ba765..7258904 100644 --- a/display.go +++ b/display.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "io/ioutil" "net/http" "path/filepath" "regexp" @@ -9,6 +10,7 @@ import ( "strings" "time" + "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" "github.com/dustin/go-humanize" "github.com/flosch/pongo2" @@ -29,14 +31,11 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - _, err := checkFile(fileName) - if err == NotFoundErr { + metadata, err := checkFile(fileName) + if err == backends.NotFoundErr { notFoundHandler(c, w, r) return - } - - metadata, err := metadataRead(fileName) - if err != nil { + } else if err != nil { oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") return } @@ -78,8 +77,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { tpl = Templates["display/pdf.html"] } else if extension == "story" { + metadata, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespHTML, err.Error()) + } + if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := fileBackend.Get(fileName) + bytes, err := ioutil.ReadAll(reader) if err == nil { extra["contents"] = string(bytes) lines = strings.Split(extra["contents"], "\n") @@ -88,8 +92,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } } else if extension == "md" { + metadata, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespHTML, err.Error()) + } + if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := fileBackend.Get(fileName) + bytes, err := ioutil.ReadAll(reader) if err == nil { unsafe := blackfriday.MarkdownCommon(bytes) html := bluemonday.UGCPolicy().SanitizeBytes(unsafe) @@ -100,8 +109,13 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } } else if strings.HasPrefix(metadata.Mimetype, "text/") || supportedBinExtension(extension) { + metadata, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespHTML, err.Error()) + } + if metadata.Size < maxDisplayFileSizeBytes { - bytes, err := fileBackend.Get(fileName) + bytes, err := ioutil.ReadAll(reader) if err == nil { extra["extension"] = extension extra["lang_hl"], extra["lang_ace"] = extensionToHlAndAceLangs(extension) @@ -117,14 +131,14 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } err = renderTemplate(tpl, pongo2.Context{ - "mime": metadata.Mimetype, - "filename": fileName, - "size": sizeHuman, - "expiry": expiryHuman, + "mime": metadata.Mimetype, + "filename": fileName, + "size": sizeHuman, + "expiry": expiryHuman, "expirylist": listExpirationTimes(), - "extra": extra, - "lines": lines, - "files": metadata.ArchiveFiles, + "extra": extra, + "lines": lines, + "files": metadata.ArchiveFiles, }, r, w) if err != nil { diff --git a/expiry.go b/expiry.go index 6d8887d..63b7757 100644 --- a/expiry.go +++ b/expiry.go @@ -24,7 +24,7 @@ type ExpirationTime struct { // Determine if the given filename is expired func isFileExpired(filename string) (bool, error) { - metadata, err := metadataRead(filename) + metadata, err := storageBackend.Head(filename) if err != nil { return false, err } diff --git a/fileserve.go b/fileserve.go index 3b20c4c..a3a249e 100644 --- a/fileserve.go +++ b/fileserve.go @@ -1,8 +1,10 @@ package main import ( + "io" "net/http" "net/url" + "strconv" "strings" "github.com/andreimarcu/linx-server/backends" @@ -14,14 +16,11 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] metadata, err := checkFile(fileName) - if err == NotFoundErr { + if err == backends.NotFoundErr { notFoundHandler(c, w, r) return - } else if err == backends.BadMetadata { - oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") - return } else if err != nil { - oopsHandler(c, w, r, RespAUTO, err.Error()) + oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") return } @@ -38,10 +37,23 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy) w.Header().Set("Referrer-Policy", Config.fileReferrerPolicy) + _, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespAUTO, err.Error()) + } + + w.Header().Set("Content-Type", metadata.Mimetype) + w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10)) w.Header().Set("Etag", metadata.Sha256sum) w.Header().Set("Cache-Control", "max-age=0") - fileBackend.ServeFile(fileName, w, r) + if r.Method != "HEAD" { + defer reader.Close() + + if _, err = io.CopyN(w, reader, metadata.Size); err != nil { + oopsHandler(c, w, r, RespAUTO, err.Error()) + } + } } func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) { @@ -69,21 +81,14 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) { } func checkFile(filename string) (metadata backends.Metadata, err error) { - _, err = fileBackend.Exists(filename) - if err != nil { - err = NotFoundErr - return - } - - metadata, err = metadataRead(filename) + metadata, err = storageBackend.Head(filename) if err != nil { return } if expiry.IsTsExpired(metadata.Expiry) { - fileBackend.Delete(filename) - metaStorageBackend.Delete(filename) - err = NotFoundErr + storageBackend.Delete(filename) + err = backends.NotFoundErr return } diff --git a/helpers/archive.go b/helpers/archive.go new file mode 100644 index 0000000..2a4380b --- /dev/null +++ b/helpers/archive.go @@ -0,0 +1,70 @@ +package helpers + +import ( + "archive/tar" + "archive/zip" + "compress/bzip2" + "compress/gzip" + "io" + "sort" +) + +type ReadSeekerAt interface { + io.Reader + io.Seeker + io.ReaderAt +} + +func ListArchiveFiles(mimetype string, size int64, r ReadSeekerAt) (files []string, err error) { + if mimetype == "application/x-tar" { + tReadr := tar.NewReader(r) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + files = append(files, hdr.Name) + } + } + sort.Strings(files) + } else if mimetype == "application/x-gzip" { + gzf, err := gzip.NewReader(r) + if err == nil { + tReadr := tar.NewReader(gzf) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + files = append(files, hdr.Name) + } + } + sort.Strings(files) + } + } else if mimetype == "application/x-bzip" { + bzf := bzip2.NewReader(r) + tReadr := tar.NewReader(bzf) + for { + hdr, err := tReadr.Next() + if err == io.EOF || err != nil { + break + } + if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { + files = append(files, hdr.Name) + } + } + sort.Strings(files) + } else if mimetype == "application/zip" { + zf, err := zip.NewReader(r, size) + if err == nil { + for _, f := range zf.File { + files = append(files, f.Name) + } + } + sort.Strings(files) + } + + return +} diff --git a/helpers/helpers.go b/helpers/helpers.go new file mode 100644 index 0000000..aef68ff --- /dev/null +++ b/helpers/helpers.go @@ -0,0 +1,67 @@ +package helpers + +import ( + "encoding/hex" + "io" + "unicode" + + "github.com/minio/sha256-simd" + "gopkg.in/h2non/filetype.v1" +) + +func DetectMime(r io.ReadSeeker) (string, error) { + // Get first 512 bytes for mimetype detection + header := make([]byte, 512) + + r.Seek(0, 0) + r.Read(header) + r.Seek(0, 0) + + kind, err := filetype.Match(header) + if err != nil { + return "application/octet-stream", err + } else if kind.MIME.Value != "" { + return kind.MIME.Value, nil + } + + // Check if the file seems anything like text + if printable(header) { + return "text/plain", nil + } else { + return "application/octet-stream", nil + } +} + +func Sha256sum(r io.ReadSeeker) (string, error) { + hasher := sha256.New() + + r.Seek(0, 0) + _, err := io.Copy(hasher, r) + if err != nil { + return "", err + } + + r.Seek(0, 0) + + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func printable(data []byte) bool { + for i, b := range data { + r := rune(b) + + // A null terminator that's not at the beginning of the file + if r == 0 && i == 0 { + return false + } else if r == 0 && i < 0 { + continue + } + + if r > unicode.MaxASCII { + return false + } + + } + + return true +} diff --git a/linx-cleanup/cleanup.go b/linx-cleanup/cleanup.go index 9ea89ae..88c2bce 100644 --- a/linx-cleanup/cleanup.go +++ b/linx-cleanup/cleanup.go @@ -5,7 +5,6 @@ import ( "log" "github.com/andreimarcu/linx-server/backends/localfs" - "github.com/andreimarcu/linx-server/backends/metajson" "github.com/andreimarcu/linx-server/expiry" ) @@ -22,17 +21,15 @@ func main() { "don't log deleted files") flag.Parse() - metaStorageBackend := localfs.NewLocalfsBackend(metaDir) - metaBackend := metajson.NewMetaJSONBackend(metaStorageBackend) - fileBackend := localfs.NewLocalfsBackend(filesDir) + fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir) - files, err := metaStorageBackend.List() + files, err := fileBackend.List() if err != nil { panic(err) } for _, filename := range files { - metadata, err := metaBackend.Get(filename) + metadata, err := fileBackend.Head(filename) if err != nil { if !noLogs { log.Printf("Failed to find metadata for %s", filename) @@ -44,7 +41,6 @@ func main() { log.Printf("Delete %s", filename) } fileBackend.Delete(filename) - metaStorageBackend.Delete(filename) } } } diff --git a/meta.go b/meta.go deleted file mode 100644 index 2fc7b81..0000000 --- a/meta.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -import ( - "archive/tar" - "archive/zip" - "compress/bzip2" - "compress/gzip" - "encoding/hex" - "errors" - "io" - "sort" - "time" - "unicode" - - "github.com/andreimarcu/linx-server/backends" - "github.com/andreimarcu/linx-server/expiry" - "github.com/dchest/uniuri" - "github.com/minio/sha256-simd" - "gopkg.in/h2non/filetype.v1" -) - -var NotFoundErr = errors.New("File not found.") - -func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) { - file, err := fileBackend.Open(fName) - if err != nil { - return - } - defer file.Close() - - m.Size, err = fileBackend.Size(fName) - if err != nil { - return - } - - m.Expiry = exp - - if delKey == "" { - m.DeleteKey = uniuri.NewLen(30) - } else { - m.DeleteKey = delKey - } - - // Get first 512 bytes for mimetype detection - header := make([]byte, 512) - file.Read(header) - - kind, err := filetype.Match(header) - if err != nil { - m.Mimetype = "application/octet-stream" - } else { - m.Mimetype = kind.MIME.Value - } - - if m.Mimetype == "" { - // Check if the file seems anything like text - if printable(header) { - m.Mimetype = "text/plain" - } else { - m.Mimetype = "application/octet-stream" - } - } - - // Compute the sha256sum - hasher := sha256.New() - file.Seek(0, 0) - _, err = io.Copy(hasher, file) - if err == nil { - m.Sha256sum = hex.EncodeToString(hasher.Sum(nil)) - } - file.Seek(0, 0) - - // If archive, grab list of filenames - if m.Mimetype == "application/x-tar" { - tReadr := tar.NewReader(file) - for { - hdr, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { - m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) - } - } - sort.Strings(m.ArchiveFiles) - } else if m.Mimetype == "application/x-gzip" { - gzf, err := gzip.NewReader(file) - if err == nil { - tReadr := tar.NewReader(gzf) - for { - hdr, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { - m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) - } - } - sort.Strings(m.ArchiveFiles) - } - } else if m.Mimetype == "application/x-bzip" { - bzf := bzip2.NewReader(file) - tReadr := tar.NewReader(bzf) - for { - hdr, err := tReadr.Next() - if err == io.EOF || err != nil { - break - } - if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg { - m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name) - } - } - sort.Strings(m.ArchiveFiles) - } else if m.Mimetype == "application/zip" { - zf, err := zip.NewReader(file, m.Size) - if err == nil { - for _, f := range zf.File { - m.ArchiveFiles = append(m.ArchiveFiles, f.Name) - } - } - sort.Strings(m.ArchiveFiles) - } - - return -} - -func metadataWrite(filename string, metadata *backends.Metadata) error { - return metaBackend.Put(filename, metadata) -} - -func metadataRead(filename string) (metadata backends.Metadata, err error) { - metadata, err = metaBackend.Get(filename) - if err != nil { - // Metadata does not exist, generate one - newMData, err := generateMetadata(filename, expiry.NeverExpire, "") - if err != nil { - return metadata, err - } - metadataWrite(filename, &newMData) - - metadata, err = metaBackend.Get(filename) - } - - return -} - -func printable(data []byte) bool { - for i, b := range data { - r := rune(b) - - // A null terminator that's not at the beginning of the file - if r == 0 && i == 0 { - return false - } else if r == 0 && i < 0 { - continue - } - - if r > unicode.MaxASCII { - return false - } - - } - - return true -} diff --git a/pages.go b/pages.go index f58fa88..bb38f37 100644 --- a/pages.go +++ b/pages.go @@ -64,12 +64,10 @@ func oopsHandler(c web.C, w http.ResponseWriter, r *http.Request, rt RespType, m w.WriteHeader(500) renderTemplate(Templates["oops.html"], pongo2.Context{"msg": msg}, r, w) return - } else if rt == RespPLAIN { w.WriteHeader(500) fmt.Fprintf(w, "%s", msg) return - } else if rt == RespJSON { js, _ := json.Marshal(map[string]string{ "error": msg, @@ -79,7 +77,6 @@ func oopsHandler(c web.C, w http.ResponseWriter, r *http.Request, rt RespType, m w.WriteHeader(500) w.Write(js) return - } else if rt == RespAUTO { if strings.EqualFold("application/json", r.Header.Get("Accept")) { oopsHandler(c, w, r, RespJSON, msg) @@ -89,11 +86,33 @@ func oopsHandler(c web.C, w http.ResponseWriter, r *http.Request, rt RespType, m } } -func badRequestHandler(c web.C, w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadRequest) - err := renderTemplate(Templates["400.html"], pongo2.Context{}, r, w) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) +func badRequestHandler(c web.C, w http.ResponseWriter, r *http.Request, rt RespType, msg string) { + if rt == RespHTML { + w.WriteHeader(http.StatusBadRequest) + err := renderTemplate(Templates["400.html"], pongo2.Context{"msg": msg}, r, w) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } else if rt == RespPLAIN { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, "%s", msg) + return + } else if rt == RespJSON { + js, _ := json.Marshal(map[string]string{ + "error": msg, + }) + + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusBadRequest) + w.Write(js) + return + } else if rt == RespAUTO { + if strings.EqualFold("application/json", r.Header.Get("Accept")) { + badRequestHandler(c, w, r, RespJSON, msg) + } else { + badRequestHandler(c, w, r, RespHTML, msg) + } } } diff --git a/server.go b/server.go index c7b4342..851a7cf 100644 --- a/server.go +++ b/server.go @@ -16,7 +16,7 @@ import ( "github.com/GeertJohan/go.rice" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends/localfs" - "github.com/andreimarcu/linx-server/backends/metajson" + "github.com/andreimarcu/linx-server/backends/s3" "github.com/flosch/pongo2" "github.com/vharitonsky/iniflags" "github.com/zenazn/goji/graceful" @@ -61,6 +61,9 @@ var Config struct { remoteAuthFile string addHeaders headerList noDirectAgents bool + s3Endpoint string + s3Region string + s3Bucket string } var Templates = make(map[string]*pongo2.Template) @@ -70,8 +73,7 @@ var timeStarted time.Time var timeStartedStr string var remoteAuthKeys []string var metaStorageBackend backends.MetaStorageBackend -var metaBackend backends.MetaBackend -var fileBackend backends.StorageBackend +var storageBackend backends.StorageBackend func setup() *web.Mux { mux := web.New() @@ -135,9 +137,11 @@ func setup() *web.Mux { Config.selifPath = Config.selifPath + "/" } - metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir) - metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend) - fileBackend = localfs.NewLocalfsBackend(Config.filesDir) + if Config.s3Bucket != "" { + storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint) + } else { + storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir) + } // Template setup p2l, err := NewPongo2TemplatesLoader() @@ -255,6 +259,12 @@ func main() { "Add an arbitrary header to the response. This option can be used multiple times.") flag.BoolVar(&Config.noDirectAgents, "nodirectagents", false, "disable serving files directly for wget/curl user agents") + flag.StringVar(&Config.s3Endpoint, "s3-endpoint", "", + "S3 endpoint") + flag.StringVar(&Config.s3Region, "s3-region", "", + "S3 region") + flag.StringVar(&Config.s3Bucket, "s3-bucket", "", + "S3 bucket to use for files and metadata") iniflags.Parse() diff --git a/server_test.go b/server_test.go index d9374ab..a1ec853 100644 --- a/server_test.go +++ b/server_test.go @@ -486,7 +486,6 @@ func TestPostJSONUploadMaxExpiry(t *testing.T) { var myjson RespOkJSON err = json.Unmarshal([]byte(w.Body.String()), &myjson) if err != nil { - fmt.Println(w.Body.String()) t.Fatal(err) } @@ -643,14 +642,45 @@ func TestPostEmptyUpload(t *testing.T) { mux.ServeHTTP(w, req) - if w.Code != 500 { + if w.Code != 400 { t.Log(w.Body.String()) - t.Fatalf("Status code is not 500, but %d", w.Code) + t.Fatalf("Status code is not 400, but %d", w.Code) } +} + +func TestPostTooLargeUpload(t *testing.T) { + mux := setup() + oldMaxSize := Config.maxSize + Config.maxSize = 2 + w := httptest.NewRecorder() - if !strings.Contains(w.Body.String(), "Empty file") { - t.Fatal("Response did not contain 'Empty file'") + filename := generateBarename() + ".txt" + + var b bytes.Buffer + mw := multipart.NewWriter(&b) + fw, err := mw.CreateFormFile("file", filename) + if err != nil { + t.Fatal(err) + } + + fw.Write([]byte("test content")) + mw.Close() + + req, err := http.NewRequest("POST", "/upload/", &b) + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.Header.Set("Referer", Config.siteURL) + if err != nil { + t.Fatal(err) + } + + mux.ServeHTTP(w, req) + + if w.Code != 400 { + t.Log(w.Body.String()) + t.Fatalf("Status code is not 400, but %d", w.Code) } + + Config.maxSize = oldMaxSize } func TestPostEmptyJSONUpload(t *testing.T) { @@ -679,9 +709,9 @@ func TestPostEmptyJSONUpload(t *testing.T) { mux.ServeHTTP(w, req) - if w.Code != 500 { + if w.Code != 400 { t.Log(w.Body.String()) - t.Fatalf("Status code is not 500, but %d", w.Code) + t.Fatalf("Status code is not 400, but %d", w.Code) } var myjson RespErrJSON @@ -690,7 +720,7 @@ func TestPostEmptyJSONUpload(t *testing.T) { t.Fatal(err) } - if myjson.Error != "Could not upload file: Empty file" { + if myjson.Error != "Empty file" { t.Fatal("Json 'error' was not 'Empty file' but " + myjson.Error) } } @@ -768,9 +798,39 @@ func TestPutEmptyUpload(t *testing.T) { mux.ServeHTTP(w, req) - if !strings.Contains(w.Body.String(), "Empty file") { - t.Fatal("Response doesn't contain'Empty file'") + if w.Code != 400 { + t.Fatalf("Status code is not 400, but %d", w.Code) + } +} + +func TestPutTooLargeUpload(t *testing.T) { + mux := setup() + oldMaxSize := Config.maxSize + Config.maxSize = 2 + + w := httptest.NewRecorder() + + filename := generateBarename() + ".file" + + req, err := http.NewRequest("PUT", "/upload/"+filename, strings.NewReader("File too big")) + if err != nil { + t.Fatal(err) + } + + req.Header.Set("Linx-Randomize", "yes") + + mux.ServeHTTP(w, req) + + if w.Code != 500 { + t.Log(w.Body.String()) + t.Fatalf("Status code is not 500, but %d", w.Code) } + + if !strings.Contains(w.Body.String(), "request body too large") { + t.Fatal("Response did not contain 'request body too large'") + } + + Config.maxSize = oldMaxSize } func TestPutJSONUpload(t *testing.T) { diff --git a/torrent.go b/torrent.go index 4155872..c5e7a58 100644 --- a/torrent.go +++ b/torrent.go @@ -2,65 +2,44 @@ package main import ( "bytes" - "crypto/sha1" "fmt" "io" "net/http" "time" "github.com/andreimarcu/linx-server/backends" + "github.com/andreimarcu/linx-server/expiry" + "github.com/andreimarcu/linx-server/torrent" "github.com/zeebo/bencode" "github.com/zenazn/goji/web" ) -const ( - TORRENT_PIECE_LENGTH = 262144 -) - -type TorrentInfo struct { - PieceLength int `bencode:"piece length"` - Pieces string `bencode:"pieces"` - Name string `bencode:"name"` - Length int `bencode:"length"` -} - -type Torrent struct { - Encoding string `bencode:"encoding"` - Info TorrentInfo `bencode:"info"` - UrlList []string `bencode:"url-list"` -} - -func hashPiece(piece []byte) []byte { - h := sha1.New() - h.Write(piece) - return h.Sum(nil) -} +func createTorrent(fileName string, f io.Reader, r *http.Request) ([]byte, error) { + url := getSiteURL(r) + Config.selifPath + fileName + chunk := make([]byte, torrent.TORRENT_PIECE_LENGTH) -func createTorrent(fileName string, f io.ReadCloser, r *http.Request) ([]byte, error) { - chunk := make([]byte, TORRENT_PIECE_LENGTH) - - torrent := Torrent{ + t := torrent.Torrent{ Encoding: "UTF-8", - Info: TorrentInfo{ - PieceLength: TORRENT_PIECE_LENGTH, + Info: torrent.TorrentInfo{ + PieceLength: torrent.TORRENT_PIECE_LENGTH, Name: fileName, }, - UrlList: []string{fmt.Sprintf("%s%s%s", getSiteURL(r), Config.selifPath, fileName)}, + UrlList: []string{url}, } for { - n, err := f.Read(chunk) + n, err := io.ReadFull(f, chunk) if err == io.EOF { break - } else if err != nil { + } else if err != nil && err != io.ErrUnexpectedEOF { return []byte{}, err } - torrent.Info.Length += n - torrent.Info.Pieces += string(hashPiece(chunk[:n])) + t.Info.Length += n + t.Info.Pieces += string(torrent.HashPiece(chunk[:n])) } - data, err := bencode.EncodeBytes(&torrent) + data, err := bencode.EncodeBytes(&t) if err != nil { return []byte{}, err } @@ -71,21 +50,24 @@ func createTorrent(fileName string, f io.ReadCloser, r *http.Request) ([]byte, e func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) { fileName := c.URLParams["name"] - _, err := checkFile(fileName) - if err == NotFoundErr { + metadata, f, err := storageBackend.Get(fileName) + if err == backends.NotFoundErr { notFoundHandler(c, w, r) return } else if err == backends.BadMetadata { oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") return + } else if err != nil { + oopsHandler(c, w, r, RespAUTO, err.Error()) + return } + defer f.Close() - f, err := fileBackend.Open(fileName) - if err != nil { - oopsHandler(c, w, r, RespHTML, "Could not create torrent.") + if expiry.IsTsExpired(metadata.Expiry) { + storageBackend.Delete(fileName) + notFoundHandler(c, w, r) return } - defer f.Close() encoded, err := createTorrent(fileName, f, r) if err != nil { diff --git a/torrent/torrent.go b/torrent/torrent.go new file mode 100644 index 0000000..a47d884 --- /dev/null +++ b/torrent/torrent.go @@ -0,0 +1,28 @@ +package torrent + +import ( + "crypto/sha1" +) + +const ( + TORRENT_PIECE_LENGTH = 262144 +) + +type TorrentInfo struct { + PieceLength int `bencode:"piece length"` + Pieces string `bencode:"pieces"` + Name string `bencode:"name"` + Length int `bencode:"length"` +} + +type Torrent struct { + Encoding string `bencode:"encoding"` + Info TorrentInfo `bencode:"info"` + UrlList []string `bencode:"url-list"` +} + +func HashPiece(piece []byte) []byte { + h := sha1.New() + h.Write(piece) + return h.Sum(nil) +} diff --git a/torrent_test.go b/torrent_test.go index b553231..1d227fd 100644 --- a/torrent_test.go +++ b/torrent_test.go @@ -5,12 +5,13 @@ import ( "os" "testing" + "github.com/andreimarcu/linx-server/torrent" "github.com/zeebo/bencode" ) func TestCreateTorrent(t *testing.T) { fileName := "server.go" - var decoded Torrent + var decoded torrent.Torrent f, err := os.Open("server.go") if err != nil { @@ -52,7 +53,7 @@ func TestCreateTorrent(t *testing.T) { } func TestCreateTorrentWithImage(t *testing.T) { - var decoded Torrent + var decoded torrent.Torrent f, err := os.Open("static/images/404.jpg") if err != nil { diff --git a/upload.go b/upload.go index acdd204..d46c4d5 100644 --- a/upload.go +++ b/upload.go @@ -22,6 +22,7 @@ import ( "gopkg.in/h2non/filetype.v1" ) +var FileTooLargeError = errors.New("File too large.") var fileBlacklist = map[string]bool{ "favicon.ico": true, "index.htm": true, @@ -34,10 +35,11 @@ var fileBlacklist = map[string]bool{ // Describes metadata directly from the user request type UploadRequest struct { src io.Reader + size int64 filename string expiry time.Duration // Seconds until expiry, 0 = never + deleteKey string // Empty string if not defined randomBarename bool - deletionKey string // Empty string if not defined } // Metadata associated with a file as it would actually be stored @@ -48,7 +50,7 @@ type Upload struct { func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { if !strictReferrerCheck(r, getSiteURL(r), []string{"Linx-Delete-Key", "Linx-Expiry", "Linx-Randomize", "X-Requested-With"}) { - badRequestHandler(c, w, r) + badRequestHandler(c, w, r, RespAUTO, "") return } @@ -65,32 +67,39 @@ func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { } defer file.Close() - r.ParseForm() - if r.Form.Get("randomize") == "true" { - upReq.randomBarename = true - } - upReq.expiry = parseExpiry(r.Form.Get("expires")) upReq.src = file + upReq.size = headers.Size upReq.filename = headers.Filename } else { - if r.FormValue("content") == "" { - oopsHandler(c, w, r, RespHTML, "Empty file") + if r.PostFormValue("content") == "" { + badRequestHandler(c, w, r, RespAUTO, "Empty file") return } - extension := r.FormValue("extension") + extension := r.PostFormValue("extension") if extension == "" { extension = "txt" } - upReq.src = strings.NewReader(r.FormValue("content")) - upReq.expiry = parseExpiry(r.FormValue("expires")) - upReq.filename = r.FormValue("filename") + "." + extension + content := r.PostFormValue("content") + + upReq.src = strings.NewReader(content) + upReq.size = int64(len(content)) + upReq.filename = r.PostFormValue("filename") + "." + extension + } + + upReq.expiry = parseExpiry(r.PostFormValue("expires")) + + if r.PostFormValue("randomize") == "true" { + upReq.randomBarename = true } upload, err := processUpload(upReq) if strings.EqualFold("application/json", r.Header.Get("Accept")) { - if err != nil { + if err == FileTooLargeError || err == backends.FileEmptyError { + badRequestHandler(c, w, r, RespJSON, err.Error()) + return + } else if err != nil { oopsHandler(c, w, r, RespJSON, "Could not upload file: "+err.Error()) return } @@ -99,14 +108,16 @@ func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.Write(js) } else { - if err != nil { + if err == FileTooLargeError || err == backends.FileEmptyError { + badRequestHandler(c, w, r, RespHTML, err.Error()) + return + } else if err != nil { oopsHandler(c, w, r, RespHTML, "Could not upload file: "+err.Error()) return } http.Redirect(w, r, Config.sitePath+upload.Filename, 303) } - } func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) { @@ -115,12 +126,15 @@ func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) { defer r.Body.Close() upReq.filename = c.URLParams["name"] - upReq.src = r.Body + upReq.src = http.MaxBytesReader(w, r.Body, Config.maxSize) upload, err := processUpload(upReq) if strings.EqualFold("application/json", r.Header.Get("Accept")) { - if err != nil { + if err == FileTooLargeError || err == backends.FileEmptyError { + badRequestHandler(c, w, r, RespJSON, err.Error()) + return + } else if err != nil { oopsHandler(c, w, r, RespJSON, "Could not upload file: "+err.Error()) return } @@ -129,7 +143,10 @@ func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json; charset=UTF-8") w.Write(js) } else { - if err != nil { + if err == FileTooLargeError || err == backends.FileEmptyError { + badRequestHandler(c, w, r, RespPLAIN, err.Error()) + return + } else if err != nil { oopsHandler(c, w, r, RespPLAIN, "Could not upload file: "+err.Error()) return } @@ -162,8 +179,8 @@ func uploadRemote(c web.C, w http.ResponseWriter, r *http.Request) { } upReq.filename = filepath.Base(grabUrl.Path) - upReq.src = resp.Body - upReq.deletionKey = r.FormValue("deletekey") + upReq.src = http.MaxBytesReader(w, resp.Body, Config.maxSize) + upReq.deleteKey = r.FormValue("deletekey") upReq.randomBarename = r.FormValue("randomize") == "yes" upReq.expiry = parseExpiry(r.FormValue("expiry")) @@ -193,15 +210,18 @@ func uploadHeaderProcess(r *http.Request, upReq *UploadRequest) { upReq.randomBarename = true } - upReq.deletionKey = r.Header.Get("Linx-Delete-Key") + upReq.deleteKey = r.Header.Get("Linx-Delete-Key") // Get seconds until expiry. Non-integer responses never expire. expStr := r.Header.Get("Linx-Expiry") upReq.expiry = parseExpiry(expStr) - } func processUpload(upReq UploadRequest) (upload Upload, err error) { + if upReq.size > Config.maxSize { + return upload, FileTooLargeError + } + // Determine the appropriate filename, then write to disk barename, extension := barePlusExt(upReq.filename) @@ -215,7 +235,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { header = make([]byte, 512) n, _ := upReq.src.Read(header) if n == 0 { - return upload, errors.New("Empty file") + return upload, backends.FileEmptyError } header = header[:n] @@ -231,13 +251,13 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { upload.Filename = strings.Join([]string{barename, extension}, ".") upload.Filename = strings.Replace(upload.Filename, " ", "", -1) - fileexists, _ := fileBackend.Exists(upload.Filename) + fileexists, _ := storageBackend.Exists(upload.Filename) // Check if the delete key matches, in which case overwrite if fileexists { - metad, merr := metadataRead(upload.Filename) + metad, merr := storageBackend.Head(upload.Filename) if merr == nil { - if upReq.deletionKey == metad.DeleteKey { + if upReq.deleteKey == metad.DeleteKey { fileexists = false } } @@ -252,7 +272,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { } upload.Filename = strings.Join([]string{barename, extension}, ".") - fileexists, err = fileBackend.Exists(upload.Filename) + fileexists, err = storageBackend.Exists(upload.Filename) } if fileBlacklist[strings.ToLower(upload.Filename)] { @@ -267,24 +287,15 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { fileExpiry = time.Now().Add(upReq.expiry) } - bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src)) - if err != nil { - return upload, err - } else if bytes > Config.maxSize { - fileBackend.Delete(upload.Filename) - return upload, errors.New("File too large") + if upReq.deleteKey == "" { + upReq.deleteKey = uniuri.NewLen(30) } - upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey) + upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deleteKey) if err != nil { - fileBackend.Delete(upload.Filename) - return - } - err = metadataWrite(upload.Filename, &upload.Metadata) - if err != nil { - fileBackend.Delete(upload.Filename) - return + return upload, err } + return } From 35c4415f8d1e96843fb65a469533062738ccb39e Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Thu, 24 Jan 2019 23:39:17 -0800 Subject: [PATCH 17/30] Document storage backend usage --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8d4ef11..957757c 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,7 @@ allowhotlink = true - ```-bind 127.0.0.1:8080``` -- what to bind to (default is 127.0.0.1:8080) - ```-sitename myLinx``` -- the site name displayed on top (default is inferred from Host header) - ```-siteurl "http://mylinx.example.org/"``` -- the site url (default is inferred from execution context) -- ```-selifpath "selif"``` -- path relative to site base url (the "selif" in https://mylinx.example.org/selif/image.jpg) where files are accessed directly (default: selif) -- ```-filespath files/``` -- Path to store uploads (default is files/) -- ```-metapath meta/``` -- Path to store information about uploads (default is meta/) +- ```-selifpath "selif"``` -- path relative to site base url (the "selif" in mylinx.example.org/selif/image.jpg) where files are accessed directly (default: selif) - ```-maxsize 4294967296``` -- maximum upload file size in bytes (default 4GB) - ```-maxexpiry 86400``` -- maximum expiration time in seconds (default is 0, which is no expiry) - ```-allowhotlink``` -- Allow file hotlinking @@ -56,6 +54,15 @@ allowhotlink = true - ```-remoteuploads``` -- (optionally) enable remote uploads (/upload?url=https://...) - ```-nologs``` -- (optionally) disable request logs in stdout +#### Storage backends +The following storage backends are available: + +|Name|Options|Notes +|----|-------|----- +|LocalFS|```-filespath files/``` -- Path to store uploads (default is files/)
```-metapath meta/``` -- Path to store information about uploads (default is meta/)|Enabled by default, this backend uses the filesystem| +|S3|```-s3-endpoint https://...``` -- S3 endpoint
```-s3-region us-east-1``` -- S3 region
```-s3-bucket mybucket``` -- S3 bucket to use for files and metadata

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).| + + #### SSL with built-in server - ```-certfile path/to/your.crt``` -- Path to the ssl certificate (required if you want to use the https server) - ```-keyfile path/to/your.key``` -- Path to the ssl key (required if you want to use the https server) From 207c19e3df34837489cee90ba4e2fa97f150f4a1 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Fri, 25 Jan 2019 08:10:06 +0000 Subject: [PATCH 18/30] Add -s3-force-path-style flag and config option (#157) This option forces path-style addressing instead of using a subdomain. This appears to be needed by Minio. --- README.md | 2 +- backends/s3/s3.go | 5 ++++- server.go | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 957757c..0ad244b 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The following storage backends are available: |Name|Options|Notes |----|-------|----- |LocalFS|```-filespath files/``` -- Path to store uploads (default is files/)
```-metapath meta/``` -- Path to store information about uploads (default is meta/)|Enabled by default, this backend uses the filesystem| -|S3|```-s3-endpoint https://...``` -- S3 endpoint
```-s3-region us-east-1``` -- S3 region
```-s3-bucket mybucket``` -- S3 bucket to use for files and metadata

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).| +|S3|```-s3-endpoint https://...``` -- S3 endpoint
```-s3-region us-east-1``` -- S3 region
```-s3-bucket mybucket``` -- S3 bucket to use for files and metadata
```-s3-use-path-style``` -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).| #### SSL with built-in server diff --git a/backends/s3/s3.go b/backends/s3/s3.go index 7ae326c..45067c1 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -177,7 +177,7 @@ func (b S3Backend) List() ([]string, error) { return output, nil } -func NewS3Backend(bucket string, region string, endpoint string) S3Backend { +func NewS3Backend(bucket string, region string, endpoint string, forcePathStyle bool) S3Backend { awsConfig := &aws.Config{} if region != "" { awsConfig.Region = aws.String(region) @@ -185,6 +185,9 @@ func NewS3Backend(bucket string, region string, endpoint string) S3Backend { if endpoint != "" { awsConfig.Endpoint = aws.String(endpoint) } + if forcePathStyle == true { + awsConfig.S3ForcePathStyle = aws.Bool(true) + } sess := session.Must(session.NewSession(awsConfig)) svc := s3.New(sess) diff --git a/server.go b/server.go index 851a7cf..7883a07 100644 --- a/server.go +++ b/server.go @@ -64,6 +64,7 @@ var Config struct { s3Endpoint string s3Region string s3Bucket string + s3ForcePathStyle bool } var Templates = make(map[string]*pongo2.Template) @@ -138,7 +139,7 @@ func setup() *web.Mux { } if Config.s3Bucket != "" { - storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint) + storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint, Config.s3ForcePathStyle) } else { storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir) } @@ -265,6 +266,8 @@ func main() { "S3 region") flag.StringVar(&Config.s3Bucket, "s3-bucket", "", "S3 bucket to use for files and metadata") + flag.BoolVar(&Config.s3ForcePathStyle, "s3-force-path-style", false, + "Force path-style addressing for S3 (e.g. https://s3.amazonaws.com/linx/example.txt)") iniflags.Parse() From f6cd7fc6fe6f5d9dc6bc27cabb47671250222b5b Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 25 Jan 2019 00:20:52 -0800 Subject: [PATCH 19/30] Tweak documentation --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0ad244b..6cb85ab 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ allowhotlink = true #### Options - ```-bind 127.0.0.1:8080``` -- what to bind to (default is 127.0.0.1:8080) - ```-sitename myLinx``` -- the site name displayed on top (default is inferred from Host header) -- ```-siteurl "http://mylinx.example.org/"``` -- the site url (default is inferred from execution context) +- ```-siteurl "https://mylinx.example.org/"``` -- the site url (default is inferred from execution context) - ```-selifpath "selif"``` -- path relative to site base url (the "selif" in mylinx.example.org/selif/image.jpg) where files are accessed directly (default: selif) - ```-maxsize 4294967296``` -- maximum upload file size in bytes (default 4GB) - ```-maxexpiry 86400``` -- maximum expiration time in seconds (default is 0, which is no expiry) @@ -57,10 +57,10 @@ allowhotlink = true #### Storage backends The following storage backends are available: -|Name|Options|Notes -|----|-------|----- -|LocalFS|```-filespath files/``` -- Path to store uploads (default is files/)
```-metapath meta/``` -- Path to store information about uploads (default is meta/)|Enabled by default, this backend uses the filesystem| -|S3|```-s3-endpoint https://...``` -- S3 endpoint
```-s3-region us-east-1``` -- S3 region
```-s3-bucket mybucket``` -- S3 bucket to use for files and metadata
```-s3-use-path-style``` -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).| +|Name|Notes|Options +|----|-----|------- +|LocalFS|Enabled by default, this backend uses the filesystem|```-filespath files/``` -- Path to store uploads (default is files/)
```-metapath meta/``` -- Path to store information about uploads (default is meta/)| +|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```-s3-endpoint https://...``` -- S3 endpoint
```-s3-region us-east-1``` -- S3 region
```-s3-bucket mybucket``` -- S3 bucket to use for files and metadata
```-s3-force-path-style``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| #### SSL with built-in server From d5aa09a65c1949dbf3d8feee5a1a6da77c4b1f15 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 25 Jan 2019 01:10:09 -0800 Subject: [PATCH 20/30] Update screenshots in readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 6cb85ab..8b55687 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,7 @@ Self-hosted file/media sharing website. ### Screenshots - - + Get release and run From 5037573eab449305d5b5d6ee672bb5c2da285bfe Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Sat, 26 Jan 2019 08:56:35 +0000 Subject: [PATCH 21/30] Clean up build.sh and build for linux/arm64 (#158) * Clean up build.sh and build for linux/arm64 --- build.sh | 141 ++++++++++++++++++++++--------------------------------- 1 file changed, 57 insertions(+), 84 deletions(-) diff --git a/build.sh b/build.sh index f7a4115..c4fc7f4 100755 --- a/build.sh +++ b/build.sh @@ -1,94 +1,67 @@ #!/bin/bash -version="$1" -mkdir -p "binairies/""$version" -name="binairies/""$version""/linx-server-v""$version""_" - -GOOS=darwin GOARCH=amd64 go build -o "$name"osx-amd64 -rice append --exec "$name"osx-amd64 - -GOOS=darwin GOARCH=386 go build -o "$name"osx-386 -rice append --exec "$name"osx-386 - -GOOS=freebsd GOARCH=amd64 go build -o "$name"freebsd-amd64 -rice append --exec "$name"freebsd-amd64 - -GOOS=freebsd GOARCH=386 go build -o "$name"freebsd-386 -rice append --exec "$name"freebsd-386 - -GOOS=openbsd GOARCH=amd64 go build -o "$name"openbsd-amd64 -rice append --exec "$name"openbsd-amd64 - -GOOS=openbsd GOARCH=386 go build -o "$name"openbsd-386 -rice append --exec "$name"openbsd-386 - -GOOS=linux GOARCH=arm go build -o "$name"linux-arm -rice append --exec "$name"linux-arm - -GOOS=linux GOARCH=amd64 go build -o "$name"linux-amd64 -rice append --exec "$name"linux-amd64 - -GOOS=linux GOARCH=386 go build -o "$name"linux-386 -rice append --exec "$name"linux-386 +function build_binary_rice { + name="$1" + + for arch in amd64 386; do + GOOS=darwin GOARCH=$arch go build -o "$name"osx-$arch + rice append --exec "$name"osx-$arch + done + + for arch in amd64 386; do + GOOS=freebsd GOARCH=$arch go build -o "$name"freebsd-$arch + rice append --exec "$name"freebsd-$arch + done + + for arch in amd64 386; do + GOOS=openbsd GOARCH=$arch go build -o "$name"openbsd-$arch + rice append --exec "$name"openbsd-$arch + done + + for arch in arm arm64 amd64 386; do + GOOS=linux GOARCH=$arch go build -o "$name"linux-$arch + rice append --exec "$name"linux-$arch + done + + for arch in amd64 386; do + GOOS=windows GOARCH=$arch go build -o "$name"windows-$arch.exe + rice append --exec "$name"windows-$arch.exe + done +} + +function build_binary { + name="$1" + + for arch in amd64 386; do + GOOS=darwin GOARCH=$arch go build -o "$name"osx-$arch + done + + for arch in amd64 386; do + GOOS=freebsd GOARCH=$arch go build -o "$name"freebsd-$arch + done + + for arch in amd64 386; do + GOOS=openbsd GOARCH=$arch go build -o "$name"openbsd-$arch + done + + for arch in arm arm64 amd64 386; do + GOOS=linux GOARCH=$arch go build -o "$name"linux-$arch + done + + for arch in amd64 386; do + GOOS=windows GOARCH=$arch go build -o "$name"windows-$arch.exe + done +} -GOOS=windows GOARCH=amd64 go build -o "$name"windows-amd64.exe -rice append --exec "$name"windows-amd64.exe - -GOOS=windows GOARCH=386 go build -o "$name"windows-386.exe -rice append --exec "$name"windows-386.exe +version="$1" +mkdir -p "binaries/""$version" +build_binary_rice "binaries/""$version""/linx-server-v""$version""_" cd linx-genkey -name="../binairies/""$version""/linx-genkey-v""$version""_" - -GOOS=darwin GOARCH=amd64 go build -o "$name"osx-amd64 - -GOOS=darwin GOARCH=386 go build -o "$name"osx-386 - -GOOS=freebsd GOARCH=amd64 go build -o "$name"freebsd-amd64 - -GOOS=freebsd GOARCH=386 go build -o "$name"freebsd-386 - -GOOS=openbsd GOARCH=amd64 go build -o "$name"openbsd-amd64 - -GOOS=openbsd GOARCH=386 go build -o "$name"openbsd-386 - -GOOS=linux GOARCH=arm go build -o "$name"linux-arm - -GOOS=linux GOARCH=amd64 go build -o "$name"linux-amd64 - -GOOS=linux GOARCH=386 go build -o "$name"linux-386 - -GOOS=windows GOARCH=amd64 go build -o "$name"windows-amd64.exe - -GOOS=windows GOARCH=386 go build -o "$name"windows-386.exe - +build_binary "../binaries/""$version""/linx-genkey-v""$version""_" cd .. - cd linx-cleanup -name="../binairies/""$version""/linx-cleanup-v""$version""_" - -GOOS=darwin GOARCH=amd64 go build -o "$name"osx-amd64 - -GOOS=darwin GOARCH=386 go build -o "$name"osx-386 - -GOOS=freebsd GOARCH=amd64 go build -o "$name"freebsd-amd64 - -GOOS=freebsd GOARCH=386 go build -o "$name"freebsd-386 - -GOOS=openbsd GOARCH=amd64 go build -o "$name"openbsd-amd64 - -GOOS=openbsd GOARCH=386 go build -o "$name"openbsd-386 - -GOOS=linux GOARCH=arm go build -o "$name"linux-arm - -GOOS=linux GOARCH=amd64 go build -o "$name"linux-amd64 - -GOOS=linux GOARCH=386 go build -o "$name"linux-386 - -GOOS=windows GOARCH=amd64 go build -o "$name"windows-amd64.exe - -GOOS=windows GOARCH=386 go build -o "$name"windows-386.exe - +build_binary "../binaries/""$version""/linx-cleanup-v""$version""_" cd .. From a79bc1898ac34dfa727dade2e80b41d535eb54f4 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Sat, 26 Jan 2019 01:00:04 -0800 Subject: [PATCH 22/30] Add binaries/ to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bd9da44..37e1e54 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ _testmain.go linx-server files/ meta/ +binaries/ linx-cleanup From 8f3108148b53f078fb19e0d2c9ac04acba8f7c8d Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Sat, 26 Jan 2019 10:04:32 +0000 Subject: [PATCH 23/30] Add option to force random filenames (fixes #86) (#159) --- README.md | 1 + pages.go | 11 ++- server.go | 3 + server_test.go | 75 +++++++++++++++++++ static/js/upload.js | 175 ++++++++++++++++++++++--------------------- templates/API.html | 14 ++-- templates/index.html | 2 +- templates/paste.html | 2 +- upload.go | 29 +++++-- 9 files changed, 209 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 8b55687..bd1d9eb 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ allowhotlink = true - ```-xframeoptions "..." ``` -- X-Frame-Options header (default is "SAMEORIGIN") - ```-remoteuploads``` -- (optionally) enable remote uploads (/upload?url=https://...) - ```-nologs``` -- (optionally) disable request logs in stdout +- ```-force-random-filename``` -- (optionally) force the use of random filenames #### Storage backends The following storage backends are available: diff --git a/pages.go b/pages.go index bb38f37..6fcc934 100644 --- a/pages.go +++ b/pages.go @@ -21,8 +21,9 @@ const ( func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["index.html"], pongo2.Context{ - "maxsize": Config.maxSize, - "expirylist": listExpirationTimes(), + "maxsize": Config.maxSize, + "expirylist": listExpirationTimes(), + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -31,7 +32,8 @@ func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) { func pasteHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["paste.html"], pongo2.Context{ - "expirylist": listExpirationTimes(), + "expirylist": listExpirationTimes(), + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { oopsHandler(c, w, r, RespHTML, "") @@ -40,7 +42,8 @@ func pasteHandler(c web.C, w http.ResponseWriter, r *http.Request) { func apiDocHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["API.html"], pongo2.Context{ - "siteurl": getSiteURL(r), + "siteurl": getSiteURL(r), + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { oopsHandler(c, w, r, RespHTML, "") diff --git a/server.go b/server.go index 7883a07..e4e1661 100644 --- a/server.go +++ b/server.go @@ -65,6 +65,7 @@ var Config struct { s3Region string s3Bucket string s3ForcePathStyle bool + forceRandomFilename bool } var Templates = make(map[string]*pongo2.Template) @@ -268,6 +269,8 @@ func main() { "S3 bucket to use for files and metadata") flag.BoolVar(&Config.s3ForcePathStyle, "s3-force-path-style", false, "Force path-style addressing for S3 (e.g. https://s3.amazonaws.com/linx/example.txt)") + flag.BoolVar(&Config.forceRandomFilename, "force-random-filename", false, + "Force all uploads to use a random filename") iniflags.Parse() diff --git a/server_test.go b/server_test.go index a1ec853..fc225ce 100644 --- a/server_test.go +++ b/server_test.go @@ -763,6 +763,32 @@ func TestPutRandomizedUpload(t *testing.T) { } } +func TestPutForceRandomUpload(t *testing.T) { + mux := setup() + w := httptest.NewRecorder() + + oldFRF := Config.forceRandomFilename + Config.forceRandomFilename = true + filename := "randomizeme.file" + + req, err := http.NewRequest("PUT", "/upload/"+filename, strings.NewReader("File content")) + if err != nil { + t.Fatal(err) + } + + // while this should also work without this header, let's try to force + // the randomized filename off to be sure + req.Header.Set("Linx-Randomize", "no") + + mux.ServeHTTP(w, req) + + if w.Body.String() == Config.siteURL+filename { + t.Fatal("Filename was not random") + } + + Config.forceRandomFilename = oldFRF +} + func TestPutNoExtensionUpload(t *testing.T) { mux := setup() w := httptest.NewRecorder() @@ -1013,6 +1039,55 @@ func TestPutAndOverwrite(t *testing.T) { } } +func TestPutAndOverwriteForceRandom(t *testing.T) { + var myjson RespOkJSON + + mux := setup() + w := httptest.NewRecorder() + + oldFRF := Config.forceRandomFilename + Config.forceRandomFilename = true + + req, err := http.NewRequest("PUT", "/upload", strings.NewReader("File content")) + if err != nil { + t.Fatal(err) + } + + req.Header.Set("Accept", "application/json") + + mux.ServeHTTP(w, req) + + err = json.Unmarshal([]byte(w.Body.String()), &myjson) + if err != nil { + t.Fatal(err) + } + + // Overwrite it + w = httptest.NewRecorder() + req, err = http.NewRequest("PUT", "/upload/"+myjson.Filename, strings.NewReader("New file content")) + req.Header.Set("Linx-Delete-Key", myjson.Delete_Key) + mux.ServeHTTP(w, req) + + if w.Code != 200 { + t.Fatal("Status code was not 200, but " + strconv.Itoa(w.Code)) + } + + // Make sure it's the new file + w = httptest.NewRecorder() + req, err = http.NewRequest("GET", "/"+Config.selifPath+myjson.Filename, nil) + mux.ServeHTTP(w, req) + + if w.Code == 404 { + t.Fatal("Status code was 404") + } + + if w.Body.String() != "New file content" { + t.Fatal("File did not contain 'New file content") + } + + Config.forceRandomFilename = oldFRF +} + func TestPutAndSpecificDelete(t *testing.T) { var myjson RespOkJSON diff --git a/static/js/upload.js b/static/js/upload.js index 159bad2..125123c 100644 --- a/static/js/upload.js +++ b/static/js/upload.js @@ -1,51 +1,54 @@ // @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later Dropzone.options.dropzone = { - init: function() { - var dzone = document.getElementById("dzone"); - dzone.style.display = "block"; - }, - addedfile: function(file) { - var upload = document.createElement("div"); - upload.className = "upload"; + init: function() { + var dzone = document.getElementById("dzone"); + dzone.style.display = "block"; + }, + addedfile: function(file) { + var upload = document.createElement("div"); + upload.className = "upload"; - var fileLabel = document.createElement("span"); - fileLabel.innerHTML = file.name; - file.fileLabel = fileLabel; - upload.appendChild(fileLabel); + var fileLabel = document.createElement("span"); + fileLabel.innerHTML = file.name; + file.fileLabel = fileLabel; + upload.appendChild(fileLabel); - var fileActions = document.createElement("div"); - fileActions.className = "right"; - file.fileActions = fileActions; - upload.appendChild(fileActions); + var fileActions = document.createElement("div"); + fileActions.className = "right"; + file.fileActions = fileActions; + upload.appendChild(fileActions); - var cancelAction = document.createElement("span"); - cancelAction.className = "cancel"; - cancelAction.innerHTML = "Cancel"; - cancelAction.addEventListener('click', function(ev) { - this.removeFile(file); - }.bind(this)); - file.cancelActionElement = cancelAction; - fileActions.appendChild(cancelAction); + var cancelAction = document.createElement("span"); + cancelAction.className = "cancel"; + cancelAction.innerHTML = "Cancel"; + cancelAction.addEventListener('click', function(ev) { + this.removeFile(file); + }.bind(this)); + file.cancelActionElement = cancelAction; + fileActions.appendChild(cancelAction); - var progress = document.createElement("span"); - file.progressElement = progress; - fileActions.appendChild(progress); + var progress = document.createElement("span"); + file.progressElement = progress; + fileActions.appendChild(progress); - file.uploadElement = upload; + file.uploadElement = upload; - document.getElementById("uploads").appendChild(upload); - }, - uploadprogress: function(file, p, bytesSent) { - p = parseInt(p); - file.progressElement.innerHTML = p + "%"; - file.uploadElement.setAttribute("style", 'background-image: -webkit-linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%); background-image: -moz-linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%); background-image: -ms-linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%); background-image: -o-linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%); background-image: linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%)'); - }, - sending: function(file, xhr, formData) { - formData.append("randomize", document.getElementById("randomize").checked); - formData.append("expires", document.getElementById("expires").value); - }, - success: function(file, resp) { + document.getElementById("uploads").appendChild(upload); + }, + uploadprogress: function(file, p, bytesSent) { + p = parseInt(p); + file.progressElement.innerHTML = p + "%"; + file.uploadElement.setAttribute("style", 'background-image: -webkit-linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%); background-image: -moz-linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%); background-image: -ms-linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%); background-image: -o-linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%); background-image: linear-gradient(left, #F2F4F7 ' + p + '%, #E2E2E2 ' + p + '%)'); + }, + sending: function(file, xhr, formData) { + var randomize = document.getElementById("randomize"); + if(randomize != null) { + formData.append("randomize", randomize.checked); + } + formData.append("expires", document.getElementById("expires").value); + }, + success: function(file, resp) { file.fileActions.removeChild(file.progressElement); var fileLabelLink = document.createElement("a"); @@ -59,61 +62,61 @@ Dropzone.options.dropzone = { var deleteAction = document.createElement("span"); deleteAction.innerHTML = "Delete"; deleteAction.className = "cancel"; - deleteAction.addEventListener('click', function(ev) { - xhr = new XMLHttpRequest(); - xhr.open("DELETE", resp.url, true); - xhr.setRequestHeader("Linx-Delete-Key", resp.delete_key); - xhr.onreadystatechange = function(file) { - if (xhr.readyState == 4 && xhr.status === 200) { - var text = document.createTextNode("Deleted "); - file.fileLabel.insertBefore(text, file.fileLabelLink); - file.fileLabel.className = "deleted"; - file.fileActions.removeChild(file.cancelActionElement); - } - }.bind(this, file); - xhr.send(); - }); - file.fileActions.removeChild(file.cancelActionElement); - file.cancelActionElement = deleteAction; - file.fileActions.appendChild(deleteAction); - }, - error: function(file, resp, xhrO) { + deleteAction.addEventListener('click', function(ev) { + xhr = new XMLHttpRequest(); + xhr.open("DELETE", resp.url, true); + xhr.setRequestHeader("Linx-Delete-Key", resp.delete_key); + xhr.onreadystatechange = function(file) { + if (xhr.readyState == 4 && xhr.status === 200) { + var text = document.createTextNode("Deleted "); + file.fileLabel.insertBefore(text, file.fileLabelLink); + file.fileLabel.className = "deleted"; + file.fileActions.removeChild(file.cancelActionElement); + } + }.bind(this, file); + xhr.send(); + }); + file.fileActions.removeChild(file.cancelActionElement); + file.cancelActionElement = deleteAction; + file.fileActions.appendChild(deleteAction); + }, + error: function(file, resp, xhrO) { file.fileActions.removeChild(file.cancelActionElement); file.fileActions.removeChild(file.progressElement); - if (file.status === "canceled") { - file.fileLabel.innerHTML = file.name + ": Canceled "; - } - else { - if (resp.error) { - file.fileLabel.innerHTML = file.name + ": " + resp.error; - } - else if (resp.includes("Optional headers with the request

+{% if not forcerandom %}

Randomize the filename
Linx-Randomize: yes

+{% endif %}

Specify a custom deletion key
Linx-Delete-Key: mysecret

@@ -56,30 +58,30 @@ {% if using_auth %} 
$ curl -H "Linx-Api-Key: mysecretkey" -T myphoto.jpg {{ siteurl }}upload/  
-{{ siteurl }}myphoto.jpg
+{{ siteurl }}{% if not forcerandom %}myphoto.jpg{% else %}7z4h4ut.jpg{% endif %} {% else %} 
$ curl -T myphoto.jpg {{ siteurl }}upload/  
-{{ siteurl }}myphoto.jpg
+{{ siteurl }}{% if not forcerandom %}myphoto.jpg{% else %}wtq7pan.jpg{% endif %} {% endif %}

Uploading myphoto.jpg with an expiry of 20 minutes

{% if using_auth %} 
$ curl -H "Linx-Api-Key: mysecretkey" -H "Linx-Expiry: 1200" -T myphoto.jpg {{ siteurl }}upload/
-{{ siteurl }}myphoto.jpg
+{{ siteurl }}{% if not forcerandom %}myphoto.jpg{% else %}jm295snf.jpg{% endif %} {% else %} 
$ curl -H "Linx-Expiry: 1200" -T myphoto.jpg {{ siteurl }}upload/
-{{ siteurl }}myphoto.jpg
+{{ siteurl }}{% if not forcerandom %}myphoto.jpg{% else %}1doym9u2.jpg{% endif %} {% endif %}

Uploading myphoto.jpg with a random filename and getting a json response:

{% if using_auth %} - 
$ curl -H "Linx-Api-Key: mysecretkey" -H "Accept: application/json" -H "Linx-Randomize: yes" -T myphoto.jpg {{ siteurl }}upload/  
+			
$ curl -H "Linx-Api-Key: mysecretkey" -H "Accept: application/json"{% if not forcerandom %} -H "Linx-Randomize: yes"{% endif %} -T myphoto.jpg {{ siteurl }}upload/  
 {"delete_key":"...","expiry":"0","filename":"f34h4iu.jpg","mimetype":"image/jpeg",
 "sha256sum":"...","size":"...","url":"{{ siteurl }}f34h4iu.jpg"}

 			{% else %}
-			
$ curl -H "Accept: application/json" -H "Linx-Randomize: yes" -T myphoto.jpg {{ siteurl }}upload/  
+			
$ curl -H "Accept: application/json"{% if not forcerandom %} -H "Linx-Randomize: yes"{% endif %} -T myphoto.jpg {{ siteurl }}upload/  
 {"delete_key":"...","expiry":"0","filename":"f34h4iu.jpg","mimetype":"image/jpeg",
 "sha256sum":"...","size":"...","url":"{{ siteurl }}f34h4iu.jpg"}

 			{% endif %}
diff --git a/templates/index.html b/templates/index.html
index d423879..2843109 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -17,7 +17,7 @@
- +
diff --git a/upload.go b/upload.go index d46c4d5..6533bfe 100644 --- a/upload.go +++ b/upload.go @@ -222,11 +222,14 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { return upload, FileTooLargeError } - // Determine the appropriate filename, then write to disk + // Determine the appropriate filename barename, extension := barePlusExt(upReq.filename) + randomize := false + // Randomize the "barename" (filename without extension) if needed if upReq.randomBarename || len(barename) == 0 { barename = generateBarename() + randomize = true } var header []byte @@ -259,16 +262,32 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { if merr == nil { if upReq.deleteKey == metad.DeleteKey { fileexists = false + } else if Config.forceRandomFilename == true { + // the file exists + // the delete key doesn't match + // force random filenames is enabled + randomize = true } } + } else if Config.forceRandomFilename == true { + // the file doesn't exist + // force random filenames is enabled + randomize = true + + // set fileexists to true to generate a new barename + fileexists = true } for fileexists { - counter, err := strconv.Atoi(string(barename[len(barename)-1])) - if err != nil { - barename = barename + "1" + if randomize { + barename = generateBarename() } else { - barename = barename[:len(barename)-1] + strconv.Itoa(counter+1) + counter, err := strconv.Atoi(string(barename[len(barename)-1])) + if err != nil { + barename = barename + "1" + } else { + barename = barename[:len(barename)-1] + strconv.Itoa(counter+1) + } } upload.Filename = strings.Join([]string{barename, extension}, ".") From cde964ffe063a613b47b2c47c6a024431ead8f56 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Sat, 26 Jan 2019 02:12:49 -0800 Subject: [PATCH 24/30] Hide filename input when force random is on --- display.go | 17 +++++++++-------- templates/display/bin.html | 2 +- templates/display/story.html | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/display.go b/display.go index 7258904..feb16da 100644 --- a/display.go +++ b/display.go @@ -131,14 +131,15 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) { } err = renderTemplate(tpl, pongo2.Context{ - "mime": metadata.Mimetype, - "filename": fileName, - "size": sizeHuman, - "expiry": expiryHuman, - "expirylist": listExpirationTimes(), - "extra": extra, - "lines": lines, - "files": metadata.ArchiveFiles, + "mime": metadata.Mimetype, + "filename": fileName, + "size": sizeHuman, + "expiry": expiryHuman, + "expirylist": listExpirationTimes(), + "extra": extra, + "forcerandom": Config.forceRandomFilename, + "lines": lines, + "files": metadata.ArchiveFiles, }, r, w) if err != nil { diff --git a/templates/display/bin.html b/templates/display/bin.html index bd029a2..dee64d8 100644 --- a/templates/display/bin.html +++ b/templates/display/bin.html @@ -22,7 +22,7 @@
- . + {% if not forcerandom %}{% endif %}.
diff --git a/templates/display/story.html b/templates/display/story.html index 20e772c..1ef96b1 100644 --- a/templates/display/story.html +++ b/templates/display/story.html @@ -20,7 +20,7 @@
- . + {% if not forcerandom %}{% endif %}.
From f46b61399bade712f0f77eef5d7762450613c4f3 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Sun, 27 Jan 2019 00:32:37 +0000 Subject: [PATCH 25/30] Fix broken page when file is missing (#160) With the localfs backend, it's possible for a file to be removed but its metadata file to remain intact. In this case, viewing the selif URL for that file would return a broken page with two error pages stacked on top of each other. This changes fixes that by replacing the output in that case with a single "Unable to open file." error message. --- fileserve.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fileserve.go b/fileserve.go index a3a249e..8948d45 100644 --- a/fileserve.go +++ b/fileserve.go @@ -38,8 +38,12 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { w.Header().Set("Referrer-Policy", Config.fileReferrerPolicy) _, reader, err := storageBackend.Get(fileName) - if err != nil { - oopsHandler(c, w, r, RespAUTO, err.Error()) + if err == backends.NotFoundErr { + notFoundHandler(c, w, r) + return + } else if err != nil { + oopsHandler(c, w, r, RespAUTO, "Unable to open file.") + return } w.Header().Set("Content-Type", metadata.Mimetype) From 73f127306c15789eb1da4b37b6f2bc49061d0fcf Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Tue, 29 Jan 2019 07:00:08 +0000 Subject: [PATCH 26/30] Improve UI a bit (#161) * Remove right margin from expiration dropdown on index * Use flexbox for bin/story display * Move Paste/Save button after expire dropdown, instead of before --- static/css/linx.css | 33 +++++----- static/js/bin.js | 116 +++++++++++++++++------------------ templates/base.html | 2 +- templates/display/base.html | 4 +- templates/display/bin.html | 23 ++++--- templates/display/story.html | 20 +++--- templates/paste.html | 6 +- 7 files changed, 105 insertions(+), 99 deletions(-) diff --git a/static/css/linx.css b/static/css/linx.css index 82e256a..15abee4 100644 --- a/static/css/linx.css +++ b/static/css/linx.css @@ -70,11 +70,21 @@ body { } #info { + background-color: white; + padding: 5px; +} + +.info-flex { display: flex; flex-wrap: wrap; + align-items: baseline; justify-content: space-between; - background-color: white; - padding: 5px 5px 5px 5px; +} + +.info-actions { + margin-left: 15px; + font-size: 13px; + text-align: right; } #info #extension, @@ -82,10 +92,6 @@ body { width: 40px; } -#info .text-right { - font-size: 13px; -} - #info a { text-decoration: none; color: #556A7F; @@ -246,11 +252,7 @@ body { justify-content: space-between; width: 100%; margin-top: 5px; - font-size:13px; -} - -#choices label:first-child { - margin-right: 15px; + font-size: 13px; } #expiry { @@ -295,14 +297,11 @@ body { } - #info input[type=text] { border: 1px solid #eaeaea; color: #556A7F; - border-radius: 4px 4px 4px 4px; - padding-left: 4px; - padding-right: 4px; - height: 15px; + padding: 2px 4px; + font-family: Arial, Helvetica, sans-serif; } .storygreen { @@ -357,4 +356,4 @@ body { height: 800px; font-size: 13px; } -/* }}} */ \ No newline at end of file +/* }}} */ diff --git a/static/js/bin.js b/static/js/bin.js index 7aed334..11c6c21 100644 --- a/static/js/bin.js +++ b/static/js/bin.js @@ -1,58 +1,58 @@ -// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later - -var navlist = document.getElementById("info").getElementsByClassName("text-right")[0]; - -init(); - -function init() { - var editA = document.createElement('a'); - - editA.setAttribute("href", "#"); - editA.addEventListener('click', function(ev) { - edit(ev); - return false; - }); - editA.innerHTML = "edit"; - - var separator = document.createTextNode(" | "); - navlist.insertBefore(editA, navlist.firstChild); - navlist.insertBefore(separator, navlist.children[1]); - - document.getElementById('save').addEventListener('click', paste); - document.getElementById('wordwrap').addEventListener('click', wrap); -} - -function edit(ev) { - ev.preventDefault(); - - navlist.remove(); - document.getElementById("filename").remove(); - document.getElementById("editform").style.display = "block"; - - var normalcontent = document.getElementById("normal-content"); - normalcontent.removeChild(document.getElementById("normal-code")); - - var editordiv = document.getElementById("inplace-editor"); - editordiv.style.display = "block"; - editordiv.addEventListener('keydown', handleTab); -} - -function paste(ev) { - var editordiv = document.getElementById("inplace-editor"); - document.getElementById("newcontent").value = editordiv.value; - document.forms["reply"].submit(); -} - -function wrap(ev) { - if (document.getElementById("wordwrap").checked) { - document.getElementById("codeb").style.wordWrap = "break-word"; - document.getElementById("codeb").style.whiteSpace = "pre-wrap"; - } - - else { - document.getElementById("codeb").style.wordWrap = "normal"; - document.getElementById("codeb").style.whiteSpace = "pre"; - } -} - -// @license-end +// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later + +var navlist = document.getElementById("info").getElementsByClassName("info-actions")[0]; + +init(); + +function init() { + var editA = document.createElement('a'); + + editA.setAttribute("href", "#"); + editA.addEventListener('click', function(ev) { + edit(ev); + return false; + }); + editA.innerHTML = "edit"; + + var separator = document.createTextNode(" | "); + navlist.insertBefore(editA, navlist.firstChild); + navlist.insertBefore(separator, navlist.children[1]); + + document.getElementById('save').addEventListener('click', paste); + document.getElementById('wordwrap').addEventListener('click', wrap); +} + +function edit(ev) { + ev.preventDefault(); + + navlist.remove(); + document.getElementById("filename").remove(); + document.getElementById("editform").style.display = "block"; + + var normalcontent = document.getElementById("normal-content"); + normalcontent.removeChild(document.getElementById("normal-code")); + + var editordiv = document.getElementById("inplace-editor"); + editordiv.style.display = "block"; + editordiv.addEventListener('keydown', handleTab); +} + +function paste(ev) { + var editordiv = document.getElementById("inplace-editor"); + document.getElementById("newcontent").value = editordiv.value; + document.forms["reply"].submit(); +} + +function wrap(ev) { + if (document.getElementById("wordwrap").checked) { + document.getElementById("codeb").style.wordWrap = "break-word"; + document.getElementById("codeb").style.whiteSpace = "pre-wrap"; + } + + else { + document.getElementById("codeb").style.wordWrap = "normal"; + document.getElementById("codeb").style.whiteSpace = "pre"; + } +} + +// @license-end diff --git a/templates/base.html b/templates/base.html index 5392f8d..d1411d4 100644 --- a/templates/base.html +++ b/templates/base.html @@ -4,7 +4,7 @@ {% block title %}{{ sitename }}{% endblock %} - + {% block head %}{% endblock %} diff --git a/templates/display/base.html b/templates/display/base.html index 011534c..935979f 100644 --- a/templates/display/base.html +++ b/templates/display/base.html @@ -6,12 +6,12 @@ {% block content %} -
+
{{ filename }}
-
+
{% if expiry %} file expires in {{ expiry }} | {% endif %} diff --git a/templates/display/bin.html b/templates/display/bin.html index dee64d8..c3ee97c 100644 --- a/templates/display/bin.html +++ b/templates/display/bin.html @@ -11,24 +11,27 @@ {% block infoleft %}
-
-
- - {% endif %}. +
+
+ + + +
- - {% if not forcerandom %}{% endif %}.
{% endblock %} -{%block infomore %} +{% block infomore %} | {% endblock %} @@ -45,5 +48,5 @@ {% endif %} - + {% endblock %} diff --git a/templates/display/story.html b/templates/display/story.html index 1ef96b1..cb30323 100644 --- a/templates/display/story.html +++ b/templates/display/story.html @@ -9,18 +9,22 @@ {% block infoleft %}
-
-
- - {% endif %}. +
+
+ + + +
- {% if not forcerandom %}{% endif %}.
@@ -38,5 +42,5 @@ - + {% endblock %} diff --git a/templates/paste.html b/templates/paste.html index ab7965f..245b7a3 100644 --- a/templates/paste.html +++ b/templates/paste.html @@ -3,18 +3,18 @@ {% block content %}
-
+
{% if not forcerandom %}{% endif %}.
- +
From 770cb204793a823057c15a53e85b222af6178f34 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Thu, 31 Jan 2019 06:52:43 +0000 Subject: [PATCH 27/30] Add support for conditional requests (#162) This change pulls in some code copied from net/http's fs.go so that we can support If-Match/If-None-Match requests. This will make it easy to put a caching proxy in front of linx-server instances. Request validation will still happen as long as the proxy can contact the origin, so expiration and deletion will still work as expected under normal circumstances. --- fileserve.go | 30 +++--- httputil/LICENSE | 27 +++++ httputil/conditional.go | 218 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 13 deletions(-) create mode 100644 httputil/LICENSE create mode 100644 httputil/conditional.go diff --git a/fileserve.go b/fileserve.go index 8948d45..202e477 100644 --- a/fileserve.go +++ b/fileserve.go @@ -1,14 +1,17 @@ package main import ( + "fmt" "io" "net/http" "net/url" "strconv" "strings" + "time" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" + "github.com/andreimarcu/linx-server/httputil" "github.com/zenazn/goji/web" ) @@ -37,21 +40,22 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Security-Policy", Config.fileContentSecurityPolicy) w.Header().Set("Referrer-Policy", Config.fileReferrerPolicy) - _, reader, err := storageBackend.Get(fileName) - if err == backends.NotFoundErr { - notFoundHandler(c, w, r) - return - } else if err != nil { - oopsHandler(c, w, r, RespAUTO, "Unable to open file.") - return - } - w.Header().Set("Content-Type", metadata.Mimetype) w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10)) - w.Header().Set("Etag", metadata.Sha256sum) - w.Header().Set("Cache-Control", "max-age=0") + w.Header().Set("Etag", fmt.Sprintf("\"%s\"", metadata.Sha256sum)) + w.Header().Set("Cache-Control", "public, no-cache") + + modtime := time.Unix(0, 0) + if done := httputil.CheckPreconditions(w, r, modtime); done == true { + return + } if r.Method != "HEAD" { + _, reader, err := storageBackend.Get(fileName) + if err != nil { + oopsHandler(c, w, r, RespAUTO, "Unable to open file.") + return + } defer reader.Close() if _, err = io.CopyN(w, reader, metadata.Size); err != nil { @@ -77,8 +81,8 @@ func staticHandler(c web.C, w http.ResponseWriter, r *http.Request) { return } - w.Header().Set("Etag", timeStartedStr) - w.Header().Set("Cache-Control", "max-age=86400") + w.Header().Set("Etag", fmt.Sprintf("\"%s\"", timeStartedStr)) + w.Header().Set("Cache-Control", "public, max-age=86400") http.ServeContent(w, r, filePath, timeStarted, file) return } diff --git a/httputil/LICENSE b/httputil/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/httputil/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/httputil/conditional.go b/httputil/conditional.go new file mode 100644 index 0000000..999b1ac --- /dev/null +++ b/httputil/conditional.go @@ -0,0 +1,218 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP file system request handler + +package httputil + +import ( + "net/http" + "net/textproto" + "strings" + "time" +) + +// scanETag determines if a syntactically valid ETag is present at s. If so, +// the ETag and remaining text after consuming ETag is returned. Otherwise, +// it returns "", "". +func scanETag(s string) (etag string, remain string) { + s = textproto.TrimString(s) + start := 0 + if strings.HasPrefix(s, "W/") { + start = 2 + } + if len(s[start:]) < 2 || s[start] != '"' { + return "", "" + } + // ETag is either W/"text" or "text". + // See RFC 7232 2.3. + for i := start + 1; i < len(s); i++ { + c := s[i] + switch { + // Character values allowed in ETags. + case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: + case c == '"': + return s[:i+1], s[i+1:] + default: + return "", "" + } + } + return "", "" +} + +// etagStrongMatch reports whether a and b match using strong ETag comparison. +// Assumes a and b are valid ETags. +func etagStrongMatch(a, b string) bool { + return a == b && a != "" && a[0] == '"' +} + +// etagWeakMatch reports whether a and b match using weak ETag comparison. +// Assumes a and b are valid ETags. +func etagWeakMatch(a, b string) bool { + return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") +} + +// condResult is the result of an HTTP request precondition check. +// See https://tools.ietf.org/html/rfc7232 section 3. +type condResult int + +const ( + condNone condResult = iota + condTrue + condFalse +) + +func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { + im := r.Header.Get("If-Match") + if im == "" { + return condNone + } + for { + im = textproto.TrimString(im) + if len(im) == 0 { + break + } + if im[0] == ',' { + im = im[1:] + continue + } + if im[0] == '*' { + return condTrue + } + etag, remain := scanETag(im) + if etag == "" { + break + } + if etagStrongMatch(etag, w.Header().Get("Etag")) { + return condTrue + } + im = remain + } + + return condFalse +} + +func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { + ius := r.Header.Get("If-Unmodified-Since") + if ius == "" || isZeroTime(modtime) { + return condNone + } + if t, err := http.ParseTime(ius); err == nil { + // The Date-Modified header truncates sub-second precision, so + // use mtime < t+1s instead of mtime <= t to check for unmodified. + if modtime.Before(t.Add(1 * time.Second)) { + return condTrue + } + return condFalse + } + return condNone +} + +func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult { + inm := r.Header.Get("If-None-Match") + if inm == "" { + return condNone + } + buf := inm + for { + buf = textproto.TrimString(buf) + if len(buf) == 0 { + break + } + if buf[0] == ',' { + buf = buf[1:] + } + if buf[0] == '*' { + return condFalse + } + etag, remain := scanETag(buf) + if etag == "" { + break + } + if etagWeakMatch(etag, w.Header().Get("Etag")) { + return condFalse + } + buf = remain + } + return condTrue +} + +func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" || isZeroTime(modtime) { + return condNone + } + t, err := http.ParseTime(ims) + if err != nil { + return condNone + } + // The Date-Modified header truncates sub-second precision, so + // use mtime < t+1s instead of mtime <= t to check for unmodified. + if modtime.Before(t.Add(1 * time.Second)) { + return condFalse + } + return condTrue +} + +var unixEpochTime = time.Unix(0, 0) + +// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). +func isZeroTime(t time.Time) bool { + return t.IsZero() || t.Equal(unixEpochTime) +} + +func setLastModified(w http.ResponseWriter, modtime time.Time) { + if !isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } +} + +func writeNotModified(w http.ResponseWriter) { + // RFC 7232 section 4.1: + // a sender SHOULD NOT generate representation metadata other than the + // above listed fields unless said metadata exists for the purpose of + // guiding cache updates (e.g., Last-Modified might be useful if the + // response does not have an ETag field). + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + if h.Get("Etag") != "" { + delete(h, "Last-Modified") + } + w.WriteHeader(http.StatusNotModified) +} + +// CheckPreconditions evaluates request preconditions and reports whether a precondition +// resulted in sending StatusNotModified or StatusPreconditionFailed. +func CheckPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time) (done bool) { + // This function carefully follows RFC 7232 section 6. + ch := checkIfMatch(w, r) + if ch == condNone { + ch = checkIfUnmodifiedSince(r, modtime) + } + if ch == condFalse { + w.WriteHeader(http.StatusPreconditionFailed) + return true + } + switch checkIfNoneMatch(w, r) { + case condFalse: + if r.Method == "GET" || r.Method == "HEAD" { + writeNotModified(w) + return true + } else { + w.WriteHeader(http.StatusPreconditionFailed) + return true + } + case condNone: + if checkIfModifiedSince(r, modtime) == condFalse { + writeNotModified(w) + return true + } + } + + return false +} From 2c0b2b2e79518b54dbd2ed4ae93919f7fc379221 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Thu, 14 Mar 2019 10:58:17 -0700 Subject: [PATCH 28/30] README reordering --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bd1d9eb..3b46bb6 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,12 @@ allowhotlink = true - ```-nologs``` -- (optionally) disable request logs in stdout - ```-force-random-filename``` -- (optionally) force the use of random filenames +#### Require API Keys for uploads +- ```-authfile path/to/authfile``` -- (optionally) require authorization for upload/delete by providing a newline-separated file of scrypted auth keys +- ```-remoteauthfile path/to/remoteauthfile``` -- (optionally) require authorization for remote uploads by providing a newline-separated file of scrypted auth keys + +A helper utility ```linx-genkey``` is provided which hashes keys to the format required in the auth files. + #### Storage backends The following storage backends are available: @@ -73,12 +79,6 @@ The following storage backends are available: #### Use with fastcgi - ```-fastcgi``` -- serve through fastcgi -#### Require API Keys for uploads -- ```-authfile path/to/authfile``` -- (optionally) require authorization for upload/delete by providing a newline-separated file of scrypted auth keys -- ```-remoteauthfile path/to/remoteauthfile``` -- (optionally) require authorization for remote uploads by providing a newline-separated file of scrypted auth keys - -A helper utility ```linx-genkey``` is provided which hashes keys to the format required in the auth files. - Cleaning up expired files ------------------------- From 8098b7e39e2fea603d1b9fbe4883e858bd5b5f45 Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Tue, 9 Apr 2019 13:28:18 -0700 Subject: [PATCH 29/30] Add PutMetadata function to storage backends (#171) * Add PutMetadata function to storage backends This function is not currently used, but it will be useful for helper scripts that need to regenerate metadata on the fly, especially scripts to migrate between storage backends. In the future, we can also use it to automatically regenerate metadata if it is found to be missing or corrupted. * Add PutMetadata function to storage backend interface and implementations * Rework metadata generation to be more efficient and work better with the PutMetadata function * Add a basic test for metadata generation * Change PutMetadata to take a Metadata type instead It's unlikely that this function is useful if it always regenerates the metadata. Instead, the caller should do that if it needs. --- backends/localfs/localfs.go | 20 ++++++++++-- backends/s3/s3.go | 48 ++++++++++++++++++---------- backends/storage.go | 1 + helpers/helpers.go | 62 +++++++++++++++++++++++-------------- helpers/helpers_test.go | 29 +++++++++++++++++ 5 files changed, 117 insertions(+), 43 deletions(-) create mode 100644 helpers/helpers_test.go diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index 3f6f5ad..47187b6 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -126,11 +126,16 @@ func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey return m, err } + dst.Seek(0 ,0) + m, err = helpers.GenerateMetadata(dst) + if err != nil { + os.Remove(filePath) + return + } + dst.Seek(0 ,0) + m.Expiry = expiry m.DeleteKey = deleteKey - m.Size = bytes - m.Mimetype, _ = helpers.DetectMime(dst) - m.Sha256sum, _ = helpers.Sha256sum(dst) m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, dst) err = b.writeMetadata(key, m) @@ -142,6 +147,15 @@ func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey return } +func (b LocalfsBackend) PutMetadata(key string, m backends.Metadata) (err error) { + err = b.writeMetadata(key, m) + if err != nil { + return + } + + return +} + func (b LocalfsBackend) Size(key string) (int64, error) { fileInfo, err := os.Stat(path.Join(b.filesPath, key)) if err != nil { diff --git a/backends/s3/s3.go b/backends/s3/s3.go index 45067c1..ad040e1 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -18,13 +18,13 @@ import ( type S3Backend struct { bucket string - svc *s3.S3 + svc *s3.S3 } func (b S3Backend) Delete(key string) error { _, err := b.svc.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(b.bucket), - Key: aws.String(key), + Key: aws.String(key), }) if err != nil { return err @@ -35,7 +35,7 @@ func (b S3Backend) Delete(key string) error { func (b S3Backend) Exists(key string) (bool, error) { _, err := b.svc.HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(b.bucket), - Key: aws.String(key), + Key: aws.String(key), }) return err == nil, err } @@ -44,7 +44,7 @@ func (b S3Backend) Head(key string) (metadata backends.Metadata, err error) { var result *s3.HeadObjectOutput result, err = b.svc.HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(b.bucket), - Key: aws.String(key), + Key: aws.String(key), }) if err != nil { if aerr, ok := err.(awserr.Error); ok { @@ -63,7 +63,7 @@ func (b S3Backend) Get(key string) (metadata backends.Metadata, r io.ReadCloser, var result *s3.GetObjectOutput result, err = b.svc.GetObject(&s3.GetObjectInput{ Bucket: aws.String(b.bucket), - Key: aws.String(key), + Key: aws.String(key), }) if err != nil { if aerr, ok := err.(awserr.Error); ok { @@ -81,11 +81,11 @@ func (b S3Backend) Get(key string) (metadata backends.Metadata, r io.ReadCloser, func mapMetadata(m backends.Metadata) map[string]*string { return map[string]*string{ - "Expiry": aws.String(strconv.FormatInt(m.Expiry.Unix(), 10)), + "Expiry": aws.String(strconv.FormatInt(m.Expiry.Unix(), 10)), "Delete_key": aws.String(m.DeleteKey), - "Size": aws.String(strconv.FormatInt(m.Size, 10)), - "Mimetype": aws.String(m.Mimetype), - "Sha256sum": aws.String(m.Sha256sum), + "Size": aws.String(strconv.FormatInt(m.Size, 10)), + "Mimetype": aws.String(m.Mimetype), + "Sha256sum": aws.String(m.Sha256sum), } } @@ -122,19 +122,20 @@ func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey stri return m, err } + m, err = helpers.GenerateMetadata(r) + if err != nil { + return + } m.Expiry = expiry m.DeleteKey = deleteKey - m.Size = bytes - m.Mimetype, _ = helpers.DetectMime(tmpDst) - m.Sha256sum, _ = helpers.Sha256sum(tmpDst) // XXX: we may not be able to write this to AWS easily //m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, tmpDst) uploader := s3manager.NewUploaderWithClient(b.svc) input := &s3manager.UploadInput{ - Bucket: aws.String(b.bucket), - Key: aws.String(key), - Body: tmpDst, + Bucket: aws.String(b.bucket), + Key: aws.String(key), + Body: tmpDst, Metadata: mapMetadata(m), } _, err = uploader.Upload(input) @@ -145,10 +146,24 @@ func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey stri return } +func (b S3Backend) PutMetadata(key string, m backends.Metadata) (err error) { + _, err = b.svc.CopyObject(&s3.CopyObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + CopySource: aws.String("/" + b.bucket + "/" + key), + Metadata: mapMetadata(m), + }) + if err != nil { + return + } + + return +} + func (b S3Backend) Size(key string) (int64, error) { input := &s3.HeadObjectInput{ Bucket: aws.String(b.bucket), - Key: aws.String(key), + Key: aws.String(key), } result, err := b.svc.HeadObject(input) if err != nil { @@ -169,7 +184,6 @@ func (b S3Backend) List() ([]string, error) { return nil, err } - for _, object := range results.Contents { output = append(output, *object.Key) } diff --git a/backends/storage.go b/backends/storage.go index d40a2b9..fdd8cd6 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -12,6 +12,7 @@ type StorageBackend interface { Head(key string) (Metadata, error) Get(key string) (Metadata, io.ReadCloser, error) Put(key string, r io.Reader, expiry time.Time, deleteKey string) (Metadata, error) + PutMetadata(key string, m Metadata) error Size(key string) (int64, error) } diff --git a/helpers/helpers.go b/helpers/helpers.go index aef68ff..f51d998 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -1,49 +1,65 @@ package helpers import ( + "bytes" "encoding/hex" "io" "unicode" + "github.com/andreimarcu/linx-server/backends" "github.com/minio/sha256-simd" "gopkg.in/h2non/filetype.v1" ) -func DetectMime(r io.ReadSeeker) (string, error) { +func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { + // Since we don't have the ability to seek within a file, we can use a + // Buffer in combination with a TeeReader to keep a copy of the bytes + // we read when detecting the file type. These bytes are still needed + // to hash the file and determine its size and cannot be discarded. + var buf bytes.Buffer + teeReader := io.TeeReader(r, &buf) + // Get first 512 bytes for mimetype detection header := make([]byte, 512) - - r.Seek(0, 0) - r.Read(header) - r.Seek(0, 0) - - kind, err := filetype.Match(header) + _, err = teeReader.Read(header) if err != nil { - return "application/octet-stream", err - } else if kind.MIME.Value != "" { - return kind.MIME.Value, nil + return } - // Check if the file seems anything like text - if printable(header) { - return "text/plain", nil + // Create a Hash and a MultiReader that includes the Buffer we created + // above along with the original Reader, which will have the rest of + // the file. + hasher := sha256.New() + multiReader := io.MultiReader(&buf, r) + + // Copy everything into the Hash, then use the number of bytes written + // as the file size. + var readLen int64 + readLen, err = io.Copy(hasher, multiReader) + if err != nil { + return } else { - return "application/octet-stream", nil + m.Size += readLen } -} -func Sha256sum(r io.ReadSeeker) (string, error) { - hasher := sha256.New() + // Get the hex-encoded string version of the Hash checksum + m.Sha256sum = hex.EncodeToString(hasher.Sum(nil)) - r.Seek(0, 0) - _, err := io.Copy(hasher, r) + // Use the bytes we extracted earlier and attempt to determine the file + // type + kind, err := filetype.Match(header) if err != nil { - return "", err + m.Mimetype = "application/octet-stream" + return m, err + } else if kind.MIME.Value != "" { + m.Mimetype = kind.MIME.Value + } else if printable(header) { + m.Mimetype = "text/plain" + } else { + m.Mimetype = "application/octet-stream" } - r.Seek(0, 0) - - return hex.EncodeToString(hasher.Sum(nil)), nil + return } func printable(data []byte) bool { diff --git a/helpers/helpers_test.go b/helpers/helpers_test.go new file mode 100644 index 0000000..800d0d2 --- /dev/null +++ b/helpers/helpers_test.go @@ -0,0 +1,29 @@ +package helpers + +import ( + "strings" + "testing" +) + +func TestGenerateMetadata(t *testing.T) { + r := strings.NewReader("This is my test content") + m, err := GenerateMetadata(r) + if err != nil { + t.Fatal(err) + } + + expectedSha256sum := "966152d20a77e739716a625373ee15af16e8f4aec631a329a27da41c204b0171" + if m.Sha256sum != expectedSha256sum { + t.Fatalf("Sha256sum was %q instead of expected value of %q", m.Sha256sum, expectedSha256sum) + } + + expectedMimetype := "text/plain" + if m.Mimetype != expectedMimetype { + t.Fatalf("Mimetype was %q instead of expected value of %q", m.Mimetype, expectedMimetype) + } + + expectedSize := int64(23) + if m.Size != expectedSize { + t.Fatalf("Size was %d instead of expected value of %d", m.Size, expectedSize) + } +} From 872340e0dc6f76fe2997520d276207ff8cd6da1c Mon Sep 17 00:00:00 2001 From: mutantmonkey Date: Thu, 9 May 2019 16:13:58 +0000 Subject: [PATCH 30/30] Fix PutMetadata with S3 backend (#176) It turns out that the S3 API expects the additional `MetadataDirective: REPLACE` option in order to update metadata. If this is not provided, then metadata will simply be copied from the source object, which is not what we wanted. --- backends/s3/s3.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backends/s3/s3.go b/backends/s3/s3.go index ad040e1..e39786a 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -148,10 +148,11 @@ func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey stri func (b S3Backend) PutMetadata(key string, m backends.Metadata) (err error) { _, err = b.svc.CopyObject(&s3.CopyObjectInput{ - Bucket: aws.String(b.bucket), - Key: aws.String(key), - CopySource: aws.String("/" + b.bucket + "/" + key), - Metadata: mapMetadata(m), + Bucket: aws.String(b.bucket), + Key: aws.String(key), + CopySource: aws.String("/" + b.bucket + "/" + key), + Metadata: mapMetadata(m), + MetadataDirective: aws.String("REPLACE"), }) if err != nil { return