Browse Source

Merge pull request #124 from mutantmonkey/cleanup_tool

Add linx-cleanup tool
pull/98/merge
Andrei Marcu 8 years ago
committed by GitHub
parent
commit
ceea32de6b
  1. 1
      .gitignore
  2. 2
      Dockerfile
  3. 17
      README.md
  4. 15
      backends/localfs/localfs.go
  5. 23
      backends/meta.go
  6. 73
      backends/metajson/metajson.go
  7. 5
      backends/storage.go
  8. 2
      delete.go
  9. 3
      display.go
  10. 23
      expiry.go
  11. 13
      expiry/expiry.go
  12. 5
      fileserve.go
  13. 46
      linx-cleanup/cleanup.go
  14. 75
      meta.go
  15. 7
      server.go
  16. 3
      torrent.go
  17. 20
      upload.go

1
.gitignore

@ -29,3 +29,4 @@ _testmain.go
linx-server
files/
meta/
linx-cleanup

2
Dockerfile

@ -1,7 +1,7 @@
FROM golang:alpine
RUN set -ex \
&& apk add --no-cache --virtual .build-deps git mercurial \
&& apk add --no-cache --virtual .build-deps git \
&& go get github.com/andreimarcu/linx-server \
&& apk del .build-deps

17
README.md

@ -71,6 +71,23 @@ allowhotlink = true
A helper utility ```linx-genkey``` is provided which hashes keys to the format required in the auth files.
Cleaning up expired files
-------------------------
When files expire, access is disabled immediately, but the files and metadata
will persist on disk until someone attempts to access them. If you'd like to
automatically clean up files that have expired, you can use the included
`linx-cleanup` utility. To run it automatically, use a cronjob or similar type
of scheduled task.
You should be careful to ensure that only one instance of `linx-client` runs at
a time to avoid unexpected behavior. It does not implement any type of locking.
#### Options
- ```-filespath files/``` -- Path to stored uploads (default is files/)
- ```-metapath meta/``` -- Path to stored information about uploads (default is meta/)
- ```-nologs``` -- (optionally) disable deletion logs in stdout
Deployment
----------
Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/).

15
backends/localfs/localfs.go

@ -65,6 +65,21 @@ func (b LocalfsBackend) Size(key string) (int64, error) {
return fileInfo.Size(), nil
}
func (b LocalfsBackend) List() ([]string, error) {
var output []string
files, err := ioutil.ReadDir(b.basePath)
if err != nil {
return nil, err
}
for _, file := range files {
output = append(output, file.Name())
}
return output, nil
}
func NewLocalfsBackend(basePath string) LocalfsBackend {
return LocalfsBackend{basePath: basePath}
}

23
backends/meta.go

@ -0,0 +1,23 @@
package backends
import (
"errors"
"time"
)
type MetaBackend interface {
Get(key string) (Metadata, error)
Put(key string, metadata *Metadata) error
}
type Metadata struct {
DeleteKey string
Sha256sum string
Mimetype string
Size int64
Expiry time.Time
ArchiveFiles []string
ShortURL string
}
var BadMetadata = errors.New("Corrupted metadata.")

73
backends/metajson/metajson.go

@ -0,0 +1,73 @@
package metajson
import (
"bytes"
"encoding/json"
"time"
"github.com/andreimarcu/linx-server/backends"
)
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
ShortURL string `json:"short_url"`
}
type MetaJSONBackend struct {
storage backends.MetaStorageBackend
}
func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error {
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
mjson.ShortURL = metadata.ShortURL
byt, err := json.Marshal(mjson)
if err != nil {
return err
}
if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil {
return err
}
return nil
}
func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) {
b, err := m.storage.Get(key)
if err != nil {
return metadata, backends.BadMetadata
}
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil {
return metadata, backends.BadMetadata
}
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
metadata.ShortURL = mjson.ShortURL
return
}
func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend {
return MetaJSONBackend{storage: storage}
}

5
backends/backends.go → backends/storage.go

@ -21,3 +21,8 @@ type StorageBackend interface {
ServeFile(key string, w http.ResponseWriter, r *http.Request)
Size(key string) (int64, error)
}
type MetaStorageBackend interface {
StorageBackend
List() ([]string, error)
}

2
delete.go

@ -28,7 +28,7 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if metadata.DeleteKey == requestKey {
fileDelErr := fileBackend.Delete(filename)
metaDelErr := metaBackend.Delete(filename)
metaDelErr := metaStorageBackend.Delete(filename)
if (fileDelErr != nil) || (metaDelErr != nil) {
oopsHandler(c, w, r, RespPLAIN, "Could not delete")

3
display.go

@ -8,6 +8,7 @@ import (
"strings"
"time"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dustin/go-humanize"
"github.com/flosch/pongo2"
"github.com/microcosm-cc/bluemonday"
@ -32,7 +33,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
return
}
var expiryHuman string
if metadata.Expiry != neverExpire {
if metadata.Expiry != expiry.NeverExpire {
expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "")
}
sizeHuman := humanize.Bytes(uint64(metadata.Size))

23
expiry.go

@ -3,6 +3,7 @@ package main
import (
"time"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dustin/go-humanize"
)
@ -21,14 +22,6 @@ type ExpirationTime struct {
Human string
}
var neverExpire = time.Unix(0, 0)
// Determine if a file with expiry set to "ts" has expired yet
func isTsExpired(ts time.Time) bool {
now := time.Now()
return ts != neverExpire && now.After(ts)
}
// Determine if the given filename is expired
func isFileExpired(filename string) (bool, error) {
metadata, err := metadataRead(filename)
@ -36,7 +29,7 @@ func isFileExpired(filename string) (bool, error) {
return false, err
}
return isTsExpired(metadata.Expiry), nil
return expiry.IsTsExpired(metadata.Expiry), nil
}
// Return a list of expiration times and their humanized versions
@ -45,16 +38,16 @@ func listExpirationTimes() []ExpirationTime {
actualExpiryInList := false
var expiryList []ExpirationTime
for _, expiry := range defaultExpiryList {
if Config.maxExpiry == 0 || expiry <= Config.maxExpiry {
if expiry == Config.maxExpiry {
for _, expiryEntry := range defaultExpiryList {
if Config.maxExpiry == 0 || expiryEntry <= Config.maxExpiry {
if expiryEntry == Config.maxExpiry {
actualExpiryInList = true
}
duration := time.Duration(expiry) * time.Second
duration := time.Duration(expiryEntry) * time.Second
expiryList = append(expiryList, ExpirationTime{
expiry,
humanize.RelTime(epoch, epoch.Add(duration), "", ""),
Seconds: expiryEntry,
Human: humanize.RelTime(epoch, epoch.Add(duration), "", ""),
})
}
}

13
expiry/expiry.go

@ -0,0 +1,13 @@
package expiry
import (
"time"
)
var NeverExpire = time.Unix(0, 0)
// Determine if a file with expiry set to "ts" has expired yet
func IsTsExpired(ts time.Time) bool {
now := time.Now()
return ts != NeverExpire && now.After(ts)
}

5
fileserve.go

@ -5,6 +5,7 @@ import (
"net/url"
"strings"
"github.com/andreimarcu/linx-server/backends"
"github.com/zenazn/goji/web"
)
@ -15,7 +16,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if err == NotFoundErr {
notFoundHandler(c, w, r)
return
} else if err == BadMetadata {
} else if err == backends.BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
}
@ -72,7 +73,7 @@ func checkFile(filename string) error {
if expired {
fileBackend.Delete(filename)
metaBackend.Delete(filename)
metaStorageBackend.Delete(filename)
return NotFoundErr
}

46
linx-cleanup/cleanup.go

@ -0,0 +1,46 @@
package main
import (
"flag"
"log"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/backends/metajson"
"github.com/andreimarcu/linx-server/expiry"
)
func main() {
var filesDir string
var metaDir string
var noLogs bool
flag.StringVar(&filesDir, "filespath", "files/",
"path to files directory")
flag.StringVar(&metaDir, "metapath", "meta/",
"path to metadata directory")
flag.BoolVar(&noLogs, "nologs", false,
"don't log deleted files")
flag.Parse()
metaStorageBackend := localfs.NewLocalfsBackend(metaDir)
metaBackend := metajson.NewMetaJSONBackend(metaStorageBackend)
fileBackend := localfs.NewLocalfsBackend(filesDir)
files, err := metaStorageBackend.List()
if err != nil {
panic(err)
}
for _, filename := range files {
metadata, err := metaBackend.Get(filename)
if err != nil {
log.Printf("Failed to find metadata for %s", filename)
}
if expiry.IsTsExpired(metadata.Expiry) {
log.Printf("Delete %s", filename)
fileBackend.Delete(filename)
metaStorageBackend.Delete(filename)
}
}
}

75
meta.go

@ -3,46 +3,25 @@ package main
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"sort"
"time"
"unicode"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dchest/uniuri"
"gopkg.in/h2non/filetype.v1"
)
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
ShortURL string `json:"short_url"`
}
type Metadata struct {
DeleteKey string
Sha256sum string
Mimetype string
Size int64
Expiry time.Time
ArchiveFiles []string
ShortURL string
}
var NotFoundErr = errors.New("File not found.")
var BadMetadata = errors.New("Corrupted metadata.")
func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) {
file, err := fileBackend.Open(fName)
if err != nil {
return
@ -145,59 +124,23 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e
return
}
func metadataWrite(filename string, metadata *Metadata) error {
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
mjson.ShortURL = metadata.ShortURL
byt, err := json.Marshal(mjson)
if err != nil {
return err
}
if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil {
return err
}
return nil
func metadataWrite(filename string, metadata *backends.Metadata) error {
return metaBackend.Put(filename, metadata)
}
func metadataRead(filename string) (metadata Metadata, err error) {
b, err := metaBackend.Get(filename)
func metadataRead(filename string) (metadata backends.Metadata, err error) {
metadata, err = metaBackend.Get(filename)
if err != nil {
// Metadata does not exist, generate one
newMData, err := generateMetadata(filename, neverExpire, "")
newMData, err := generateMetadata(filename, expiry.NeverExpire, "")
if err != nil {
return metadata, err
}
metadataWrite(filename, &newMData)
b, err = metaBackend.Get(filename)
if err != nil {
return metadata, BadMetadata
}
}
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil {
return metadata, BadMetadata
metadata, err = metaBackend.Get(filename)
}
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
metadata.ShortURL = mjson.ShortURL
return
}

7
server.go

@ -16,6 +16,7 @@ import (
"github.com/GeertJohan/go.rice"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/backends/metajson"
"github.com/flosch/pongo2"
"github.com/vharitonsky/iniflags"
"github.com/zenazn/goji/graceful"
@ -65,7 +66,8 @@ var staticBox *rice.Box
var timeStarted time.Time
var timeStartedStr string
var remoteAuthKeys []string
var metaBackend backends.StorageBackend
var metaStorageBackend backends.MetaStorageBackend
var metaBackend backends.MetaBackend
var fileBackend backends.StorageBackend
func setup() *web.Mux {
@ -124,7 +126,8 @@ func setup() *web.Mux {
Config.sitePath = "/"
}
metaBackend = localfs.NewLocalfsBackend(Config.metaDir)
metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir)
metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend)
fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
// Template setup

3
torrent.go

@ -8,6 +8,7 @@ import (
"net/http"
"time"
"github.com/andreimarcu/linx-server/backends"
"github.com/zeebo/bencode"
"github.com/zenazn/goji/web"
)
@ -74,7 +75,7 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if err == NotFoundErr {
notFoundHandler(c, w, r)
return
} else if err == BadMetadata {
} else if err == backends.BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return
}

20
upload.go

@ -15,6 +15,8 @@ import (
"strings"
"time"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dchest/uniuri"
"github.com/zenazn/goji/web"
"gopkg.in/h2non/filetype.v1"
@ -41,7 +43,7 @@ type UploadRequest struct {
// Metadata associated with a file as it would actually be stored
type Upload struct {
Filename string // Final filename on disk
Metadata Metadata
Metadata backends.Metadata
}
func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@ -258,11 +260,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
}
// Get the rest of the metadata needed for storage
var expiry time.Time
var fileExpiry time.Time
if upReq.expiry == 0 {
expiry = neverExpire
fileExpiry = expiry.NeverExpire
} else {
expiry = time.Now().Add(upReq.expiry)
fileExpiry = time.Now().Add(upReq.expiry)
}
bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
@ -273,7 +275,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
return upload, errors.New("File too large")
}
upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey)
if err != nil {
fileBackend.Delete(upload.Filename)
return
@ -342,14 +344,14 @@ func parseExpiry(expStr string) time.Duration {
if expStr == "" {
return time.Duration(Config.maxExpiry) * time.Second
} else {
expiry, err := strconv.ParseUint(expStr, 10, 64)
fileExpiry, err := strconv.ParseUint(expStr, 10, 64)
if err != nil {
return time.Duration(Config.maxExpiry) * time.Second
} else {
if Config.maxExpiry > 0 && (expiry > Config.maxExpiry || expiry == 0) {
expiry = Config.maxExpiry
if Config.maxExpiry > 0 && (fileExpiry > Config.maxExpiry || fileExpiry == 0) {
fileExpiry = Config.maxExpiry
}
return time.Duration(expiry) * time.Second
return time.Duration(fileExpiry) * time.Second
}
}
}
Loading…
Cancel
Save