Browse Source

Merge pull request #124 from mutantmonkey/cleanup_tool

Add linx-cleanup tool
pull/98/merge
Andrei Marcu 8 years ago
committed by GitHub
parent
commit
ceea32de6b
  1. 1
      .gitignore
  2. 2
      Dockerfile
  3. 17
      README.md
  4. 15
      backends/localfs/localfs.go
  5. 23
      backends/meta.go
  6. 73
      backends/metajson/metajson.go
  7. 5
      backends/storage.go
  8. 2
      delete.go
  9. 3
      display.go
  10. 23
      expiry.go
  11. 13
      expiry/expiry.go
  12. 5
      fileserve.go
  13. 46
      linx-cleanup/cleanup.go
  14. 75
      meta.go
  15. 7
      server.go
  16. 3
      torrent.go
  17. 20
      upload.go

1
.gitignore

@ -29,3 +29,4 @@ _testmain.go
linx-server linx-server
files/ files/
meta/ meta/
linx-cleanup

2
Dockerfile

@ -1,7 +1,7 @@
FROM golang:alpine FROM golang:alpine
RUN set -ex \ RUN set -ex \
&& apk add --no-cache --virtual .build-deps git mercurial \
&& apk add --no-cache --virtual .build-deps git \
&& go get github.com/andreimarcu/linx-server \ && go get github.com/andreimarcu/linx-server \
&& apk del .build-deps && apk del .build-deps

17
README.md

@ -71,6 +71,23 @@ allowhotlink = true
A helper utility ```linx-genkey``` is provided which hashes keys to the format required in the auth files. A helper utility ```linx-genkey``` is provided which hashes keys to the format required in the auth files.
Cleaning up expired files
-------------------------
When files expire, access is disabled immediately, but the files and metadata
will persist on disk until someone attempts to access them. If you'd like to
automatically clean up files that have expired, you can use the included
`linx-cleanup` utility. To run it automatically, use a cronjob or similar type
of scheduled task.
You should be careful to ensure that only one instance of `linx-client` runs at
a time to avoid unexpected behavior. It does not implement any type of locking.
#### Options
- ```-filespath files/``` -- Path to stored uploads (default is files/)
- ```-metapath meta/``` -- Path to stored information about uploads (default is meta/)
- ```-nologs``` -- (optionally) disable deletion logs in stdout
Deployment Deployment
---------- ----------
Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/). Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/).

15
backends/localfs/localfs.go

@ -65,6 +65,21 @@ func (b LocalfsBackend) Size(key string) (int64, error) {
return fileInfo.Size(), nil return fileInfo.Size(), nil
} }
func (b LocalfsBackend) List() ([]string, error) {
var output []string
files, err := ioutil.ReadDir(b.basePath)
if err != nil {
return nil, err
}
for _, file := range files {
output = append(output, file.Name())
}
return output, nil
}
func NewLocalfsBackend(basePath string) LocalfsBackend { func NewLocalfsBackend(basePath string) LocalfsBackend {
return LocalfsBackend{basePath: basePath} return LocalfsBackend{basePath: basePath}
} }

23
backends/meta.go

@ -0,0 +1,23 @@
package backends
import (
"errors"
"time"
)
type MetaBackend interface {
Get(key string) (Metadata, error)
Put(key string, metadata *Metadata) error
}
type Metadata struct {
DeleteKey string
Sha256sum string
Mimetype string
Size int64
Expiry time.Time
ArchiveFiles []string
ShortURL string
}
var BadMetadata = errors.New("Corrupted metadata.")

73
backends/metajson/metajson.go

@ -0,0 +1,73 @@
package metajson
import (
"bytes"
"encoding/json"
"time"
"github.com/andreimarcu/linx-server/backends"
)
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
ShortURL string `json:"short_url"`
}
type MetaJSONBackend struct {
storage backends.MetaStorageBackend
}
func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error {
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
mjson.ShortURL = metadata.ShortURL
byt, err := json.Marshal(mjson)
if err != nil {
return err
}
if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil {
return err
}
return nil
}
func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) {
b, err := m.storage.Get(key)
if err != nil {
return metadata, backends.BadMetadata
}
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil {
return metadata, backends.BadMetadata
}
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
metadata.ShortURL = mjson.ShortURL
return
}
func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend {
return MetaJSONBackend{storage: storage}
}

5
backends/backends.go → backends/storage.go

@ -21,3 +21,8 @@ type StorageBackend interface {
ServeFile(key string, w http.ResponseWriter, r *http.Request) ServeFile(key string, w http.ResponseWriter, r *http.Request)
Size(key string) (int64, error) Size(key string) (int64, error)
} }
type MetaStorageBackend interface {
StorageBackend
List() ([]string, error)
}

2
delete.go

@ -28,7 +28,7 @@ func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if metadata.DeleteKey == requestKey { if metadata.DeleteKey == requestKey {
fileDelErr := fileBackend.Delete(filename) fileDelErr := fileBackend.Delete(filename)
metaDelErr := metaBackend.Delete(filename)
metaDelErr := metaStorageBackend.Delete(filename)
if (fileDelErr != nil) || (metaDelErr != nil) { if (fileDelErr != nil) || (metaDelErr != nil) {
oopsHandler(c, w, r, RespPLAIN, "Could not delete") oopsHandler(c, w, r, RespPLAIN, "Could not delete")

3
display.go

@ -8,6 +8,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"github.com/flosch/pongo2" "github.com/flosch/pongo2"
"github.com/microcosm-cc/bluemonday" "github.com/microcosm-cc/bluemonday"
@ -32,7 +33,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request) {
return return
} }
var expiryHuman string var expiryHuman string
if metadata.Expiry != neverExpire {
if metadata.Expiry != expiry.NeverExpire {
expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "") expiryHuman = humanize.RelTime(time.Now(), metadata.Expiry, "", "")
} }
sizeHuman := humanize.Bytes(uint64(metadata.Size)) sizeHuman := humanize.Bytes(uint64(metadata.Size))

23
expiry.go

@ -3,6 +3,7 @@ package main
import ( import (
"time" "time"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
) )
@ -21,14 +22,6 @@ type ExpirationTime struct {
Human string Human string
} }
var neverExpire = time.Unix(0, 0)
// Determine if a file with expiry set to "ts" has expired yet
func isTsExpired(ts time.Time) bool {
now := time.Now()
return ts != neverExpire && now.After(ts)
}
// Determine if the given filename is expired // Determine if the given filename is expired
func isFileExpired(filename string) (bool, error) { func isFileExpired(filename string) (bool, error) {
metadata, err := metadataRead(filename) metadata, err := metadataRead(filename)
@ -36,7 +29,7 @@ func isFileExpired(filename string) (bool, error) {
return false, err return false, err
} }
return isTsExpired(metadata.Expiry), nil
return expiry.IsTsExpired(metadata.Expiry), nil
} }
// Return a list of expiration times and their humanized versions // Return a list of expiration times and their humanized versions
@ -45,16 +38,16 @@ func listExpirationTimes() []ExpirationTime {
actualExpiryInList := false actualExpiryInList := false
var expiryList []ExpirationTime var expiryList []ExpirationTime
for _, expiry := range defaultExpiryList {
if Config.maxExpiry == 0 || expiry <= Config.maxExpiry {
if expiry == Config.maxExpiry {
for _, expiryEntry := range defaultExpiryList {
if Config.maxExpiry == 0 || expiryEntry <= Config.maxExpiry {
if expiryEntry == Config.maxExpiry {
actualExpiryInList = true actualExpiryInList = true
} }
duration := time.Duration(expiry) * time.Second
duration := time.Duration(expiryEntry) * time.Second
expiryList = append(expiryList, ExpirationTime{ expiryList = append(expiryList, ExpirationTime{
expiry,
humanize.RelTime(epoch, epoch.Add(duration), "", ""),
Seconds: expiryEntry,
Human: humanize.RelTime(epoch, epoch.Add(duration), "", ""),
}) })
} }
} }

13
expiry/expiry.go

@ -0,0 +1,13 @@
package expiry
import (
"time"
)
var NeverExpire = time.Unix(0, 0)
// Determine if a file with expiry set to "ts" has expired yet
func IsTsExpired(ts time.Time) bool {
now := time.Now()
return ts != NeverExpire && now.After(ts)
}

5
fileserve.go

@ -5,6 +5,7 @@ import (
"net/url" "net/url"
"strings" "strings"
"github.com/andreimarcu/linx-server/backends"
"github.com/zenazn/goji/web" "github.com/zenazn/goji/web"
) )
@ -15,7 +16,7 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if err == NotFoundErr { if err == NotFoundErr {
notFoundHandler(c, w, r) notFoundHandler(c, w, r)
return return
} else if err == BadMetadata {
} else if err == backends.BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return return
} }
@ -72,7 +73,7 @@ func checkFile(filename string) error {
if expired { if expired {
fileBackend.Delete(filename) fileBackend.Delete(filename)
metaBackend.Delete(filename)
metaStorageBackend.Delete(filename)
return NotFoundErr return NotFoundErr
} }

46
linx-cleanup/cleanup.go

@ -0,0 +1,46 @@
package main
import (
"flag"
"log"
"github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/backends/metajson"
"github.com/andreimarcu/linx-server/expiry"
)
func main() {
var filesDir string
var metaDir string
var noLogs bool
flag.StringVar(&filesDir, "filespath", "files/",
"path to files directory")
flag.StringVar(&metaDir, "metapath", "meta/",
"path to metadata directory")
flag.BoolVar(&noLogs, "nologs", false,
"don't log deleted files")
flag.Parse()
metaStorageBackend := localfs.NewLocalfsBackend(metaDir)
metaBackend := metajson.NewMetaJSONBackend(metaStorageBackend)
fileBackend := localfs.NewLocalfsBackend(filesDir)
files, err := metaStorageBackend.List()
if err != nil {
panic(err)
}
for _, filename := range files {
metadata, err := metaBackend.Get(filename)
if err != nil {
log.Printf("Failed to find metadata for %s", filename)
}
if expiry.IsTsExpired(metadata.Expiry) {
log.Printf("Delete %s", filename)
fileBackend.Delete(filename)
metaStorageBackend.Delete(filename)
}
}
}

75
meta.go

@ -3,46 +3,25 @@ package main
import ( import (
"archive/tar" "archive/tar"
"archive/zip" "archive/zip"
"bytes"
"compress/bzip2" "compress/bzip2"
"compress/gzip" "compress/gzip"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/json"
"errors" "errors"
"io" "io"
"sort" "sort"
"time" "time"
"unicode" "unicode"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dchest/uniuri" "github.com/dchest/uniuri"
"gopkg.in/h2non/filetype.v1" "gopkg.in/h2non/filetype.v1"
) )
type MetadataJSON struct {
DeleteKey string `json:"delete_key"`
Sha256sum string `json:"sha256sum"`
Mimetype string `json:"mimetype"`
Size int64 `json:"size"`
Expiry int64 `json:"expiry"`
ArchiveFiles []string `json:"archive_files,omitempty"`
ShortURL string `json:"short_url"`
}
type Metadata struct {
DeleteKey string
Sha256sum string
Mimetype string
Size int64
Expiry time.Time
ArchiveFiles []string
ShortURL string
}
var NotFoundErr = errors.New("File not found.") var NotFoundErr = errors.New("File not found.")
var BadMetadata = errors.New("Corrupted metadata.")
func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
func generateMetadata(fName string, exp time.Time, delKey string) (m backends.Metadata, err error) {
file, err := fileBackend.Open(fName) file, err := fileBackend.Open(fName)
if err != nil { if err != nil {
return return
@ -145,59 +124,23 @@ func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, e
return return
} }
func metadataWrite(filename string, metadata *Metadata) error {
mjson := MetadataJSON{}
mjson.DeleteKey = metadata.DeleteKey
mjson.Mimetype = metadata.Mimetype
mjson.ArchiveFiles = metadata.ArchiveFiles
mjson.Sha256sum = metadata.Sha256sum
mjson.Expiry = metadata.Expiry.Unix()
mjson.Size = metadata.Size
mjson.ShortURL = metadata.ShortURL
byt, err := json.Marshal(mjson)
if err != nil {
return err
}
if _, err := metaBackend.Put(filename, bytes.NewBuffer(byt)); err != nil {
return err
}
return nil
func metadataWrite(filename string, metadata *backends.Metadata) error {
return metaBackend.Put(filename, metadata)
} }
func metadataRead(filename string) (metadata Metadata, err error) {
b, err := metaBackend.Get(filename)
func metadataRead(filename string) (metadata backends.Metadata, err error) {
metadata, err = metaBackend.Get(filename)
if err != nil { if err != nil {
// Metadata does not exist, generate one // Metadata does not exist, generate one
newMData, err := generateMetadata(filename, neverExpire, "")
newMData, err := generateMetadata(filename, expiry.NeverExpire, "")
if err != nil { if err != nil {
return metadata, err return metadata, err
} }
metadataWrite(filename, &newMData) metadataWrite(filename, &newMData)
b, err = metaBackend.Get(filename)
if err != nil {
return metadata, BadMetadata
}
}
mjson := MetadataJSON{}
err = json.Unmarshal(b, &mjson)
if err != nil {
return metadata, BadMetadata
metadata, err = metaBackend.Get(filename)
} }
metadata.DeleteKey = mjson.DeleteKey
metadata.Mimetype = mjson.Mimetype
metadata.ArchiveFiles = mjson.ArchiveFiles
metadata.Sha256sum = mjson.Sha256sum
metadata.Expiry = time.Unix(mjson.Expiry, 0)
metadata.Size = mjson.Size
metadata.ShortURL = mjson.ShortURL
return return
} }

7
server.go

@ -16,6 +16,7 @@ import (
"github.com/GeertJohan/go.rice" "github.com/GeertJohan/go.rice"
"github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/backends/localfs" "github.com/andreimarcu/linx-server/backends/localfs"
"github.com/andreimarcu/linx-server/backends/metajson"
"github.com/flosch/pongo2" "github.com/flosch/pongo2"
"github.com/vharitonsky/iniflags" "github.com/vharitonsky/iniflags"
"github.com/zenazn/goji/graceful" "github.com/zenazn/goji/graceful"
@ -65,7 +66,8 @@ var staticBox *rice.Box
var timeStarted time.Time var timeStarted time.Time
var timeStartedStr string var timeStartedStr string
var remoteAuthKeys []string var remoteAuthKeys []string
var metaBackend backends.StorageBackend
var metaStorageBackend backends.MetaStorageBackend
var metaBackend backends.MetaBackend
var fileBackend backends.StorageBackend var fileBackend backends.StorageBackend
func setup() *web.Mux { func setup() *web.Mux {
@ -124,7 +126,8 @@ func setup() *web.Mux {
Config.sitePath = "/" Config.sitePath = "/"
} }
metaBackend = localfs.NewLocalfsBackend(Config.metaDir)
metaStorageBackend = localfs.NewLocalfsBackend(Config.metaDir)
metaBackend = metajson.NewMetaJSONBackend(metaStorageBackend)
fileBackend = localfs.NewLocalfsBackend(Config.filesDir) fileBackend = localfs.NewLocalfsBackend(Config.filesDir)
// Template setup // Template setup

3
torrent.go

@ -8,6 +8,7 @@ import (
"net/http" "net/http"
"time" "time"
"github.com/andreimarcu/linx-server/backends"
"github.com/zeebo/bencode" "github.com/zeebo/bencode"
"github.com/zenazn/goji/web" "github.com/zenazn/goji/web"
) )
@ -74,7 +75,7 @@ func fileTorrentHandler(c web.C, w http.ResponseWriter, r *http.Request) {
if err == NotFoundErr { if err == NotFoundErr {
notFoundHandler(c, w, r) notFoundHandler(c, w, r)
return return
} else if err == BadMetadata {
} else if err == backends.BadMetadata {
oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.") oopsHandler(c, w, r, RespAUTO, "Corrupt metadata.")
return return
} }

20
upload.go

@ -15,6 +15,8 @@ import (
"strings" "strings"
"time" "time"
"github.com/andreimarcu/linx-server/backends"
"github.com/andreimarcu/linx-server/expiry"
"github.com/dchest/uniuri" "github.com/dchest/uniuri"
"github.com/zenazn/goji/web" "github.com/zenazn/goji/web"
"gopkg.in/h2non/filetype.v1" "gopkg.in/h2non/filetype.v1"
@ -41,7 +43,7 @@ type UploadRequest struct {
// Metadata associated with a file as it would actually be stored // Metadata associated with a file as it would actually be stored
type Upload struct { type Upload struct {
Filename string // Final filename on disk Filename string // Final filename on disk
Metadata Metadata
Metadata backends.Metadata
} }
func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) {
@ -258,11 +260,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
} }
// Get the rest of the metadata needed for storage // Get the rest of the metadata needed for storage
var expiry time.Time
var fileExpiry time.Time
if upReq.expiry == 0 { if upReq.expiry == 0 {
expiry = neverExpire
fileExpiry = expiry.NeverExpire
} else { } else {
expiry = time.Now().Add(upReq.expiry)
fileExpiry = time.Now().Add(upReq.expiry)
} }
bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src)) bytes, err := fileBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src))
@ -273,7 +275,7 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) {
return upload, errors.New("File too large") return upload, errors.New("File too large")
} }
upload.Metadata, err = generateMetadata(upload.Filename, expiry, upReq.deletionKey)
upload.Metadata, err = generateMetadata(upload.Filename, fileExpiry, upReq.deletionKey)
if err != nil { if err != nil {
fileBackend.Delete(upload.Filename) fileBackend.Delete(upload.Filename)
return return
@ -342,14 +344,14 @@ func parseExpiry(expStr string) time.Duration {
if expStr == "" { if expStr == "" {
return time.Duration(Config.maxExpiry) * time.Second return time.Duration(Config.maxExpiry) * time.Second
} else { } else {
expiry, err := strconv.ParseUint(expStr, 10, 64)
fileExpiry, err := strconv.ParseUint(expStr, 10, 64)
if err != nil { if err != nil {
return time.Duration(Config.maxExpiry) * time.Second return time.Duration(Config.maxExpiry) * time.Second
} else { } else {
if Config.maxExpiry > 0 && (expiry > Config.maxExpiry || expiry == 0) {
expiry = Config.maxExpiry
if Config.maxExpiry > 0 && (fileExpiry > Config.maxExpiry || fileExpiry == 0) {
fileExpiry = Config.maxExpiry
} }
return time.Duration(expiry) * time.Second
return time.Duration(fileExpiry) * time.Second
} }
} }
} }
Loading…
Cancel
Save