Browse Source
Merge pull request #124 from mutantmonkey/cleanup_tool
Merge pull request #124 from mutantmonkey/cleanup_tool
Add linx-cleanup toolpull/98/merge
Andrei Marcu
8 years ago
committed by
GitHub
17 changed files with 235 additions and 98 deletions
-
1.gitignore
-
2Dockerfile
-
17README.md
-
15backends/localfs/localfs.go
-
23backends/meta.go
-
73backends/metajson/metajson.go
-
5backends/storage.go
-
2delete.go
-
3display.go
-
23expiry.go
-
13expiry/expiry.go
-
5fileserve.go
-
46linx-cleanup/cleanup.go
-
75meta.go
-
7server.go
-
3torrent.go
-
20upload.go
@ -0,0 +1,23 @@ |
|||||
|
package backends |
||||
|
|
||||
|
import ( |
||||
|
"errors" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
type MetaBackend interface { |
||||
|
Get(key string) (Metadata, error) |
||||
|
Put(key string, metadata *Metadata) error |
||||
|
} |
||||
|
|
||||
|
type Metadata struct { |
||||
|
DeleteKey string |
||||
|
Sha256sum string |
||||
|
Mimetype string |
||||
|
Size int64 |
||||
|
Expiry time.Time |
||||
|
ArchiveFiles []string |
||||
|
ShortURL string |
||||
|
} |
||||
|
|
||||
|
var BadMetadata = errors.New("Corrupted metadata.") |
@ -0,0 +1,73 @@ |
|||||
|
package metajson |
||||
|
|
||||
|
import ( |
||||
|
"bytes" |
||||
|
"encoding/json" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/andreimarcu/linx-server/backends" |
||||
|
) |
||||
|
|
||||
|
type MetadataJSON struct { |
||||
|
DeleteKey string `json:"delete_key"` |
||||
|
Sha256sum string `json:"sha256sum"` |
||||
|
Mimetype string `json:"mimetype"` |
||||
|
Size int64 `json:"size"` |
||||
|
Expiry int64 `json:"expiry"` |
||||
|
ArchiveFiles []string `json:"archive_files,omitempty"` |
||||
|
ShortURL string `json:"short_url"` |
||||
|
} |
||||
|
|
||||
|
type MetaJSONBackend struct { |
||||
|
storage backends.MetaStorageBackend |
||||
|
} |
||||
|
|
||||
|
func (m MetaJSONBackend) Put(key string, metadata *backends.Metadata) error { |
||||
|
mjson := MetadataJSON{} |
||||
|
mjson.DeleteKey = metadata.DeleteKey |
||||
|
mjson.Mimetype = metadata.Mimetype |
||||
|
mjson.ArchiveFiles = metadata.ArchiveFiles |
||||
|
mjson.Sha256sum = metadata.Sha256sum |
||||
|
mjson.Expiry = metadata.Expiry.Unix() |
||||
|
mjson.Size = metadata.Size |
||||
|
mjson.ShortURL = metadata.ShortURL |
||||
|
|
||||
|
byt, err := json.Marshal(mjson) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
if _, err := m.storage.Put(key, bytes.NewBuffer(byt)); err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (m MetaJSONBackend) Get(key string) (metadata backends.Metadata, err error) { |
||||
|
b, err := m.storage.Get(key) |
||||
|
if err != nil { |
||||
|
return metadata, backends.BadMetadata |
||||
|
} |
||||
|
|
||||
|
mjson := MetadataJSON{} |
||||
|
|
||||
|
err = json.Unmarshal(b, &mjson) |
||||
|
if err != nil { |
||||
|
return metadata, backends.BadMetadata |
||||
|
} |
||||
|
|
||||
|
metadata.DeleteKey = mjson.DeleteKey |
||||
|
metadata.Mimetype = mjson.Mimetype |
||||
|
metadata.ArchiveFiles = mjson.ArchiveFiles |
||||
|
metadata.Sha256sum = mjson.Sha256sum |
||||
|
metadata.Expiry = time.Unix(mjson.Expiry, 0) |
||||
|
metadata.Size = mjson.Size |
||||
|
metadata.ShortURL = mjson.ShortURL |
||||
|
|
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func NewMetaJSONBackend(storage backends.MetaStorageBackend) MetaJSONBackend { |
||||
|
return MetaJSONBackend{storage: storage} |
||||
|
} |
@ -0,0 +1,13 @@ |
|||||
|
package expiry |
||||
|
|
||||
|
import ( |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
var NeverExpire = time.Unix(0, 0) |
||||
|
|
||||
|
// Determine if a file with expiry set to "ts" has expired yet
|
||||
|
func IsTsExpired(ts time.Time) bool { |
||||
|
now := time.Now() |
||||
|
return ts != NeverExpire && now.After(ts) |
||||
|
} |
@ -0,0 +1,46 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"flag" |
||||
|
"log" |
||||
|
|
||||
|
"github.com/andreimarcu/linx-server/backends/localfs" |
||||
|
"github.com/andreimarcu/linx-server/backends/metajson" |
||||
|
"github.com/andreimarcu/linx-server/expiry" |
||||
|
) |
||||
|
|
||||
|
func main() { |
||||
|
var filesDir string |
||||
|
var metaDir string |
||||
|
var noLogs bool |
||||
|
|
||||
|
flag.StringVar(&filesDir, "filespath", "files/", |
||||
|
"path to files directory") |
||||
|
flag.StringVar(&metaDir, "metapath", "meta/", |
||||
|
"path to metadata directory") |
||||
|
flag.BoolVar(&noLogs, "nologs", false, |
||||
|
"don't log deleted files") |
||||
|
flag.Parse() |
||||
|
|
||||
|
metaStorageBackend := localfs.NewLocalfsBackend(metaDir) |
||||
|
metaBackend := metajson.NewMetaJSONBackend(metaStorageBackend) |
||||
|
fileBackend := localfs.NewLocalfsBackend(filesDir) |
||||
|
|
||||
|
files, err := metaStorageBackend.List() |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
|
||||
|
for _, filename := range files { |
||||
|
metadata, err := metaBackend.Get(filename) |
||||
|
if err != nil { |
||||
|
log.Printf("Failed to find metadata for %s", filename) |
||||
|
} |
||||
|
|
||||
|
if expiry.IsTsExpired(metadata.Expiry) { |
||||
|
log.Printf("Delete %s", filename) |
||||
|
fileBackend.Delete(filename) |
||||
|
metaStorageBackend.Delete(filename) |
||||
|
} |
||||
|
} |
||||
|
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue