Contains the Concourse pipeline definition for building a line-server container
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

198 lines
4.4 KiB

  1. package main
  2. import (
  3. "archive/tar"
  4. "archive/zip"
  5. "compress/bzip2"
  6. "compress/gzip"
  7. "crypto/sha256"
  8. "encoding/hex"
  9. "encoding/json"
  10. "errors"
  11. "io"
  12. "io/ioutil"
  13. "os"
  14. "path"
  15. "sort"
  16. "time"
  17. "unicode/utf8"
  18. "bitbucket.org/taruti/mimemagic"
  19. "github.com/dchest/uniuri"
  20. )
  21. type MetadataJSON struct {
  22. DeleteKey string `json:"delete_key"`
  23. Sha256sum string `json:"sha256sum"`
  24. Mimetype string `json:"mimetype"`
  25. Size int64 `json:"size"`
  26. Expiry int64 `json:"expiry"`
  27. ArchiveFiles []string `json:"archive_files,omitempty"`
  28. }
  29. type Metadata struct {
  30. DeleteKey string
  31. Sha256sum string
  32. Mimetype string
  33. Size int64
  34. Expiry time.Time
  35. ArchiveFiles []string
  36. }
  37. var NotFoundErr = errors.New("File not found.")
  38. var BadMetadata = errors.New("Corrupted metadata.")
  39. func generateMetadata(fName string, exp time.Time, delKey string) (m Metadata, err error) {
  40. file, err := os.Open(path.Join(Config.filesDir, fName))
  41. fileInfo, err := os.Stat(path.Join(Config.filesDir, fName))
  42. if err != nil {
  43. return
  44. }
  45. defer file.Close()
  46. m.Size = fileInfo.Size()
  47. m.Expiry = exp
  48. if delKey == "" {
  49. m.DeleteKey = uniuri.NewLen(30)
  50. } else {
  51. m.DeleteKey = delKey
  52. }
  53. // Get first 512 bytes for mimetype detection
  54. header := make([]byte, 512)
  55. file.Read(header)
  56. m.Mimetype = mimemagic.Match("", header)
  57. if m.Mimetype == "" {
  58. // Check if the file seems anything like text
  59. if utf8.Valid(header) {
  60. m.Mimetype = "text/plain"
  61. } else {
  62. m.Mimetype = "application/octet-stream"
  63. }
  64. }
  65. // Compute the sha256sum
  66. hasher := sha256.New()
  67. file.Seek(0, 0)
  68. _, err = io.Copy(hasher, file)
  69. if err == nil {
  70. m.Sha256sum = hex.EncodeToString(hasher.Sum(nil))
  71. }
  72. file.Seek(0, 0)
  73. // If archive, grab list of filenames
  74. if m.Mimetype == "application/x-tar" {
  75. tReadr := tar.NewReader(file)
  76. for {
  77. hdr, err := tReadr.Next()
  78. if err == io.EOF || err != nil {
  79. break
  80. }
  81. if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
  82. m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
  83. }
  84. }
  85. sort.Strings(m.ArchiveFiles)
  86. } else if m.Mimetype == "application/x-gzip" {
  87. gzf, err := gzip.NewReader(file)
  88. if err == nil {
  89. tReadr := tar.NewReader(gzf)
  90. for {
  91. hdr, err := tReadr.Next()
  92. if err == io.EOF || err != nil {
  93. break
  94. }
  95. if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
  96. m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
  97. }
  98. }
  99. sort.Strings(m.ArchiveFiles)
  100. }
  101. } else if m.Mimetype == "application/x-bzip" {
  102. bzf := bzip2.NewReader(file)
  103. tReadr := tar.NewReader(bzf)
  104. for {
  105. hdr, err := tReadr.Next()
  106. if err == io.EOF || err != nil {
  107. break
  108. }
  109. if hdr.Typeflag == tar.TypeDir || hdr.Typeflag == tar.TypeReg {
  110. m.ArchiveFiles = append(m.ArchiveFiles, hdr.Name)
  111. }
  112. }
  113. sort.Strings(m.ArchiveFiles)
  114. } else if m.Mimetype == "application/zip" {
  115. zf, err := zip.NewReader(file, m.Size)
  116. if err == nil {
  117. for _, f := range zf.File {
  118. m.ArchiveFiles = append(m.ArchiveFiles, f.Name)
  119. }
  120. }
  121. sort.Strings(m.ArchiveFiles)
  122. }
  123. return
  124. }
  125. func metadataWrite(filename string, metadata *Metadata) error {
  126. file, err := os.Create(path.Join(Config.metaDir, filename))
  127. if err != nil {
  128. return err
  129. }
  130. defer file.Close()
  131. mjson := MetadataJSON{}
  132. mjson.DeleteKey = metadata.DeleteKey
  133. mjson.Mimetype = metadata.Mimetype
  134. mjson.ArchiveFiles = metadata.ArchiveFiles
  135. mjson.Sha256sum = metadata.Sha256sum
  136. mjson.Expiry = metadata.Expiry.Unix()
  137. mjson.Size = metadata.Size
  138. byt, err := json.Marshal(mjson)
  139. if err != nil {
  140. return err
  141. }
  142. _, err = file.Write(byt)
  143. if err != nil {
  144. return err
  145. }
  146. return nil
  147. }
  148. func metadataRead(filename string) (metadata Metadata, err error) {
  149. b, err := ioutil.ReadFile(path.Join(Config.metaDir, filename))
  150. if err != nil {
  151. // Metadata does not exist, generate one
  152. newMData, err := generateMetadata(filename, neverExpire, "")
  153. if err != nil {
  154. return metadata, err
  155. }
  156. metadataWrite(filename, &newMData)
  157. b, err = ioutil.ReadFile(path.Join(Config.metaDir, filename))
  158. if err != nil {
  159. return metadata, BadMetadata
  160. }
  161. }
  162. mjson := MetadataJSON{}
  163. err = json.Unmarshal(b, &mjson)
  164. if err != nil {
  165. return metadata, BadMetadata
  166. }
  167. metadata.DeleteKey = mjson.DeleteKey
  168. metadata.Mimetype = mjson.Mimetype
  169. metadata.ArchiveFiles = mjson.ArchiveFiles
  170. metadata.Sha256sum = mjson.Sha256sum
  171. metadata.Expiry = time.Unix(mjson.Expiry, 0)
  172. metadata.Size = mjson.Size
  173. return
  174. }