You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

355 lines
10 KiB

7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
5 years ago
4 years ago
  1. package s3api
  2. import (
  3. "context"
  4. "encoding/xml"
  5. "fmt"
  6. "github.com/chrislusf/seaweedfs/weed/glog"
  7. "io"
  8. "net/http"
  9. "net/url"
  10. "path/filepath"
  11. "strconv"
  12. "strings"
  13. "time"
  14. "github.com/chrislusf/seaweedfs/weed/filer"
  15. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  16. xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
  17. "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
  18. )
  19. type ListBucketResultV2 struct {
  20. XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
  21. Name string `xml:"Name"`
  22. Prefix string `xml:"Prefix"`
  23. MaxKeys int `xml:"MaxKeys"`
  24. Delimiter string `xml:"Delimiter,omitempty"`
  25. IsTruncated bool `xml:"IsTruncated"`
  26. Contents []ListEntry `xml:"Contents,omitempty"`
  27. CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
  28. ContinuationToken string `xml:"ContinuationToken,omitempty"`
  29. NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
  30. KeyCount int `xml:"KeyCount"`
  31. StartAfter string `xml:"StartAfter,omitempty"`
  32. }
  33. func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
  34. // https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
  35. // collect parameters
  36. bucket, _ := getBucketAndObject(r)
  37. originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
  38. if maxKeys < 0 {
  39. writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL)
  40. return
  41. }
  42. if delimiter != "" && delimiter != "/" {
  43. writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
  44. return
  45. }
  46. marker := continuationToken
  47. if continuationToken == "" {
  48. marker = startAfter
  49. }
  50. response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
  51. if err != nil {
  52. writeErrorResponse(w, s3err.ErrInternalError, r.URL)
  53. return
  54. }
  55. responseV2 := &ListBucketResultV2{
  56. XMLName: response.XMLName,
  57. Name: response.Name,
  58. CommonPrefixes: response.CommonPrefixes,
  59. Contents: response.Contents,
  60. ContinuationToken: continuationToken,
  61. Delimiter: response.Delimiter,
  62. IsTruncated: response.IsTruncated,
  63. KeyCount: len(response.Contents),
  64. MaxKeys: response.MaxKeys,
  65. NextContinuationToken: response.NextMarker,
  66. Prefix: response.Prefix,
  67. StartAfter: startAfter,
  68. }
  69. writeSuccessResponseXML(w, encodeResponse(responseV2))
  70. }
  71. func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
  72. // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
  73. // collect parameters
  74. bucket, _ := getBucketAndObject(r)
  75. originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
  76. if maxKeys < 0 {
  77. writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL)
  78. return
  79. }
  80. if delimiter != "" && delimiter != "/" {
  81. writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
  82. return
  83. }
  84. response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
  85. if err != nil {
  86. writeErrorResponse(w, s3err.ErrInternalError, r.URL)
  87. return
  88. }
  89. writeSuccessResponseXML(w, encodeResponse(response))
  90. }
  91. func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {
  92. // convert full path prefix into directory name and prefix for entry name
  93. reqDir, prefix := filepath.Split(originalPrefix)
  94. if strings.HasPrefix(reqDir, "/") {
  95. reqDir = reqDir[1:]
  96. }
  97. bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
  98. reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
  99. if strings.HasSuffix(reqDir, "/") {
  100. // remove trailing "/"
  101. reqDir = reqDir[:len(reqDir)-1]
  102. }
  103. var contents []ListEntry
  104. var commonPrefixes []PrefixEntry
  105. var isTruncated bool
  106. var doErr error
  107. var nextMarker string
  108. // check filer
  109. err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  110. _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) {
  111. if entry.IsDirectory {
  112. if delimiter == "/" {
  113. commonPrefixes = append(commonPrefixes, PrefixEntry{
  114. Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
  115. })
  116. }
  117. } else {
  118. storageClass := "STANDARD"
  119. if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
  120. storageClass = string(v)
  121. }
  122. contents = append(contents, ListEntry{
  123. Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
  124. LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
  125. ETag: "\"" + filer.ETag(entry) + "\"",
  126. Size: int64(filer.FileSize(entry)),
  127. Owner: CanonicalUser{
  128. ID: fmt.Sprintf("%x", entry.Attributes.Uid),
  129. DisplayName: entry.Attributes.UserName,
  130. },
  131. StorageClass: StorageClass(storageClass),
  132. })
  133. }
  134. })
  135. if doErr != nil {
  136. return doErr
  137. }
  138. if !isTruncated {
  139. nextMarker = ""
  140. }
  141. response = ListBucketResult{
  142. Name: bucket,
  143. Prefix: originalPrefix,
  144. Marker: marker,
  145. NextMarker: nextMarker,
  146. MaxKeys: maxKeys,
  147. Delimiter: delimiter,
  148. IsTruncated: isTruncated,
  149. Contents: contents,
  150. CommonPrefixes: commonPrefixes,
  151. }
  152. return nil
  153. })
  154. return
  155. }
  156. func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
  157. // invariants
  158. // prefix and marker should be under dir, marker may contain "/"
  159. // maxKeys should be updated for each recursion
  160. if prefix == "/" && delimiter == "/" {
  161. return
  162. }
  163. if maxKeys <= 0 {
  164. return
  165. }
  166. if strings.Contains(marker, "/") {
  167. sepIndex := strings.Index(marker, "/")
  168. subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
  169. // println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
  170. subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn)
  171. if subErr != nil {
  172. err = subErr
  173. return
  174. }
  175. isTruncated = isTruncated || subIsTruncated
  176. maxKeys -= subCounter
  177. nextMarker = subDir + "/" + subNextMarker
  178. counter += subCounter
  179. // finished processing this sub directory
  180. marker = subDir
  181. }
  182. // now marker is also a direct child of dir
  183. request := &filer_pb.ListEntriesRequest{
  184. Directory: dir,
  185. Prefix: prefix,
  186. Limit: uint32(maxKeys + 1),
  187. StartFromFileName: marker,
  188. InclusiveStartFrom: false,
  189. }
  190. ctx, cancel := context.WithCancel(context.Background())
  191. defer cancel()
  192. stream, listErr := client.ListEntries(ctx, request)
  193. if listErr != nil {
  194. err = fmt.Errorf("list entires %+v: %v", request, listErr)
  195. return
  196. }
  197. for {
  198. resp, recvErr := stream.Recv()
  199. if recvErr != nil {
  200. if recvErr == io.EOF {
  201. break
  202. } else {
  203. err = fmt.Errorf("iterating entires %+v: %v", request, recvErr)
  204. return
  205. }
  206. }
  207. if counter >= maxKeys {
  208. isTruncated = true
  209. return
  210. }
  211. entry := resp.Entry
  212. nextMarker = entry.Name
  213. if entry.IsDirectory {
  214. // println("ListEntries", dir, "dir:", entry.Name)
  215. if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
  216. if delimiter != "/" {
  217. eachEntryFn(dir, entry)
  218. // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
  219. subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn)
  220. if subErr != nil {
  221. err = fmt.Errorf("doListFilerEntries2: %v", subErr)
  222. return
  223. }
  224. // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
  225. counter += subCounter
  226. nextMarker = entry.Name + "/" + subNextMarker
  227. if subIsTruncated {
  228. isTruncated = true
  229. return
  230. }
  231. } else {
  232. var isEmpty bool
  233. if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
  234. return
  235. }
  236. if !isEmpty {
  237. eachEntryFn(dir, entry)
  238. counter++
  239. }
  240. }
  241. }
  242. } else {
  243. // println("ListEntries", dir, "file:", entry.Name)
  244. eachEntryFn(dir, entry)
  245. counter++
  246. }
  247. }
  248. return
  249. }
  250. func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
  251. prefix = values.Get("prefix")
  252. token = values.Get("continuation-token")
  253. startAfter = values.Get("start-after")
  254. delimiter = values.Get("delimiter")
  255. if values.Get("max-keys") != "" {
  256. maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
  257. } else {
  258. maxkeys = maxObjectListSizeLimit
  259. }
  260. fetchOwner = values.Get("fetch-owner") == "true"
  261. return
  262. }
  263. func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
  264. prefix = values.Get("prefix")
  265. marker = values.Get("marker")
  266. delimiter = values.Get("delimiter")
  267. if values.Get("max-keys") != "" {
  268. maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
  269. } else {
  270. maxkeys = maxObjectListSizeLimit
  271. }
  272. return
  273. }
  274. func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, parentDir, name string) (isEmpty bool, err error) {
  275. // println("+ isDirectoryAllEmpty", dir, name)
  276. var fileCounter int
  277. var subDirs []string
  278. currentDir := parentDir + "/" + name
  279. var startFrom string
  280. var isExhausted bool
  281. for fileCounter == 0 && !isExhausted {
  282. err = filer_pb.SeaweedList(filerClient, currentDir, "", func(entry *filer_pb.Entry, isLast bool) error {
  283. if entry.IsDirectory {
  284. subDirs = append(subDirs, entry.Name)
  285. } else {
  286. fileCounter++
  287. }
  288. startFrom = entry.Name
  289. isExhausted = isExhausted || isLast
  290. return nil
  291. }, startFrom, false, 8)
  292. }
  293. if err != nil {
  294. return false, err
  295. }
  296. if fileCounter > 0 {
  297. return false, nil
  298. }
  299. for _, subDir := range subDirs {
  300. isSubEmpty, subErr := s3a.isDirectoryAllEmpty(filerClient, currentDir, subDir)
  301. if subErr != nil {
  302. return false, subErr
  303. }
  304. if !isSubEmpty {
  305. return false, nil
  306. }
  307. }
  308. glog.V(1).Infof("deleting empty folder %s", currentDir)
  309. if err = doDeleteEntry(filerClient, parentDir, name, true, true); err != nil {
  310. return
  311. }
  312. return true, nil
  313. }