You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

385 lines
11 KiB

7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
4 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
5 years ago
4 years ago
4 years ago
4 years ago
  1. package s3api
  2. import (
  3. "context"
  4. "encoding/xml"
  5. "fmt"
  6. "github.com/chrislusf/seaweedfs/weed/glog"
  7. "io"
  8. "net/http"
  9. "net/url"
  10. "path/filepath"
  11. "strconv"
  12. "strings"
  13. "time"
  14. "github.com/chrislusf/seaweedfs/weed/filer"
  15. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  16. xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
  17. "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
  18. )
  19. type ListBucketResultV2 struct {
  20. XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
  21. Name string `xml:"Name"`
  22. Prefix string `xml:"Prefix"`
  23. MaxKeys int `xml:"MaxKeys"`
  24. Delimiter string `xml:"Delimiter,omitempty"`
  25. IsTruncated bool `xml:"IsTruncated"`
  26. Contents []ListEntry `xml:"Contents,omitempty"`
  27. CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
  28. ContinuationToken string `xml:"ContinuationToken,omitempty"`
  29. NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
  30. KeyCount int `xml:"KeyCount"`
  31. StartAfter string `xml:"StartAfter,omitempty"`
  32. }
  33. func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
  34. // https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
  35. // collect parameters
  36. bucket, _ := xhttp.GetBucketAndObject(r)
  37. glog.V(3).Infof("ListObjectsV2Handler %s", bucket)
  38. originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
  39. if maxKeys < 0 {
  40. s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
  41. return
  42. }
  43. if delimiter != "" && delimiter != "/" {
  44. s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
  45. return
  46. }
  47. marker := continuationToken
  48. if continuationToken == "" {
  49. marker = startAfter
  50. }
  51. response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
  52. if err != nil {
  53. s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
  54. return
  55. }
  56. if len(response.Contents) == 0 {
  57. if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
  58. s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
  59. return
  60. }
  61. }
  62. responseV2 := &ListBucketResultV2{
  63. XMLName: response.XMLName,
  64. Name: response.Name,
  65. CommonPrefixes: response.CommonPrefixes,
  66. Contents: response.Contents,
  67. ContinuationToken: continuationToken,
  68. Delimiter: response.Delimiter,
  69. IsTruncated: response.IsTruncated,
  70. KeyCount: len(response.Contents) + len(response.CommonPrefixes),
  71. MaxKeys: response.MaxKeys,
  72. NextContinuationToken: response.NextMarker,
  73. Prefix: response.Prefix,
  74. StartAfter: startAfter,
  75. }
  76. writeSuccessResponseXML(w, r, responseV2)
  77. }
  78. func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
  79. // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
  80. // collect parameters
  81. bucket, _ := xhttp.GetBucketAndObject(r)
  82. glog.V(3).Infof("ListObjectsV1Handler %s", bucket)
  83. originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
  84. if maxKeys < 0 {
  85. s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
  86. return
  87. }
  88. if delimiter != "" && delimiter != "/" {
  89. s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
  90. return
  91. }
  92. response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
  93. if err != nil {
  94. s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
  95. return
  96. }
  97. if len(response.Contents) == 0 {
  98. if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
  99. s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
  100. return
  101. }
  102. }
  103. writeSuccessResponseXML(w, r, response)
  104. }
  105. func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {
  106. // convert full path prefix into directory name and prefix for entry name
  107. reqDir, prefix := filepath.Split(originalPrefix)
  108. if strings.HasPrefix(reqDir, "/") {
  109. reqDir = reqDir[1:]
  110. }
  111. bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
  112. reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
  113. if strings.HasSuffix(reqDir, "/") {
  114. // remove trailing "/"
  115. reqDir = reqDir[:len(reqDir)-1]
  116. }
  117. var contents []ListEntry
  118. var commonPrefixes []PrefixEntry
  119. var isTruncated bool
  120. var doErr error
  121. var nextMarker string
  122. // check filer
  123. err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  124. _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) {
  125. if entry.IsDirectory {
  126. if delimiter == "/" {
  127. commonPrefixes = append(commonPrefixes, PrefixEntry{
  128. Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
  129. })
  130. }
  131. } else {
  132. storageClass := "STANDARD"
  133. if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
  134. storageClass = string(v)
  135. }
  136. contents = append(contents, ListEntry{
  137. Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
  138. LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
  139. ETag: "\"" + filer.ETag(entry) + "\"",
  140. Size: int64(filer.FileSize(entry)),
  141. Owner: CanonicalUser{
  142. ID: fmt.Sprintf("%x", entry.Attributes.Uid),
  143. DisplayName: entry.Attributes.UserName,
  144. },
  145. StorageClass: StorageClass(storageClass),
  146. })
  147. }
  148. })
  149. if doErr != nil {
  150. return doErr
  151. }
  152. if !isTruncated {
  153. nextMarker = ""
  154. }
  155. response = ListBucketResult{
  156. Name: bucket,
  157. Prefix: originalPrefix,
  158. Marker: marker,
  159. NextMarker: nextMarker,
  160. MaxKeys: maxKeys,
  161. Delimiter: delimiter,
  162. IsTruncated: isTruncated,
  163. Contents: contents,
  164. CommonPrefixes: commonPrefixes,
  165. }
  166. return nil
  167. })
  168. return
  169. }
  170. func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
  171. // invariants
  172. // prefix and marker should be under dir, marker may contain "/"
  173. // maxKeys should be updated for each recursion
  174. if prefix == "/" && delimiter == "/" {
  175. return
  176. }
  177. if maxKeys <= 0 {
  178. return
  179. }
  180. if strings.Contains(marker, "/") {
  181. sepIndex := strings.Index(marker, "/")
  182. subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
  183. // println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
  184. subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn)
  185. if subErr != nil {
  186. err = subErr
  187. return
  188. }
  189. counter += subCounter
  190. isTruncated = isTruncated || subIsTruncated
  191. maxKeys -= subCounter
  192. nextMarker = subDir + "/" + subNextMarker
  193. // finished processing this sub directory
  194. marker = subDir
  195. }
  196. if maxKeys <= 0 {
  197. return
  198. }
  199. // now marker is also a direct child of dir
  200. request := &filer_pb.ListEntriesRequest{
  201. Directory: dir,
  202. Prefix: prefix,
  203. Limit: uint32(maxKeys + 1),
  204. StartFromFileName: marker,
  205. InclusiveStartFrom: false,
  206. }
  207. ctx, cancel := context.WithCancel(context.Background())
  208. defer cancel()
  209. stream, listErr := client.ListEntries(ctx, request)
  210. if listErr != nil {
  211. err = fmt.Errorf("list entires %+v: %v", request, listErr)
  212. return
  213. }
  214. for {
  215. resp, recvErr := stream.Recv()
  216. if recvErr != nil {
  217. if recvErr == io.EOF {
  218. break
  219. } else {
  220. err = fmt.Errorf("iterating entires %+v: %v", request, recvErr)
  221. return
  222. }
  223. }
  224. if counter >= maxKeys {
  225. isTruncated = true
  226. return
  227. }
  228. entry := resp.Entry
  229. nextMarker = entry.Name
  230. if entry.IsDirectory {
  231. // println("ListEntries", dir, "dir:", entry.Name)
  232. if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
  233. if delimiter != "/" {
  234. eachEntryFn(dir, entry)
  235. // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
  236. subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn)
  237. if subErr != nil {
  238. err = fmt.Errorf("doListFilerEntries2: %v", subErr)
  239. return
  240. }
  241. // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
  242. counter += subCounter
  243. nextMarker = entry.Name + "/" + subNextMarker
  244. if subIsTruncated {
  245. isTruncated = true
  246. return
  247. }
  248. } else {
  249. var isEmpty bool
  250. if !s3a.option.AllowEmptyFolder {
  251. if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
  252. glog.Errorf("check empty folder %s: %v", dir, err)
  253. }
  254. }
  255. if !isEmpty {
  256. eachEntryFn(dir, entry)
  257. counter++
  258. }
  259. }
  260. }
  261. } else {
  262. // println("ListEntries", dir, "file:", entry.Name)
  263. eachEntryFn(dir, entry)
  264. counter++
  265. }
  266. }
  267. return
  268. }
  269. func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
  270. prefix = values.Get("prefix")
  271. token = values.Get("continuation-token")
  272. startAfter = values.Get("start-after")
  273. delimiter = values.Get("delimiter")
  274. if values.Get("max-keys") != "" {
  275. maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
  276. } else {
  277. maxkeys = maxObjectListSizeLimit
  278. }
  279. fetchOwner = values.Get("fetch-owner") == "true"
  280. return
  281. }
  282. func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
  283. prefix = values.Get("prefix")
  284. marker = values.Get("marker")
  285. delimiter = values.Get("delimiter")
  286. if values.Get("max-keys") != "" {
  287. maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
  288. } else {
  289. maxkeys = maxObjectListSizeLimit
  290. }
  291. return
  292. }
  293. func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, parentDir, name string) (isEmpty bool, err error) {
  294. // println("+ isDirectoryAllEmpty", dir, name)
  295. glog.V(4).Infof("+ isEmpty %s/%s", parentDir, name)
  296. defer glog.V(4).Infof("- isEmpty %s/%s %v", parentDir, name, isEmpty)
  297. var fileCounter int
  298. var subDirs []string
  299. currentDir := parentDir + "/" + name
  300. var startFrom string
  301. var isExhausted bool
  302. var foundEntry bool
  303. for fileCounter == 0 && !isExhausted && err == nil {
  304. err = filer_pb.SeaweedList(filerClient, currentDir, "", func(entry *filer_pb.Entry, isLast bool) error {
  305. foundEntry = true
  306. if entry.IsDirectory {
  307. subDirs = append(subDirs, entry.Name)
  308. } else {
  309. fileCounter++
  310. }
  311. startFrom = entry.Name
  312. isExhausted = isExhausted || isLast
  313. glog.V(4).Infof(" * %s/%s isLast: %t", currentDir, startFrom, isLast)
  314. return nil
  315. }, startFrom, false, 8)
  316. if !foundEntry {
  317. break
  318. }
  319. }
  320. if err != nil {
  321. return false, err
  322. }
  323. if fileCounter > 0 {
  324. return false, nil
  325. }
  326. for _, subDir := range subDirs {
  327. isSubEmpty, subErr := s3a.isDirectoryAllEmpty(filerClient, currentDir, subDir)
  328. if subErr != nil {
  329. return false, subErr
  330. }
  331. if !isSubEmpty {
  332. return false, nil
  333. }
  334. }
  335. glog.V(1).Infof("deleting empty folder %s", currentDir)
  336. if err = doDeleteEntry(filerClient, parentDir, name, true, true); err != nil {
  337. return
  338. }
  339. return true, nil
  340. }