You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

237 lines
5.7 KiB

  1. package s3api
  2. import (
  3. "fmt"
  4. "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
  5. "github.com/aws/aws-sdk-go/service/s3"
  6. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  7. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
  8. "github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
  9. "github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
  10. "reflect"
  11. "sync"
  12. "testing"
  13. "time"
  14. )
  15. type BucketMetadataTestCase struct {
  16. filerEntry *filer_pb.Entry
  17. expectBucketMetadata *BucketMetaData
  18. }
  19. var (
  20. //bad entry
  21. badEntry = &filer_pb.Entry{
  22. Name: "badEntry",
  23. }
  24. //good entry
  25. goodEntryAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
  26. Owner: &s3.Owner{
  27. DisplayName: &s3account.AccountAdmin.Name,
  28. ID: &s3account.AccountAdmin.Id,
  29. },
  30. Grants: s3_constants.PublicRead,
  31. })
  32. goodEntry = &filer_pb.Entry{
  33. Name: "entryWithValidAcp",
  34. Extended: map[string][]byte{
  35. s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
  36. s3_constants.ExtAcpKey: goodEntryAcp,
  37. },
  38. }
  39. //ownership is ""
  40. ownershipEmptyStr = &filer_pb.Entry{
  41. Name: "ownershipEmptyStr",
  42. Extended: map[string][]byte{
  43. s3_constants.ExtOwnershipKey: []byte(""),
  44. },
  45. }
  46. //ownership valid
  47. ownershipValid = &filer_pb.Entry{
  48. Name: "ownershipValid",
  49. Extended: map[string][]byte{
  50. s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
  51. },
  52. }
  53. //acp is ""
  54. acpEmptyStr = &filer_pb.Entry{
  55. Name: "acpEmptyStr",
  56. Extended: map[string][]byte{
  57. s3_constants.ExtAcpKey: []byte(""),
  58. },
  59. }
  60. //acp is empty object
  61. acpEmptyObjectAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
  62. Owner: nil,
  63. Grants: nil,
  64. })
  65. acpEmptyObject = &filer_pb.Entry{
  66. Name: "acpEmptyObject",
  67. Extended: map[string][]byte{
  68. s3_constants.ExtAcpKey: acpEmptyObjectAcp,
  69. },
  70. }
  71. //acp owner is nil
  72. acpOwnerNilAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
  73. Owner: nil,
  74. Grants: make([]*s3.Grant, 1),
  75. })
  76. acpOwnerNil = &filer_pb.Entry{
  77. Name: "acpOwnerNil",
  78. Extended: map[string][]byte{
  79. s3_constants.ExtAcpKey: acpOwnerNilAcp,
  80. },
  81. }
  82. //load filer is
  83. loadFilerBucket = make(map[string]int, 1)
  84. //override `loadBucketMetadataFromFiler` to avoid really load from filer
  85. )
  86. var tcs = []*BucketMetadataTestCase{
  87. {
  88. badEntry, &BucketMetaData{
  89. Name: badEntry.Name,
  90. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  91. Owner: &s3.Owner{
  92. DisplayName: &s3account.AccountAdmin.Name,
  93. ID: &s3account.AccountAdmin.Id,
  94. },
  95. Acl: nil,
  96. },
  97. },
  98. {
  99. goodEntry, &BucketMetaData{
  100. Name: goodEntry.Name,
  101. ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
  102. Owner: &s3.Owner{
  103. DisplayName: &s3account.AccountAdmin.Name,
  104. ID: &s3account.AccountAdmin.Id,
  105. },
  106. Acl: s3_constants.PublicRead,
  107. },
  108. },
  109. {
  110. ownershipEmptyStr, &BucketMetaData{
  111. Name: ownershipEmptyStr.Name,
  112. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  113. Owner: &s3.Owner{
  114. DisplayName: &s3account.AccountAdmin.Name,
  115. ID: &s3account.AccountAdmin.Id,
  116. },
  117. Acl: nil,
  118. },
  119. },
  120. {
  121. ownershipValid, &BucketMetaData{
  122. Name: ownershipValid.Name,
  123. ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
  124. Owner: &s3.Owner{
  125. DisplayName: &s3account.AccountAdmin.Name,
  126. ID: &s3account.AccountAdmin.Id,
  127. },
  128. Acl: nil,
  129. },
  130. },
  131. {
  132. acpEmptyStr, &BucketMetaData{
  133. Name: acpEmptyStr.Name,
  134. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  135. Owner: &s3.Owner{
  136. DisplayName: &s3account.AccountAdmin.Name,
  137. ID: &s3account.AccountAdmin.Id,
  138. },
  139. Acl: nil,
  140. },
  141. },
  142. {
  143. acpEmptyObject, &BucketMetaData{
  144. Name: acpEmptyObject.Name,
  145. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  146. Owner: &s3.Owner{
  147. DisplayName: &s3account.AccountAdmin.Name,
  148. ID: &s3account.AccountAdmin.Id,
  149. },
  150. Acl: nil,
  151. },
  152. },
  153. {
  154. acpOwnerNil, &BucketMetaData{
  155. Name: acpOwnerNil.Name,
  156. ObjectOwnership: s3_constants.DefaultOwnershipForExists,
  157. Owner: &s3.Owner{
  158. DisplayName: &s3account.AccountAdmin.Name,
  159. ID: &s3account.AccountAdmin.Id,
  160. },
  161. Acl: make([]*s3.Grant, 0),
  162. },
  163. },
  164. }
  165. func TestBuildBucketMetadata(t *testing.T) {
  166. for _, tc := range tcs {
  167. resultBucketMetadata := buildBucketMetadata(tc.filerEntry)
  168. if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {
  169. t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata)
  170. }
  171. }
  172. }
  173. func TestGetBucketMetadata(t *testing.T) {
  174. loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
  175. time.Sleep(time.Second)
  176. loadFilerBucket[bucketName] = loadFilerBucket[bucketName] + 1
  177. return &BucketMetaData{
  178. Name: bucketName,
  179. }, nil
  180. }
  181. br := &BucketRegistry{
  182. metadataCache: make(map[string]*BucketMetaData),
  183. notFound: make(map[string]struct{}),
  184. s3a: nil,
  185. }
  186. //start 40 goroutine for
  187. var wg sync.WaitGroup
  188. closeCh := make(chan struct{})
  189. for i := 0; i < 40; i++ {
  190. wg.Add(1)
  191. go func() {
  192. defer wg.Done()
  193. outLoop:
  194. for {
  195. for j := 0; j < 5; j++ {
  196. select {
  197. case <-closeCh:
  198. break outLoop
  199. default:
  200. reqBucket := fmt.Sprintf("%c", 67+j)
  201. _, errCode := br.GetBucketMetadata(reqBucket)
  202. if errCode != s3err.ErrNone {
  203. close(closeCh)
  204. t.Error("not expect")
  205. }
  206. }
  207. }
  208. time.Sleep(10 * time.Microsecond)
  209. }
  210. }()
  211. }
  212. time.Sleep(time.Second)
  213. close(closeCh)
  214. wg.Wait()
  215. //Each bucket is loaded from the filer only once
  216. for bucketName, loadCount := range loadFilerBucket {
  217. if loadCount != 1 {
  218. t.Fatalf("lock is uneffict: %s, %d", bucketName, loadCount)
  219. }
  220. }
  221. }