You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

412 lines
12 KiB

3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
3 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package s3api
  2. // the related code is copied and modified from minio source code
  3. /*
  4. * Minio Cloud Storage, (C) 2016 Minio, Inc.
  5. *
  6. * Licensed under the Apache License, Version 2.0 (the "License");
  7. * you may not use this file except in compliance with the License.
  8. * You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. import (
  19. "bufio"
  20. "bytes"
  21. "crypto/sha256"
  22. "encoding/hex"
  23. "errors"
  24. xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
  25. "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
  26. "hash"
  27. "io"
  28. "net/http"
  29. "time"
  30. "github.com/dustin/go-humanize"
  31. )
  32. // getChunkSignature - get chunk signature.
  33. func getChunkSignature(secretKey string, seedSignature string, region string, date time.Time, hashedChunk string) string {
  34. // Calculate string to sign.
  35. stringToSign := signV4ChunkedAlgorithm + "\n" +
  36. date.Format(iso8601Format) + "\n" +
  37. getScope(date, region) + "\n" +
  38. seedSignature + "\n" +
  39. emptySHA256 + "\n" +
  40. hashedChunk
  41. // Get hmac signing key.
  42. signingKey := getSigningKey(secretKey, date, region, "s3")
  43. // Calculate signature.
  44. newSignature := getSignature(signingKey, stringToSign)
  45. return newSignature
  46. }
  47. // calculateSeedSignature - Calculate seed signature in accordance with
  48. // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
  49. // returns signature, error otherwise if the signature mismatches or any other
  50. // error while parsing and validating.
  51. func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cred *Credential, signature string, region string, date time.Time, errCode s3err.ErrorCode) {
  52. // Copy request.
  53. req := *r
  54. // Save authorization header.
  55. v4Auth := req.Header.Get("Authorization")
  56. // Parse signature version '4' header.
  57. signV4Values, errCode := parseSignV4(v4Auth)
  58. if errCode != s3err.ErrNone {
  59. return nil, "", "", time.Time{}, errCode
  60. }
  61. // Payload streaming.
  62. payload := streamingContentSHA256
  63. // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
  64. if payload != req.Header.Get("X-Amz-Content-Sha256") {
  65. return nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
  66. }
  67. // Extract all the signed headers along with its values.
  68. extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
  69. if errCode != s3err.ErrNone {
  70. return nil, "", "", time.Time{}, errCode
  71. }
  72. // Verify if the access key id matches.
  73. identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey)
  74. if !found {
  75. return nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
  76. }
  77. bucket, _ := xhttp.GetBucketAndObject(r)
  78. if !identity.canDo("Write", bucket) {
  79. errCode = s3err.ErrAccessDenied
  80. return
  81. }
  82. // Verify if region is valid.
  83. region = signV4Values.Credential.scope.region
  84. // Extract date, if not present throw error.
  85. var dateStr string
  86. if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
  87. if dateStr = r.Header.Get("Date"); dateStr == "" {
  88. return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader
  89. }
  90. }
  91. // Parse date header.
  92. var err error
  93. date, err = time.Parse(iso8601Format, dateStr)
  94. if err != nil {
  95. return nil, "", "", time.Time{}, s3err.ErrMalformedDate
  96. }
  97. // Query string.
  98. queryStr := req.URL.Query().Encode()
  99. // Get canonical request.
  100. canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method)
  101. // Get string to sign from canonical request.
  102. stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
  103. // Get hmac signing key.
  104. signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, "s3")
  105. // Calculate signature.
  106. newSignature := getSignature(signingKey, stringToSign)
  107. // Verify if signature match.
  108. if !compareSignatureV4(newSignature, signV4Values.Signature) {
  109. return nil, "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch
  110. }
  111. // Return caculated signature.
  112. return cred, newSignature, region, date, s3err.ErrNone
  113. }
  114. const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB
  115. // lineTooLong is generated as chunk header is bigger than 4KiB.
  116. var errLineTooLong = errors.New("header line too long")
  117. // Malformed encoding is generated when chunk header is wrongly formed.
  118. var errMalformedEncoding = errors.New("malformed chunked encoding")
  119. // newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r
  120. // out of HTTP "chunked" format before returning it.
  121. // The s3ChunkedReader returns io.EOF when the final 0-length chunk is read.
  122. func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, s3err.ErrorCode) {
  123. ident, seedSignature, region, seedDate, errCode := iam.calculateSeedSignature(req)
  124. if errCode != s3err.ErrNone {
  125. return nil, errCode
  126. }
  127. return &s3ChunkedReader{
  128. cred: ident,
  129. reader: bufio.NewReader(req.Body),
  130. seedSignature: seedSignature,
  131. seedDate: seedDate,
  132. region: region,
  133. chunkSHA256Writer: sha256.New(),
  134. state: readChunkHeader,
  135. }, s3err.ErrNone
  136. }
  137. // Represents the overall state that is required for decoding a
  138. // AWS Signature V4 chunked reader.
  139. type s3ChunkedReader struct {
  140. cred *Credential
  141. reader *bufio.Reader
  142. seedSignature string
  143. seedDate time.Time
  144. region string
  145. state chunkState
  146. lastChunk bool
  147. chunkSignature string
  148. chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data.
  149. n uint64 // Unread bytes in chunk
  150. err error
  151. }
  152. // Read chunk reads the chunk token signature portion.
  153. func (cr *s3ChunkedReader) readS3ChunkHeader() {
  154. // Read the first chunk line until CRLF.
  155. var hexChunkSize, hexChunkSignature []byte
  156. hexChunkSize, hexChunkSignature, cr.err = readChunkLine(cr.reader)
  157. if cr.err != nil {
  158. return
  159. }
  160. // <hex>;token=value - converts the hex into its uint64 form.
  161. cr.n, cr.err = parseHexUint(hexChunkSize)
  162. if cr.err != nil {
  163. return
  164. }
  165. if cr.n == 0 {
  166. cr.err = io.EOF
  167. }
  168. // Save the incoming chunk signature.
  169. cr.chunkSignature = string(hexChunkSignature)
  170. }
  171. type chunkState int
  172. const (
  173. readChunkHeader chunkState = iota
  174. readChunkTrailer
  175. readChunk
  176. verifyChunk
  177. eofChunk
  178. )
  179. func (cs chunkState) String() string {
  180. stateString := ""
  181. switch cs {
  182. case readChunkHeader:
  183. stateString = "readChunkHeader"
  184. case readChunkTrailer:
  185. stateString = "readChunkTrailer"
  186. case readChunk:
  187. stateString = "readChunk"
  188. case verifyChunk:
  189. stateString = "verifyChunk"
  190. case eofChunk:
  191. stateString = "eofChunk"
  192. }
  193. return stateString
  194. }
  195. func (cr *s3ChunkedReader) Close() (err error) {
  196. return nil
  197. }
  198. // Read - implements `io.Reader`, which transparently decodes
  199. // the incoming AWS Signature V4 streaming signature.
  200. func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
  201. for {
  202. switch cr.state {
  203. case readChunkHeader:
  204. cr.readS3ChunkHeader()
  205. // If we're at the end of a chunk.
  206. if cr.n == 0 && cr.err == io.EOF {
  207. cr.state = readChunkTrailer
  208. cr.lastChunk = true
  209. continue
  210. }
  211. if cr.err != nil {
  212. return 0, cr.err
  213. }
  214. cr.state = readChunk
  215. case readChunkTrailer:
  216. cr.err = readCRLF(cr.reader)
  217. if cr.err != nil {
  218. return 0, errMalformedEncoding
  219. }
  220. cr.state = verifyChunk
  221. case readChunk:
  222. // There is no more space left in the request buffer.
  223. if len(buf) == 0 {
  224. return n, nil
  225. }
  226. rbuf := buf
  227. // The request buffer is larger than the current chunk size.
  228. // Read only the current chunk from the underlying reader.
  229. if uint64(len(rbuf)) > cr.n {
  230. rbuf = rbuf[:cr.n]
  231. }
  232. var n0 int
  233. n0, cr.err = cr.reader.Read(rbuf)
  234. if cr.err != nil {
  235. // We have lesser than chunk size advertised in chunkHeader, this is 'unexpected'.
  236. if cr.err == io.EOF {
  237. cr.err = io.ErrUnexpectedEOF
  238. }
  239. return 0, cr.err
  240. }
  241. // Calculate sha256.
  242. cr.chunkSHA256Writer.Write(rbuf[:n0])
  243. // Update the bytes read into request buffer so far.
  244. n += n0
  245. buf = buf[n0:]
  246. // Update bytes to be read of the current chunk before verifying chunk's signature.
  247. cr.n -= uint64(n0)
  248. // If we're at the end of a chunk.
  249. if cr.n == 0 {
  250. cr.state = readChunkTrailer
  251. continue
  252. }
  253. case verifyChunk:
  254. // Calculate the hashed chunk.
  255. hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))
  256. // Calculate the chunk signature.
  257. newSignature := getChunkSignature(cr.cred.SecretKey, cr.seedSignature, cr.region, cr.seedDate, hashedChunk)
  258. if !compareSignatureV4(cr.chunkSignature, newSignature) {
  259. // Chunk signature doesn't match we return signature does not match.
  260. cr.err = errors.New("chunk signature does not match")
  261. return 0, cr.err
  262. }
  263. // Newly calculated signature becomes the seed for the next chunk
  264. // this follows the chaining.
  265. cr.seedSignature = newSignature
  266. cr.chunkSHA256Writer.Reset()
  267. if cr.lastChunk {
  268. cr.state = eofChunk
  269. } else {
  270. cr.state = readChunkHeader
  271. }
  272. case eofChunk:
  273. return n, io.EOF
  274. }
  275. }
  276. }
  277. // readCRLF - check if reader only has '\r\n' CRLF character.
  278. // returns malformed encoding if it doesn't.
  279. func readCRLF(reader io.Reader) error {
  280. buf := make([]byte, 2)
  281. _, err := io.ReadFull(reader, buf[:2])
  282. if err != nil {
  283. return err
  284. }
  285. if buf[0] != '\r' || buf[1] != '\n' {
  286. return errMalformedEncoding
  287. }
  288. return nil
  289. }
  290. // Read a line of bytes (up to \n) from b.
  291. // Give up if the line exceeds maxLineLength.
  292. // The returned bytes are owned by the bufio.Reader
  293. // so they are only valid until the next bufio read.
  294. func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) {
  295. buf, err := b.ReadSlice('\n')
  296. if err != nil {
  297. // We always know when EOF is coming.
  298. // If the caller asked for a line, there should be a line.
  299. if err == io.EOF {
  300. err = io.ErrUnexpectedEOF
  301. } else if err == bufio.ErrBufferFull {
  302. err = errLineTooLong
  303. }
  304. return nil, nil, err
  305. }
  306. if len(buf) >= maxLineLength {
  307. return nil, nil, errLineTooLong
  308. }
  309. // Parse s3 specific chunk extension and fetch the values.
  310. hexChunkSize, hexChunkSignature := parseS3ChunkExtension(buf)
  311. return hexChunkSize, hexChunkSignature, nil
  312. }
  313. // trimTrailingWhitespace - trim trailing white space.
  314. func trimTrailingWhitespace(b []byte) []byte {
  315. for len(b) > 0 && isASCIISpace(b[len(b)-1]) {
  316. b = b[:len(b)-1]
  317. }
  318. return b
  319. }
  320. // isASCIISpace - is ascii space?
  321. func isASCIISpace(b byte) bool {
  322. return b == ' ' || b == '\t' || b == '\n' || b == '\r'
  323. }
  324. // Constant s3 chunk encoding signature.
  325. const s3ChunkSignatureStr = ";chunk-signature="
  326. // parses3ChunkExtension removes any s3 specific chunk-extension from buf.
  327. // For example,
  328. // "10000;chunk-signature=..." => "10000", "chunk-signature=..."
  329. func parseS3ChunkExtension(buf []byte) ([]byte, []byte) {
  330. buf = trimTrailingWhitespace(buf)
  331. semi := bytes.Index(buf, []byte(s3ChunkSignatureStr))
  332. // Chunk signature not found, return the whole buffer.
  333. if semi == -1 {
  334. return buf, nil
  335. }
  336. return buf[:semi], parseChunkSignature(buf[semi:])
  337. }
  338. // parseChunkSignature - parse chunk signature.
  339. func parseChunkSignature(chunk []byte) []byte {
  340. chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2)
  341. return chunkSplits[1]
  342. }
  343. // parse hex to uint64.
  344. func parseHexUint(v []byte) (n uint64, err error) {
  345. for i, b := range v {
  346. switch {
  347. case '0' <= b && b <= '9':
  348. b = b - '0'
  349. case 'a' <= b && b <= 'f':
  350. b = b - 'a' + 10
  351. case 'A' <= b && b <= 'F':
  352. b = b - 'A' + 10
  353. default:
  354. return 0, errors.New("invalid byte in chunk length")
  355. }
  356. if i == 16 {
  357. return 0, errors.New("http chunk length too large")
  358. }
  359. n <<= 4
  360. n |= uint64(b)
  361. }
  362. return
  363. }