diff options
author | Rutger Broekhoff | 2023-12-29 21:31:53 +0100 |
---|---|---|
committer | Rutger Broekhoff | 2023-12-29 21:31:53 +0100 |
commit | 404aeae4545d2426c089a5f8d5e82dae56f5212b (patch) | |
tree | 2d84e00af272b39fc04f3795ae06bc48970e57b5 /vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go | |
parent | 209d8b0187ed025dec9ac149ebcced3462877bff (diff) | |
download | gitolfs3-404aeae4545d2426c089a5f8d5e82dae56f5212b.tar.gz gitolfs3-404aeae4545d2426c089a5f8d5e82dae56f5212b.zip |
Make Nix builds work
Diffstat (limited to 'vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go')
-rw-r--r-- | vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go | 403 |
1 files changed, 403 insertions, 0 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go new file mode 100644 index 0000000..1c2f1dc --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-streaming.go | |||
@@ -0,0 +1,403 @@ | |||
1 | /* | ||
2 | * MinIO Go Library for Amazon S3 Compatible Cloud Storage | ||
3 | * Copyright 2017 MinIO, Inc. | ||
4 | * | ||
5 | * Licensed under the Apache License, Version 2.0 (the "License"); | ||
6 | * you may not use this file except in compliance with the License. | ||
7 | * You may obtain a copy of the License at | ||
8 | * | ||
9 | * http://www.apache.org/licenses/LICENSE-2.0 | ||
10 | * | ||
11 | * Unless required by applicable law or agreed to in writing, software | ||
12 | * distributed under the License is distributed on an "AS IS" BASIS, | ||
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
14 | * See the License for the specific language governing permissions and | ||
15 | * limitations under the License. | ||
16 | */ | ||
17 | |||
18 | package signer | ||
19 | |||
20 | import ( | ||
21 | "bytes" | ||
22 | "encoding/hex" | ||
23 | "fmt" | ||
24 | "io" | ||
25 | "net/http" | ||
26 | "strconv" | ||
27 | "strings" | ||
28 | "time" | ||
29 | |||
30 | md5simd "github.com/minio/md5-simd" | ||
31 | ) | ||
32 | |||
33 | // Reference for constants used below - | ||
34 | // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming | ||
35 | const ( | ||
36 | streamingSignAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" | ||
37 | streamingSignTrailerAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" | ||
38 | streamingPayloadHdr = "AWS4-HMAC-SHA256-PAYLOAD" | ||
39 | streamingTrailerHdr = "AWS4-HMAC-SHA256-TRAILER" | ||
40 | emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" | ||
41 | payloadChunkSize = 64 * 1024 | ||
42 | chunkSigConstLen = 17 // ";chunk-signature=" | ||
43 | signatureStrLen = 64 // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2" | ||
44 | crlfLen = 2 // CRLF | ||
45 | trailerKVSeparator = ":" | ||
46 | trailerSignature = "x-amz-trailer-signature" | ||
47 | ) | ||
48 | |||
49 | // Request headers to be ignored while calculating seed signature for | ||
50 | // a request. | ||
51 | var ignoredStreamingHeaders = map[string]bool{ | ||
52 | "Authorization": true, | ||
53 | "User-Agent": true, | ||
54 | "Content-Type": true, | ||
55 | } | ||
56 | |||
57 | // getSignedChunkLength - calculates the length of chunk metadata | ||
58 | func getSignedChunkLength(chunkDataSize int64) int64 { | ||
59 | return int64(len(fmt.Sprintf("%x", chunkDataSize))) + | ||
60 | chunkSigConstLen + | ||
61 | signatureStrLen + | ||
62 | crlfLen + | ||
63 | chunkDataSize + | ||
64 | crlfLen | ||
65 | } | ||
66 | |||
67 | // getStreamLength - calculates the length of the overall stream (data + metadata) | ||
68 | func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 { | ||
69 | if dataLen <= 0 { | ||
70 | return 0 | ||
71 | } | ||
72 | |||
73 | chunksCount := int64(dataLen / chunkSize) | ||
74 | remainingBytes := int64(dataLen % chunkSize) | ||
75 | streamLen := int64(0) | ||
76 | streamLen += chunksCount * getSignedChunkLength(chunkSize) | ||
77 | if remainingBytes > 0 { | ||
78 | streamLen += getSignedChunkLength(remainingBytes) | ||
79 | } | ||
80 | streamLen += getSignedChunkLength(0) | ||
81 | if len(trailers) > 0 { | ||
82 | for name, placeholder := range trailers { | ||
83 | if len(placeholder) > 0 { | ||
84 | streamLen += int64(len(name) + len(trailerKVSeparator) + len(placeholder[0]) + 1) | ||
85 | } | ||
86 | } | ||
87 | streamLen += int64(len(trailerSignature)+len(trailerKVSeparator)) + signatureStrLen + crlfLen + crlfLen | ||
88 | } | ||
89 | |||
90 | return streamLen | ||
91 | } | ||
92 | |||
93 | // buildChunkStringToSign - returns the string to sign given chunk data | ||
94 | // and previous signature. | ||
95 | func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string { | ||
96 | stringToSignParts := []string{ | ||
97 | streamingPayloadHdr, | ||
98 | t.Format(iso8601DateFormat), | ||
99 | getScope(region, t, ServiceTypeS3), | ||
100 | previousSig, | ||
101 | emptySHA256, | ||
102 | chunkChecksum, | ||
103 | } | ||
104 | |||
105 | return strings.Join(stringToSignParts, "\n") | ||
106 | } | ||
107 | |||
108 | // buildTrailerChunkStringToSign - returns the string to sign given chunk data | ||
109 | // and previous signature. | ||
110 | func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string { | ||
111 | stringToSignParts := []string{ | ||
112 | streamingTrailerHdr, | ||
113 | t.Format(iso8601DateFormat), | ||
114 | getScope(region, t, ServiceTypeS3), | ||
115 | previousSig, | ||
116 | chunkChecksum, | ||
117 | } | ||
118 | |||
119 | return strings.Join(stringToSignParts, "\n") | ||
120 | } | ||
121 | |||
122 | // prepareStreamingRequest - prepares a request with appropriate | ||
123 | // headers before computing the seed signature. | ||
124 | func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) { | ||
125 | // Set x-amz-content-sha256 header. | ||
126 | if len(req.Trailer) == 0 { | ||
127 | req.Header.Set("X-Amz-Content-Sha256", streamingSignAlgorithm) | ||
128 | } else { | ||
129 | req.Header.Set("X-Amz-Content-Sha256", streamingSignTrailerAlgorithm) | ||
130 | for k := range req.Trailer { | ||
131 | req.Header.Add("X-Amz-Trailer", strings.ToLower(k)) | ||
132 | } | ||
133 | req.TransferEncoding = []string{"aws-chunked"} | ||
134 | } | ||
135 | |||
136 | if sessionToken != "" { | ||
137 | req.Header.Set("X-Amz-Security-Token", sessionToken) | ||
138 | } | ||
139 | |||
140 | req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) | ||
141 | // Set content length with streaming signature for each chunk included. | ||
142 | req.ContentLength = getStreamLength(dataLen, int64(payloadChunkSize), req.Trailer) | ||
143 | req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLen, 10)) | ||
144 | } | ||
145 | |||
146 | // buildChunkHeader - returns the chunk header. | ||
147 | // e.g string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n | ||
148 | func buildChunkHeader(chunkLen int64, signature string) []byte { | ||
149 | return []byte(strconv.FormatInt(chunkLen, 16) + ";chunk-signature=" + signature + "\r\n") | ||
150 | } | ||
151 | |||
152 | // buildChunkSignature - returns chunk signature for a given chunk and previous signature. | ||
153 | func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region, | ||
154 | previousSignature, secretAccessKey string, | ||
155 | ) string { | ||
156 | chunkStringToSign := buildChunkStringToSign(reqTime, region, | ||
157 | previousSignature, chunkCheckSum) | ||
158 | signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) | ||
159 | return getSignature(signingKey, chunkStringToSign) | ||
160 | } | ||
161 | |||
162 | // buildChunkSignature - returns chunk signature for a given chunk and previous signature. | ||
163 | func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region, | ||
164 | previousSignature, secretAccessKey string, | ||
165 | ) string { | ||
166 | chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region, | ||
167 | previousSignature, chunkChecksum) | ||
168 | signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) | ||
169 | return getSignature(signingKey, chunkStringToSign) | ||
170 | } | ||
171 | |||
172 | // getSeedSignature - returns the seed signature for a given request. | ||
173 | func (s *StreamingReader) setSeedSignature(req *http.Request) { | ||
174 | // Get canonical request | ||
175 | canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req)) | ||
176 | |||
177 | // Get string to sign from canonical request. | ||
178 | stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3) | ||
179 | |||
180 | signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3) | ||
181 | |||
182 | // Calculate signature. | ||
183 | s.seedSignature = getSignature(signingKey, stringToSign) | ||
184 | } | ||
185 | |||
186 | // StreamingReader implements chunked upload signature as a reader on | ||
187 | // top of req.Body's ReaderCloser chunk header;data;... repeat | ||
188 | type StreamingReader struct { | ||
189 | accessKeyID string | ||
190 | secretAccessKey string | ||
191 | sessionToken string | ||
192 | region string | ||
193 | prevSignature string | ||
194 | seedSignature string | ||
195 | contentLen int64 // Content-Length from req header | ||
196 | baseReadCloser io.ReadCloser // underlying io.Reader | ||
197 | bytesRead int64 // bytes read from underlying io.Reader | ||
198 | buf bytes.Buffer // holds signed chunk | ||
199 | chunkBuf []byte // holds raw data read from req Body | ||
200 | chunkBufLen int // no. of bytes read so far into chunkBuf | ||
201 | done bool // done reading the underlying reader to EOF | ||
202 | reqTime time.Time | ||
203 | chunkNum int | ||
204 | totalChunks int | ||
205 | lastChunkSize int | ||
206 | trailer http.Header | ||
207 | sh256 md5simd.Hasher | ||
208 | } | ||
209 | |||
210 | // signChunk - signs a chunk read from s.baseReader of chunkLen size. | ||
211 | func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) { | ||
212 | // Compute chunk signature for next header | ||
213 | s.sh256.Reset() | ||
214 | s.sh256.Write(s.chunkBuf[:chunkLen]) | ||
215 | chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil)) | ||
216 | |||
217 | signature := buildChunkSignature(chunckChecksum, s.reqTime, | ||
218 | s.region, s.prevSignature, s.secretAccessKey) | ||
219 | |||
220 | // For next chunk signature computation | ||
221 | s.prevSignature = signature | ||
222 | |||
223 | // Write chunk header into streaming buffer | ||
224 | chunkHdr := buildChunkHeader(int64(chunkLen), signature) | ||
225 | s.buf.Write(chunkHdr) | ||
226 | |||
227 | // Write chunk data into streaming buffer | ||
228 | s.buf.Write(s.chunkBuf[:chunkLen]) | ||
229 | |||
230 | // Write the chunk trailer. | ||
231 | if addCrLf { | ||
232 | s.buf.Write([]byte("\r\n")) | ||
233 | } | ||
234 | |||
235 | // Reset chunkBufLen for next chunk read. | ||
236 | s.chunkBufLen = 0 | ||
237 | s.chunkNum++ | ||
238 | } | ||
239 | |||
240 | // addSignedTrailer - adds a trailer with the provided headers, | ||
241 | // then signs a chunk and adds it to output. | ||
242 | func (s *StreamingReader) addSignedTrailer(h http.Header) { | ||
243 | olen := len(s.chunkBuf) | ||
244 | s.chunkBuf = s.chunkBuf[:0] | ||
245 | for k, v := range h { | ||
246 | s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...) | ||
247 | } | ||
248 | |||
249 | s.sh256.Reset() | ||
250 | s.sh256.Write(s.chunkBuf) | ||
251 | chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil)) | ||
252 | // Compute chunk signature | ||
253 | signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime, | ||
254 | s.region, s.prevSignature, s.secretAccessKey) | ||
255 | |||
256 | // For next chunk signature computation | ||
257 | s.prevSignature = signature | ||
258 | |||
259 | s.buf.Write(s.chunkBuf) | ||
260 | s.buf.WriteString("\r\n" + trailerSignature + trailerKVSeparator + signature + "\r\n\r\n") | ||
261 | |||
262 | // Reset chunkBufLen for next chunk read. | ||
263 | s.chunkBuf = s.chunkBuf[:olen] | ||
264 | s.chunkBufLen = 0 | ||
265 | s.chunkNum++ | ||
266 | } | ||
267 | |||
268 | // setStreamingAuthHeader - builds and sets authorization header value | ||
269 | // for streaming signature. | ||
270 | func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) { | ||
271 | credential := GetCredential(s.accessKeyID, s.region, s.reqTime, ServiceTypeS3) | ||
272 | authParts := []string{ | ||
273 | signV4Algorithm + " Credential=" + credential, | ||
274 | "SignedHeaders=" + getSignedHeaders(*req, ignoredStreamingHeaders), | ||
275 | "Signature=" + s.seedSignature, | ||
276 | } | ||
277 | |||
278 | // Set authorization header. | ||
279 | auth := strings.Join(authParts, ",") | ||
280 | req.Header.Set("Authorization", auth) | ||
281 | } | ||
282 | |||
283 | // StreamingSignV4 - provides chunked upload signatureV4 support by | ||
284 | // implementing io.Reader. | ||
285 | func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken, | ||
286 | region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher, | ||
287 | ) *http.Request { | ||
288 | // Set headers needed for streaming signature. | ||
289 | prepareStreamingRequest(req, sessionToken, dataLen, reqTime) | ||
290 | |||
291 | if req.Body == nil { | ||
292 | req.Body = io.NopCloser(bytes.NewReader([]byte(""))) | ||
293 | } | ||
294 | |||
295 | stReader := &StreamingReader{ | ||
296 | baseReadCloser: req.Body, | ||
297 | accessKeyID: accessKeyID, | ||
298 | secretAccessKey: secretAccessKey, | ||
299 | sessionToken: sessionToken, | ||
300 | region: region, | ||
301 | reqTime: reqTime, | ||
302 | chunkBuf: make([]byte, payloadChunkSize), | ||
303 | contentLen: dataLen, | ||
304 | chunkNum: 1, | ||
305 | totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1, | ||
306 | lastChunkSize: int(dataLen % payloadChunkSize), | ||
307 | sh256: sh256, | ||
308 | } | ||
309 | if len(req.Trailer) > 0 { | ||
310 | stReader.trailer = req.Trailer | ||
311 | // Remove... | ||
312 | req.Trailer = nil | ||
313 | } | ||
314 | |||
315 | // Add the request headers required for chunk upload signing. | ||
316 | |||
317 | // Compute the seed signature. | ||
318 | stReader.setSeedSignature(req) | ||
319 | |||
320 | // Set the authorization header with the seed signature. | ||
321 | stReader.setStreamingAuthHeader(req) | ||
322 | |||
323 | // Set seed signature as prevSignature for subsequent | ||
324 | // streaming signing process. | ||
325 | stReader.prevSignature = stReader.seedSignature | ||
326 | req.Body = stReader | ||
327 | |||
328 | return req | ||
329 | } | ||
330 | |||
331 | // Read - this method performs chunk upload signature providing a | ||
332 | // io.Reader interface. | ||
333 | func (s *StreamingReader) Read(buf []byte) (int, error) { | ||
334 | switch { | ||
335 | // After the last chunk is read from underlying reader, we | ||
336 | // never re-fill s.buf. | ||
337 | case s.done: | ||
338 | |||
339 | // s.buf will be (re-)filled with next chunk when has lesser | ||
340 | // bytes than asked for. | ||
341 | case s.buf.Len() < len(buf): | ||
342 | s.chunkBufLen = 0 | ||
343 | for { | ||
344 | n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:]) | ||
345 | // Usually we validate `err` first, but in this case | ||
346 | // we are validating n > 0 for the following reasons. | ||
347 | // | ||
348 | // 1. n > 0, err is one of io.EOF, nil (near end of stream) | ||
349 | // A Reader returning a non-zero number of bytes at the end | ||
350 | // of the input stream may return either err == EOF or err == nil | ||
351 | // | ||
352 | // 2. n == 0, err is io.EOF (actual end of stream) | ||
353 | // | ||
354 | // Callers should always process the n > 0 bytes returned | ||
355 | // before considering the error err. | ||
356 | if n1 > 0 { | ||
357 | s.chunkBufLen += n1 | ||
358 | s.bytesRead += int64(n1) | ||
359 | |||
360 | if s.chunkBufLen == payloadChunkSize || | ||
361 | (s.chunkNum == s.totalChunks-1 && | ||
362 | s.chunkBufLen == s.lastChunkSize) { | ||
363 | // Sign the chunk and write it to s.buf. | ||
364 | s.signChunk(s.chunkBufLen, true) | ||
365 | break | ||
366 | } | ||
367 | } | ||
368 | if err != nil { | ||
369 | if err == io.EOF { | ||
370 | // No more data left in baseReader - last chunk. | ||
371 | // Done reading the last chunk from baseReader. | ||
372 | s.done = true | ||
373 | |||
374 | // bytes read from baseReader different than | ||
375 | // content length provided. | ||
376 | if s.bytesRead != s.contentLen { | ||
377 | return 0, fmt.Errorf("http: ContentLength=%d with Body length %d", s.contentLen, s.bytesRead) | ||
378 | } | ||
379 | |||
380 | // Sign the chunk and write it to s.buf. | ||
381 | s.signChunk(0, len(s.trailer) == 0) | ||
382 | if len(s.trailer) > 0 { | ||
383 | // Trailer must be set now. | ||
384 | s.addSignedTrailer(s.trailer) | ||
385 | } | ||
386 | break | ||
387 | } | ||
388 | return 0, err | ||
389 | } | ||
390 | |||
391 | } | ||
392 | } | ||
393 | return s.buf.Read(buf) | ||
394 | } | ||
395 | |||
396 | // Close - this method makes underlying io.ReadCloser's Close method available. | ||
397 | func (s *StreamingReader) Close() error { | ||
398 | if s.sh256 != nil { | ||
399 | s.sh256.Close() | ||
400 | s.sh256 = nil | ||
401 | } | ||
402 | return s.baseReadCloser.Close() | ||
403 | } | ||