diff options
author | Rutger Broekhoff | 2023-12-29 21:31:53 +0100 |
---|---|---|
committer | Rutger Broekhoff | 2023-12-29 21:31:53 +0100 |
commit | 404aeae4545d2426c089a5f8d5e82dae56f5212b (patch) | |
tree | 2d84e00af272b39fc04f3795ae06bc48970e57b5 /vendor/github.com/minio/minio-go/v7/utils.go | |
parent | 209d8b0187ed025dec9ac149ebcced3462877bff (diff) | |
download | gitolfs3-404aeae4545d2426c089a5f8d5e82dae56f5212b.tar.gz gitolfs3-404aeae4545d2426c089a5f8d5e82dae56f5212b.zip |
Make Nix builds work
Diffstat (limited to 'vendor/github.com/minio/minio-go/v7/utils.go')
-rw-r--r-- | vendor/github.com/minio/minio-go/v7/utils.go | 693 |
1 files changed, 693 insertions, 0 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/utils.go b/vendor/github.com/minio/minio-go/v7/utils.go new file mode 100644 index 0000000..e39eba0 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/utils.go | |||
@@ -0,0 +1,693 @@ | |||
1 | /* | ||
2 | * MinIO Go Library for Amazon S3 Compatible Cloud Storage | ||
3 | * Copyright 2015-2017 MinIO, Inc. | ||
4 | * | ||
5 | * Licensed under the Apache License, Version 2.0 (the "License"); | ||
6 | * you may not use this file except in compliance with the License. | ||
7 | * You may obtain a copy of the License at | ||
8 | * | ||
9 | * http://www.apache.org/licenses/LICENSE-2.0 | ||
10 | * | ||
11 | * Unless required by applicable law or agreed to in writing, software | ||
12 | * distributed under the License is distributed on an "AS IS" BASIS, | ||
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
14 | * See the License for the specific language governing permissions and | ||
15 | * limitations under the License. | ||
16 | */ | ||
17 | |||
18 | package minio | ||
19 | |||
20 | import ( | ||
21 | "context" | ||
22 | "crypto/md5" | ||
23 | fipssha256 "crypto/sha256" | ||
24 | "encoding/base64" | ||
25 | "encoding/hex" | ||
26 | "encoding/xml" | ||
27 | "errors" | ||
28 | "fmt" | ||
29 | "hash" | ||
30 | "io" | ||
31 | "math/rand" | ||
32 | "net" | ||
33 | "net/http" | ||
34 | "net/url" | ||
35 | "regexp" | ||
36 | "strconv" | ||
37 | "strings" | ||
38 | "sync" | ||
39 | "time" | ||
40 | |||
41 | md5simd "github.com/minio/md5-simd" | ||
42 | "github.com/minio/minio-go/v7/pkg/encrypt" | ||
43 | "github.com/minio/minio-go/v7/pkg/s3utils" | ||
44 | "github.com/minio/sha256-simd" | ||
45 | ) | ||
46 | |||
47 | func trimEtag(etag string) string { | ||
48 | etag = strings.TrimPrefix(etag, "\"") | ||
49 | return strings.TrimSuffix(etag, "\"") | ||
50 | } | ||
51 | |||
52 | var expirationRegex = regexp.MustCompile(`expiry-date="(.*?)", rule-id="(.*?)"`) | ||
53 | |||
54 | func amzExpirationToExpiryDateRuleID(expiration string) (time.Time, string) { | ||
55 | if matches := expirationRegex.FindStringSubmatch(expiration); len(matches) == 3 { | ||
56 | expTime, err := parseRFC7231Time(matches[1]) | ||
57 | if err != nil { | ||
58 | return time.Time{}, "" | ||
59 | } | ||
60 | return expTime, matches[2] | ||
61 | } | ||
62 | return time.Time{}, "" | ||
63 | } | ||
64 | |||
65 | var restoreRegex = regexp.MustCompile(`ongoing-request="(.*?)"(, expiry-date="(.*?)")?`) | ||
66 | |||
67 | func amzRestoreToStruct(restore string) (ongoing bool, expTime time.Time, err error) { | ||
68 | matches := restoreRegex.FindStringSubmatch(restore) | ||
69 | if len(matches) != 4 { | ||
70 | return false, time.Time{}, errors.New("unexpected restore header") | ||
71 | } | ||
72 | ongoing, err = strconv.ParseBool(matches[1]) | ||
73 | if err != nil { | ||
74 | return false, time.Time{}, err | ||
75 | } | ||
76 | if matches[3] != "" { | ||
77 | expTime, err = parseRFC7231Time(matches[3]) | ||
78 | if err != nil { | ||
79 | return false, time.Time{}, err | ||
80 | } | ||
81 | } | ||
82 | return | ||
83 | } | ||
84 | |||
85 | // xmlDecoder provide decoded value in xml. | ||
86 | func xmlDecoder(body io.Reader, v interface{}) error { | ||
87 | d := xml.NewDecoder(body) | ||
88 | return d.Decode(v) | ||
89 | } | ||
90 | |||
91 | // sum256 calculate sha256sum for an input byte array, returns hex encoded. | ||
92 | func sum256Hex(data []byte) string { | ||
93 | hash := newSHA256Hasher() | ||
94 | defer hash.Close() | ||
95 | hash.Write(data) | ||
96 | return hex.EncodeToString(hash.Sum(nil)) | ||
97 | } | ||
98 | |||
99 | // sumMD5Base64 calculate md5sum for an input byte array, returns base64 encoded. | ||
100 | func sumMD5Base64(data []byte) string { | ||
101 | hash := newMd5Hasher() | ||
102 | defer hash.Close() | ||
103 | hash.Write(data) | ||
104 | return base64.StdEncoding.EncodeToString(hash.Sum(nil)) | ||
105 | } | ||
106 | |||
107 | // getEndpointURL - construct a new endpoint. | ||
108 | func getEndpointURL(endpoint string, secure bool) (*url.URL, error) { | ||
109 | // If secure is false, use 'http' scheme. | ||
110 | scheme := "https" | ||
111 | if !secure { | ||
112 | scheme = "http" | ||
113 | } | ||
114 | |||
115 | // Construct a secured endpoint URL. | ||
116 | endpointURLStr := scheme + "://" + endpoint | ||
117 | endpointURL, err := url.Parse(endpointURLStr) | ||
118 | if err != nil { | ||
119 | return nil, err | ||
120 | } | ||
121 | |||
122 | // Validate incoming endpoint URL. | ||
123 | if err := isValidEndpointURL(*endpointURL); err != nil { | ||
124 | return nil, err | ||
125 | } | ||
126 | return endpointURL, nil | ||
127 | } | ||
128 | |||
129 | // closeResponse close non nil response with any response Body. | ||
130 | // convenient wrapper to drain any remaining data on response body. | ||
131 | // | ||
132 | // Subsequently this allows golang http RoundTripper | ||
133 | // to re-use the same connection for future requests. | ||
134 | func closeResponse(resp *http.Response) { | ||
135 | // Callers should close resp.Body when done reading from it. | ||
136 | // If resp.Body is not closed, the Client's underlying RoundTripper | ||
137 | // (typically Transport) may not be able to re-use a persistent TCP | ||
138 | // connection to the server for a subsequent "keep-alive" request. | ||
139 | if resp != nil && resp.Body != nil { | ||
140 | // Drain any remaining Body and then close the connection. | ||
141 | // Without this closing connection would disallow re-using | ||
142 | // the same connection for future uses. | ||
143 | // - http://stackoverflow.com/a/17961593/4465767 | ||
144 | io.Copy(io.Discard, resp.Body) | ||
145 | resp.Body.Close() | ||
146 | } | ||
147 | } | ||
148 | |||
149 | var ( | ||
150 | // Hex encoded string of nil sha256sum bytes. | ||
151 | emptySHA256Hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" | ||
152 | |||
153 | // Sentinel URL is the default url value which is invalid. | ||
154 | sentinelURL = url.URL{} | ||
155 | ) | ||
156 | |||
157 | // Verify if input endpoint URL is valid. | ||
158 | func isValidEndpointURL(endpointURL url.URL) error { | ||
159 | if endpointURL == sentinelURL { | ||
160 | return errInvalidArgument("Endpoint url cannot be empty.") | ||
161 | } | ||
162 | if endpointURL.Path != "/" && endpointURL.Path != "" { | ||
163 | return errInvalidArgument("Endpoint url cannot have fully qualified paths.") | ||
164 | } | ||
165 | host := endpointURL.Hostname() | ||
166 | if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) { | ||
167 | msg := "Endpoint: " + endpointURL.Host + " does not follow ip address or domain name standards." | ||
168 | return errInvalidArgument(msg) | ||
169 | } | ||
170 | |||
171 | if strings.Contains(host, ".s3.amazonaws.com") { | ||
172 | if !s3utils.IsAmazonEndpoint(endpointURL) { | ||
173 | return errInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") | ||
174 | } | ||
175 | } | ||
176 | if strings.Contains(host, ".googleapis.com") { | ||
177 | if !s3utils.IsGoogleEndpoint(endpointURL) { | ||
178 | return errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.") | ||
179 | } | ||
180 | } | ||
181 | return nil | ||
182 | } | ||
183 | |||
184 | // Verify if input expires value is valid. | ||
185 | func isValidExpiry(expires time.Duration) error { | ||
186 | expireSeconds := int64(expires / time.Second) | ||
187 | if expireSeconds < 1 { | ||
188 | return errInvalidArgument("Expires cannot be lesser than 1 second.") | ||
189 | } | ||
190 | if expireSeconds > 604800 { | ||
191 | return errInvalidArgument("Expires cannot be greater than 7 days.") | ||
192 | } | ||
193 | return nil | ||
194 | } | ||
195 | |||
196 | // Extract only necessary metadata header key/values by | ||
197 | // filtering them out with a list of custom header keys. | ||
198 | func extractObjMetadata(header http.Header) http.Header { | ||
199 | preserveKeys := []string{ | ||
200 | "Content-Type", | ||
201 | "Cache-Control", | ||
202 | "Content-Encoding", | ||
203 | "Content-Language", | ||
204 | "Content-Disposition", | ||
205 | "X-Amz-Storage-Class", | ||
206 | "X-Amz-Object-Lock-Mode", | ||
207 | "X-Amz-Object-Lock-Retain-Until-Date", | ||
208 | "X-Amz-Object-Lock-Legal-Hold", | ||
209 | "X-Amz-Website-Redirect-Location", | ||
210 | "X-Amz-Server-Side-Encryption", | ||
211 | "X-Amz-Tagging-Count", | ||
212 | "X-Amz-Meta-", | ||
213 | // Add new headers to be preserved. | ||
214 | // if you add new headers here, please extend | ||
215 | // PutObjectOptions{} to preserve them | ||
216 | // upon upload as well. | ||
217 | } | ||
218 | filteredHeader := make(http.Header) | ||
219 | for k, v := range header { | ||
220 | var found bool | ||
221 | for _, prefix := range preserveKeys { | ||
222 | if !strings.HasPrefix(k, prefix) { | ||
223 | continue | ||
224 | } | ||
225 | found = true | ||
226 | break | ||
227 | } | ||
228 | if found { | ||
229 | filteredHeader[k] = v | ||
230 | } | ||
231 | } | ||
232 | return filteredHeader | ||
233 | } | ||
234 | |||
235 | const ( | ||
236 | // RFC 7231#section-7.1.1.1 timetamp format. e.g Tue, 29 Apr 2014 18:30:38 GMT | ||
237 | rfc822TimeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" | ||
238 | rfc822TimeFormatSingleDigitDay = "Mon, _2 Jan 2006 15:04:05 GMT" | ||
239 | rfc822TimeFormatSingleDigitDayTwoDigitYear = "Mon, _2 Jan 06 15:04:05 GMT" | ||
240 | ) | ||
241 | |||
242 | func parseTime(t string, formats ...string) (time.Time, error) { | ||
243 | for _, format := range formats { | ||
244 | tt, err := time.Parse(format, t) | ||
245 | if err == nil { | ||
246 | return tt, nil | ||
247 | } | ||
248 | } | ||
249 | return time.Time{}, fmt.Errorf("unable to parse %s in any of the input formats: %s", t, formats) | ||
250 | } | ||
251 | |||
252 | func parseRFC7231Time(lastModified string) (time.Time, error) { | ||
253 | return parseTime(lastModified, rfc822TimeFormat, rfc822TimeFormatSingleDigitDay, rfc822TimeFormatSingleDigitDayTwoDigitYear) | ||
254 | } | ||
255 | |||
256 | // ToObjectInfo converts http header values into ObjectInfo type, | ||
257 | // extracts metadata and fills in all the necessary fields in ObjectInfo. | ||
258 | func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, error) { | ||
259 | var err error | ||
260 | // Trim off the odd double quotes from ETag in the beginning and end. | ||
261 | etag := trimEtag(h.Get("ETag")) | ||
262 | |||
263 | // Parse content length is exists | ||
264 | var size int64 = -1 | ||
265 | contentLengthStr := h.Get("Content-Length") | ||
266 | if contentLengthStr != "" { | ||
267 | size, err = strconv.ParseInt(contentLengthStr, 10, 64) | ||
268 | if err != nil { | ||
269 | // Content-Length is not valid | ||
270 | return ObjectInfo{}, ErrorResponse{ | ||
271 | Code: "InternalError", | ||
272 | Message: fmt.Sprintf("Content-Length is not an integer, failed with %v", err), | ||
273 | BucketName: bucketName, | ||
274 | Key: objectName, | ||
275 | RequestID: h.Get("x-amz-request-id"), | ||
276 | HostID: h.Get("x-amz-id-2"), | ||
277 | Region: h.Get("x-amz-bucket-region"), | ||
278 | } | ||
279 | } | ||
280 | } | ||
281 | |||
282 | // Parse Last-Modified has http time format. | ||
283 | mtime, err := parseRFC7231Time(h.Get("Last-Modified")) | ||
284 | if err != nil { | ||
285 | return ObjectInfo{}, ErrorResponse{ | ||
286 | Code: "InternalError", | ||
287 | Message: fmt.Sprintf("Last-Modified time format is invalid, failed with %v", err), | ||
288 | BucketName: bucketName, | ||
289 | Key: objectName, | ||
290 | RequestID: h.Get("x-amz-request-id"), | ||
291 | HostID: h.Get("x-amz-id-2"), | ||
292 | Region: h.Get("x-amz-bucket-region"), | ||
293 | } | ||
294 | } | ||
295 | |||
296 | // Fetch content type if any present. | ||
297 | contentType := strings.TrimSpace(h.Get("Content-Type")) | ||
298 | if contentType == "" { | ||
299 | contentType = "application/octet-stream" | ||
300 | } | ||
301 | |||
302 | expiryStr := h.Get("Expires") | ||
303 | var expiry time.Time | ||
304 | if expiryStr != "" { | ||
305 | expiry, err = parseRFC7231Time(expiryStr) | ||
306 | if err != nil { | ||
307 | return ObjectInfo{}, ErrorResponse{ | ||
308 | Code: "InternalError", | ||
309 | Message: fmt.Sprintf("'Expiry' is not in supported format: %v", err), | ||
310 | BucketName: bucketName, | ||
311 | Key: objectName, | ||
312 | RequestID: h.Get("x-amz-request-id"), | ||
313 | HostID: h.Get("x-amz-id-2"), | ||
314 | Region: h.Get("x-amz-bucket-region"), | ||
315 | } | ||
316 | } | ||
317 | } | ||
318 | |||
319 | metadata := extractObjMetadata(h) | ||
320 | userMetadata := make(map[string]string) | ||
321 | for k, v := range metadata { | ||
322 | if strings.HasPrefix(k, "X-Amz-Meta-") { | ||
323 | userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0] | ||
324 | } | ||
325 | } | ||
326 | userTags := s3utils.TagDecode(h.Get(amzTaggingHeader)) | ||
327 | |||
328 | var tagCount int | ||
329 | if count := h.Get(amzTaggingCount); count != "" { | ||
330 | tagCount, err = strconv.Atoi(count) | ||
331 | if err != nil { | ||
332 | return ObjectInfo{}, ErrorResponse{ | ||
333 | Code: "InternalError", | ||
334 | Message: fmt.Sprintf("x-amz-tagging-count is not an integer, failed with %v", err), | ||
335 | BucketName: bucketName, | ||
336 | Key: objectName, | ||
337 | RequestID: h.Get("x-amz-request-id"), | ||
338 | HostID: h.Get("x-amz-id-2"), | ||
339 | Region: h.Get("x-amz-bucket-region"), | ||
340 | } | ||
341 | } | ||
342 | } | ||
343 | |||
344 | // Nil if not found | ||
345 | var restore *RestoreInfo | ||
346 | if restoreHdr := h.Get(amzRestore); restoreHdr != "" { | ||
347 | ongoing, expTime, err := amzRestoreToStruct(restoreHdr) | ||
348 | if err != nil { | ||
349 | return ObjectInfo{}, err | ||
350 | } | ||
351 | restore = &RestoreInfo{OngoingRestore: ongoing, ExpiryTime: expTime} | ||
352 | } | ||
353 | |||
354 | // extract lifecycle expiry date and rule ID | ||
355 | expTime, ruleID := amzExpirationToExpiryDateRuleID(h.Get(amzExpiration)) | ||
356 | |||
357 | deleteMarker := h.Get(amzDeleteMarker) == "true" | ||
358 | |||
359 | // Save object metadata info. | ||
360 | return ObjectInfo{ | ||
361 | ETag: etag, | ||
362 | Key: objectName, | ||
363 | Size: size, | ||
364 | LastModified: mtime, | ||
365 | ContentType: contentType, | ||
366 | Expires: expiry, | ||
367 | VersionID: h.Get(amzVersionID), | ||
368 | IsDeleteMarker: deleteMarker, | ||
369 | ReplicationStatus: h.Get(amzReplicationStatus), | ||
370 | Expiration: expTime, | ||
371 | ExpirationRuleID: ruleID, | ||
372 | // Extract only the relevant header keys describing the object. | ||
373 | // following function filters out a list of standard set of keys | ||
374 | // which are not part of object metadata. | ||
375 | Metadata: metadata, | ||
376 | UserMetadata: userMetadata, | ||
377 | UserTags: userTags, | ||
378 | UserTagCount: tagCount, | ||
379 | Restore: restore, | ||
380 | |||
381 | // Checksum values | ||
382 | ChecksumCRC32: h.Get("x-amz-checksum-crc32"), | ||
383 | ChecksumCRC32C: h.Get("x-amz-checksum-crc32c"), | ||
384 | ChecksumSHA1: h.Get("x-amz-checksum-sha1"), | ||
385 | ChecksumSHA256: h.Get("x-amz-checksum-sha256"), | ||
386 | }, nil | ||
387 | } | ||
388 | |||
389 | var readFull = func(r io.Reader, buf []byte) (n int, err error) { | ||
390 | // ReadFull reads exactly len(buf) bytes from r into buf. | ||
391 | // It returns the number of bytes copied and an error if | ||
392 | // fewer bytes were read. The error is EOF only if no bytes | ||
393 | // were read. If an EOF happens after reading some but not | ||
394 | // all the bytes, ReadFull returns ErrUnexpectedEOF. | ||
395 | // On return, n == len(buf) if and only if err == nil. | ||
396 | // If r returns an error having read at least len(buf) bytes, | ||
397 | // the error is dropped. | ||
398 | for n < len(buf) && err == nil { | ||
399 | var nn int | ||
400 | nn, err = r.Read(buf[n:]) | ||
401 | // Some spurious io.Reader's return | ||
402 | // io.ErrUnexpectedEOF when nn == 0 | ||
403 | // this behavior is undocumented | ||
404 | // so we are on purpose not using io.ReadFull | ||
405 | // implementation because this can lead | ||
406 | // to custom handling, to avoid that | ||
407 | // we simply modify the original io.ReadFull | ||
408 | // implementation to avoid this issue. | ||
409 | // io.ErrUnexpectedEOF with nn == 0 really | ||
410 | // means that io.EOF | ||
411 | if err == io.ErrUnexpectedEOF && nn == 0 { | ||
412 | err = io.EOF | ||
413 | } | ||
414 | n += nn | ||
415 | } | ||
416 | if n >= len(buf) { | ||
417 | err = nil | ||
418 | } else if n > 0 && err == io.EOF { | ||
419 | err = io.ErrUnexpectedEOF | ||
420 | } | ||
421 | return | ||
422 | } | ||
423 | |||
424 | // regCred matches credential string in HTTP header | ||
425 | var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/") | ||
426 | |||
427 | // regCred matches signature string in HTTP header | ||
428 | var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)") | ||
429 | |||
430 | // Redact out signature value from authorization string. | ||
431 | func redactSignature(origAuth string) string { | ||
432 | if !strings.HasPrefix(origAuth, signV4Algorithm) { | ||
433 | // Set a temporary redacted auth | ||
434 | return "AWS **REDACTED**:**REDACTED**" | ||
435 | } | ||
436 | |||
437 | // Signature V4 authorization header. | ||
438 | |||
439 | // Strip out accessKeyID from: | ||
440 | // Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request | ||
441 | newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/") | ||
442 | |||
443 | // Strip out 256-bit signature from: Signature=<256-bit signature> | ||
444 | return regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**") | ||
445 | } | ||
446 | |||
447 | // Get default location returns the location based on the input | ||
448 | // URL `u`, if region override is provided then all location | ||
449 | // defaults to regionOverride. | ||
450 | // | ||
451 | // If no other cases match then the location is set to `us-east-1` | ||
452 | // as a last resort. | ||
453 | func getDefaultLocation(u url.URL, regionOverride string) (location string) { | ||
454 | if regionOverride != "" { | ||
455 | return regionOverride | ||
456 | } | ||
457 | region := s3utils.GetRegionFromURL(u) | ||
458 | if region == "" { | ||
459 | region = "us-east-1" | ||
460 | } | ||
461 | return region | ||
462 | } | ||
463 | |||
464 | var supportedHeaders = map[string]bool{ | ||
465 | "content-type": true, | ||
466 | "cache-control": true, | ||
467 | "content-encoding": true, | ||
468 | "content-disposition": true, | ||
469 | "content-language": true, | ||
470 | "x-amz-website-redirect-location": true, | ||
471 | "x-amz-object-lock-mode": true, | ||
472 | "x-amz-metadata-directive": true, | ||
473 | "x-amz-object-lock-retain-until-date": true, | ||
474 | "expires": true, | ||
475 | "x-amz-replication-status": true, | ||
476 | // Add more supported headers here. | ||
477 | // Must be lower case. | ||
478 | } | ||
479 | |||
480 | // isStorageClassHeader returns true if the header is a supported storage class header | ||
481 | func isStorageClassHeader(headerKey string) bool { | ||
482 | return strings.EqualFold(amzStorageClass, headerKey) | ||
483 | } | ||
484 | |||
485 | // isStandardHeader returns true if header is a supported header and not a custom header | ||
486 | func isStandardHeader(headerKey string) bool { | ||
487 | return supportedHeaders[strings.ToLower(headerKey)] | ||
488 | } | ||
489 | |||
490 | // sseHeaders is list of server side encryption headers | ||
491 | var sseHeaders = map[string]bool{ | ||
492 | "x-amz-server-side-encryption": true, | ||
493 | "x-amz-server-side-encryption-aws-kms-key-id": true, | ||
494 | "x-amz-server-side-encryption-context": true, | ||
495 | "x-amz-server-side-encryption-customer-algorithm": true, | ||
496 | "x-amz-server-side-encryption-customer-key": true, | ||
497 | "x-amz-server-side-encryption-customer-key-md5": true, | ||
498 | // Add more supported headers here. | ||
499 | // Must be lower case. | ||
500 | } | ||
501 | |||
502 | // isSSEHeader returns true if header is a server side encryption header. | ||
503 | func isSSEHeader(headerKey string) bool { | ||
504 | return sseHeaders[strings.ToLower(headerKey)] | ||
505 | } | ||
506 | |||
507 | // isAmzHeader returns true if header is a x-amz-meta-* or x-amz-acl header. | ||
508 | func isAmzHeader(headerKey string) bool { | ||
509 | key := strings.ToLower(headerKey) | ||
510 | |||
511 | return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey) || strings.HasPrefix(key, "x-amz-checksum-") | ||
512 | } | ||
513 | |||
514 | // supportedQueryValues is a list of query strings that can be passed in when using GetObject. | ||
515 | var supportedQueryValues = map[string]bool{ | ||
516 | "partNumber": true, | ||
517 | "versionId": true, | ||
518 | "response-cache-control": true, | ||
519 | "response-content-disposition": true, | ||
520 | "response-content-encoding": true, | ||
521 | "response-content-language": true, | ||
522 | "response-content-type": true, | ||
523 | "response-expires": true, | ||
524 | } | ||
525 | |||
526 | // isStandardQueryValue will return true when the passed in query string parameter is supported rather than customized. | ||
527 | func isStandardQueryValue(qsKey string) bool { | ||
528 | return supportedQueryValues[qsKey] | ||
529 | } | ||
530 | |||
531 | // Per documentation at https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html#LogFormatCustom, the | ||
532 | // set of query params starting with "x-" are ignored by S3. | ||
533 | const allowedCustomQueryPrefix = "x-" | ||
534 | |||
535 | func isCustomQueryValue(qsKey string) bool { | ||
536 | return strings.HasPrefix(qsKey, allowedCustomQueryPrefix) | ||
537 | } | ||
538 | |||
539 | var ( | ||
540 | md5Pool = sync.Pool{New: func() interface{} { return md5.New() }} | ||
541 | sha256Pool = sync.Pool{New: func() interface{} { return sha256.New() }} | ||
542 | ) | ||
543 | |||
544 | func newMd5Hasher() md5simd.Hasher { | ||
545 | return &hashWrapper{Hash: md5Pool.Get().(hash.Hash), isMD5: true} | ||
546 | } | ||
547 | |||
548 | func newSHA256Hasher() md5simd.Hasher { | ||
549 | if encrypt.FIPS { | ||
550 | return &hashWrapper{Hash: fipssha256.New(), isSHA256: true} | ||
551 | } | ||
552 | return &hashWrapper{Hash: sha256Pool.Get().(hash.Hash), isSHA256: true} | ||
553 | } | ||
554 | |||
555 | // hashWrapper implements the md5simd.Hasher interface. | ||
556 | type hashWrapper struct { | ||
557 | hash.Hash | ||
558 | isMD5 bool | ||
559 | isSHA256 bool | ||
560 | } | ||
561 | |||
562 | // Close will put the hasher back into the pool. | ||
563 | func (m *hashWrapper) Close() { | ||
564 | if m.isMD5 && m.Hash != nil { | ||
565 | m.Reset() | ||
566 | md5Pool.Put(m.Hash) | ||
567 | } | ||
568 | if m.isSHA256 && m.Hash != nil { | ||
569 | m.Reset() | ||
570 | sha256Pool.Put(m.Hash) | ||
571 | } | ||
572 | m.Hash = nil | ||
573 | } | ||
574 | |||
575 | const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" | ||
576 | const ( | ||
577 | letterIdxBits = 6 // 6 bits to represent a letter index | ||
578 | letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits | ||
579 | letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits | ||
580 | ) | ||
581 | |||
582 | // randString generates random names and prepends them with a known prefix. | ||
583 | func randString(n int, src rand.Source, prefix string) string { | ||
584 | b := make([]byte, n) | ||
585 | // A rand.Int63() generates 63 random bits, enough for letterIdxMax letters! | ||
586 | for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; { | ||
587 | if remain == 0 { | ||
588 | cache, remain = src.Int63(), letterIdxMax | ||
589 | } | ||
590 | if idx := int(cache & letterIdxMask); idx < len(letterBytes) { | ||
591 | b[i] = letterBytes[idx] | ||
592 | i-- | ||
593 | } | ||
594 | cache >>= letterIdxBits | ||
595 | remain-- | ||
596 | } | ||
597 | return prefix + string(b[0:30-len(prefix)]) | ||
598 | } | ||
599 | |||
600 | // IsNetworkOrHostDown - if there was a network error or if the host is down. | ||
601 | // expectTimeouts indicates that *context* timeouts are expected and does not | ||
602 | // indicate a downed host. Other timeouts still returns down. | ||
603 | func IsNetworkOrHostDown(err error, expectTimeouts bool) bool { | ||
604 | if err == nil { | ||
605 | return false | ||
606 | } | ||
607 | |||
608 | if errors.Is(err, context.Canceled) { | ||
609 | return false | ||
610 | } | ||
611 | |||
612 | if expectTimeouts && errors.Is(err, context.DeadlineExceeded) { | ||
613 | return false | ||
614 | } | ||
615 | |||
616 | if errors.Is(err, context.DeadlineExceeded) { | ||
617 | return true | ||
618 | } | ||
619 | |||
620 | // We need to figure if the error either a timeout | ||
621 | // or a non-temporary error. | ||
622 | urlErr := &url.Error{} | ||
623 | if errors.As(err, &urlErr) { | ||
624 | switch urlErr.Err.(type) { | ||
625 | case *net.DNSError, *net.OpError, net.UnknownNetworkError: | ||
626 | return true | ||
627 | } | ||
628 | } | ||
629 | var e net.Error | ||
630 | if errors.As(err, &e) { | ||
631 | if e.Timeout() { | ||
632 | return true | ||
633 | } | ||
634 | } | ||
635 | |||
636 | // Fallback to other mechanisms. | ||
637 | switch { | ||
638 | case strings.Contains(err.Error(), "Connection closed by foreign host"): | ||
639 | return true | ||
640 | case strings.Contains(err.Error(), "TLS handshake timeout"): | ||
641 | // If error is - tlsHandshakeTimeoutError. | ||
642 | return true | ||
643 | case strings.Contains(err.Error(), "i/o timeout"): | ||
644 | // If error is - tcp timeoutError. | ||
645 | return true | ||
646 | case strings.Contains(err.Error(), "connection timed out"): | ||
647 | // If err is a net.Dial timeout. | ||
648 | return true | ||
649 | case strings.Contains(err.Error(), "connection refused"): | ||
650 | // If err is connection refused | ||
651 | return true | ||
652 | |||
653 | case strings.Contains(strings.ToLower(err.Error()), "503 service unavailable"): | ||
654 | // Denial errors | ||
655 | return true | ||
656 | } | ||
657 | return false | ||
658 | } | ||
659 | |||
660 | // newHashReaderWrapper will hash all reads done through r. | ||
661 | // When r returns io.EOF the done function will be called with the sum. | ||
662 | func newHashReaderWrapper(r io.Reader, h hash.Hash, done func(hash []byte)) *hashReaderWrapper { | ||
663 | return &hashReaderWrapper{ | ||
664 | r: r, | ||
665 | h: h, | ||
666 | done: done, | ||
667 | } | ||
668 | } | ||
669 | |||
670 | type hashReaderWrapper struct { | ||
671 | r io.Reader | ||
672 | h hash.Hash | ||
673 | done func(hash []byte) | ||
674 | } | ||
675 | |||
676 | // Read implements the io.Reader interface. | ||
677 | func (h *hashReaderWrapper) Read(p []byte) (n int, err error) { | ||
678 | n, err = h.r.Read(p) | ||
679 | if n > 0 { | ||
680 | n2, err := h.h.Write(p[:n]) | ||
681 | if err != nil { | ||
682 | return 0, err | ||
683 | } | ||
684 | if n2 != n { | ||
685 | return 0, io.ErrShortWrite | ||
686 | } | ||
687 | } | ||
688 | if err == io.EOF { | ||
689 | // Call back | ||
690 | h.done(h.h.Sum(nil)) | ||
691 | } | ||
692 | return n, err | ||
693 | } | ||