aboutsummaryrefslogtreecommitdiffstats
path: root/vendor/github.com/minio/minio-go/v7/api-list.go
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/github.com/minio/minio-go/v7/api-list.go')
-rw-r--r--vendor/github.com/minio/minio-go/v7/api-list.go1057
1 files changed, 1057 insertions, 0 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/api-list.go b/vendor/github.com/minio/minio-go/v7/api-list.go
new file mode 100644
index 0000000..31b6edf
--- /dev/null
+++ b/vendor/github.com/minio/minio-go/v7/api-list.go
@@ -0,0 +1,1057 @@
1/*
2 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
3 * Copyright 2015-2020 MinIO, Inc.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18package minio
19
20import (
21 "context"
22 "fmt"
23 "net/http"
24 "net/url"
25 "time"
26
27 "github.com/minio/minio-go/v7/pkg/s3utils"
28)
29
30// ListBuckets list all buckets owned by this authenticated user.
31//
32// This call requires explicit authentication, no anonymous requests are
33// allowed for listing buckets.
34//
35// api := client.New(....)
36// for message := range api.ListBuckets(context.Background()) {
37// fmt.Println(message)
38// }
39func (c *Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
40 // Execute GET on service.
41 resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{contentSHA256Hex: emptySHA256Hex})
42 defer closeResponse(resp)
43 if err != nil {
44 return nil, err
45 }
46 if resp != nil {
47 if resp.StatusCode != http.StatusOK {
48 return nil, httpRespToErrorResponse(resp, "", "")
49 }
50 }
51 listAllMyBucketsResult := listAllMyBucketsResult{}
52 err = xmlDecoder(resp.Body, &listAllMyBucketsResult)
53 if err != nil {
54 return nil, err
55 }
56 return listAllMyBucketsResult.Buckets.Bucket, nil
57}
58
59// Bucket List Operations.
60func (c *Client) listObjectsV2(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
61 // Allocate new list objects channel.
62 objectStatCh := make(chan ObjectInfo, 1)
63 // Default listing is delimited at "/"
64 delimiter := "/"
65 if opts.Recursive {
66 // If recursive we do not delimit.
67 delimiter = ""
68 }
69
70 // Return object owner information by default
71 fetchOwner := true
72
73 sendObjectInfo := func(info ObjectInfo) {
74 select {
75 case objectStatCh <- info:
76 case <-ctx.Done():
77 }
78 }
79
80 // Validate bucket name.
81 if err := s3utils.CheckValidBucketName(bucketName); err != nil {
82 defer close(objectStatCh)
83 sendObjectInfo(ObjectInfo{
84 Err: err,
85 })
86 return objectStatCh
87 }
88
89 // Validate incoming object prefix.
90 if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
91 defer close(objectStatCh)
92 sendObjectInfo(ObjectInfo{
93 Err: err,
94 })
95 return objectStatCh
96 }
97
98 // Initiate list objects goroutine here.
99 go func(objectStatCh chan<- ObjectInfo) {
100 defer func() {
101 if contextCanceled(ctx) {
102 objectStatCh <- ObjectInfo{
103 Err: ctx.Err(),
104 }
105 }
106 close(objectStatCh)
107 }()
108
109 // Save continuationToken for next request.
110 var continuationToken string
111 for {
112 // Get list of objects a maximum of 1000 per request.
113 result, err := c.listObjectsV2Query(ctx, bucketName, opts.Prefix, continuationToken,
114 fetchOwner, opts.WithMetadata, delimiter, opts.StartAfter, opts.MaxKeys, opts.headers)
115 if err != nil {
116 sendObjectInfo(ObjectInfo{
117 Err: err,
118 })
119 return
120 }
121
122 // If contents are available loop through and send over channel.
123 for _, object := range result.Contents {
124 object.ETag = trimEtag(object.ETag)
125 select {
126 // Send object content.
127 case objectStatCh <- object:
128 // If receives done from the caller, return here.
129 case <-ctx.Done():
130 return
131 }
132 }
133
134 // Send all common prefixes if any.
135 // NOTE: prefixes are only present if the request is delimited.
136 for _, obj := range result.CommonPrefixes {
137 select {
138 // Send object prefixes.
139 case objectStatCh <- ObjectInfo{Key: obj.Prefix}:
140 // If receives done from the caller, return here.
141 case <-ctx.Done():
142 return
143 }
144 }
145
146 // If continuation token present, save it for next request.
147 if result.NextContinuationToken != "" {
148 continuationToken = result.NextContinuationToken
149 }
150
151 // Listing ends result is not truncated, return right here.
152 if !result.IsTruncated {
153 return
154 }
155
156 // Add this to catch broken S3 API implementations.
157 if continuationToken == "" {
158 sendObjectInfo(ObjectInfo{
159 Err: fmt.Errorf("listObjectsV2 is truncated without continuationToken, %s S3 server is incompatible with S3 API", c.endpointURL),
160 })
161 return
162 }
163 }
164 }(objectStatCh)
165 return objectStatCh
166}
167
168// listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket.
169//
170// You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
171// request parameters :-
172// ---------
173// ?prefix - Limits the response to keys that begin with the specified prefix.
174// ?continuation-token - Used to continue iterating over a set of objects
175// ?metadata - Specifies if we want metadata for the objects as part of list operation.
176// ?delimiter - A delimiter is a character you use to group keys.
177// ?start-after - Sets a marker to start listing lexically at this key onwards.
178// ?max-keys - Sets the maximum number of keys returned in the response body.
179func (c *Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter, startAfter string, maxkeys int, headers http.Header) (ListBucketV2Result, error) {
180 // Validate bucket name.
181 if err := s3utils.CheckValidBucketName(bucketName); err != nil {
182 return ListBucketV2Result{}, err
183 }
184 // Validate object prefix.
185 if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil {
186 return ListBucketV2Result{}, err
187 }
188 // Get resources properly escaped and lined up before
189 // using them in http request.
190 urlValues := make(url.Values)
191
192 // Always set list-type in ListObjects V2
193 urlValues.Set("list-type", "2")
194
195 if metadata {
196 urlValues.Set("metadata", "true")
197 }
198
199 // Set this conditionally if asked
200 if startAfter != "" {
201 urlValues.Set("start-after", startAfter)
202 }
203
204 // Always set encoding-type in ListObjects V2
205 urlValues.Set("encoding-type", "url")
206
207 // Set object prefix, prefix value to be set to empty is okay.
208 urlValues.Set("prefix", objectPrefix)
209
210 // Set delimiter, delimiter value to be set to empty is okay.
211 urlValues.Set("delimiter", delimiter)
212
213 // Set continuation token
214 if continuationToken != "" {
215 urlValues.Set("continuation-token", continuationToken)
216 }
217
218 // Fetch owner when listing
219 if fetchOwner {
220 urlValues.Set("fetch-owner", "true")
221 }
222
223 // Set max keys.
224 if maxkeys > 0 {
225 urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys))
226 }
227
228 // Execute GET on bucket to list objects.
229 resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
230 bucketName: bucketName,
231 queryValues: urlValues,
232 contentSHA256Hex: emptySHA256Hex,
233 customHeader: headers,
234 })
235 defer closeResponse(resp)
236 if err != nil {
237 return ListBucketV2Result{}, err
238 }
239 if resp != nil {
240 if resp.StatusCode != http.StatusOK {
241 return ListBucketV2Result{}, httpRespToErrorResponse(resp, bucketName, "")
242 }
243 }
244
245 // Decode listBuckets XML.
246 listBucketResult := ListBucketV2Result{}
247 if err = xmlDecoder(resp.Body, &listBucketResult); err != nil {
248 return listBucketResult, err
249 }
250
251 // This is an additional verification check to make
252 // sure proper responses are received.
253 if listBucketResult.IsTruncated && listBucketResult.NextContinuationToken == "" {
254 return listBucketResult, ErrorResponse{
255 Code: "NotImplemented",
256 Message: "Truncated response should have continuation token set",
257 }
258 }
259
260 for i, obj := range listBucketResult.Contents {
261 listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType)
262 if err != nil {
263 return listBucketResult, err
264 }
265 listBucketResult.Contents[i].LastModified = listBucketResult.Contents[i].LastModified.Truncate(time.Millisecond)
266 }
267
268 for i, obj := range listBucketResult.CommonPrefixes {
269 listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType)
270 if err != nil {
271 return listBucketResult, err
272 }
273 }
274
275 // Success.
276 return listBucketResult, nil
277}
278
279func (c *Client) listObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
280 // Allocate new list objects channel.
281 objectStatCh := make(chan ObjectInfo, 1)
282 // Default listing is delimited at "/"
283 delimiter := "/"
284 if opts.Recursive {
285 // If recursive we do not delimit.
286 delimiter = ""
287 }
288
289 sendObjectInfo := func(info ObjectInfo) {
290 select {
291 case objectStatCh <- info:
292 case <-ctx.Done():
293 }
294 }
295
296 // Validate bucket name.
297 if err := s3utils.CheckValidBucketName(bucketName); err != nil {
298 defer close(objectStatCh)
299 sendObjectInfo(ObjectInfo{
300 Err: err,
301 })
302 return objectStatCh
303 }
304 // Validate incoming object prefix.
305 if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
306 defer close(objectStatCh)
307 sendObjectInfo(ObjectInfo{
308 Err: err,
309 })
310 return objectStatCh
311 }
312
313 // Initiate list objects goroutine here.
314 go func(objectStatCh chan<- ObjectInfo) {
315 defer func() {
316 if contextCanceled(ctx) {
317 objectStatCh <- ObjectInfo{
318 Err: ctx.Err(),
319 }
320 }
321 close(objectStatCh)
322 }()
323
324 marker := opts.StartAfter
325 for {
326 // Get list of objects a maximum of 1000 per request.
327 result, err := c.listObjectsQuery(ctx, bucketName, opts.Prefix, marker, delimiter, opts.MaxKeys, opts.headers)
328 if err != nil {
329 sendObjectInfo(ObjectInfo{
330 Err: err,
331 })
332 return
333 }
334
335 // If contents are available loop through and send over channel.
336 for _, object := range result.Contents {
337 // Save the marker.
338 marker = object.Key
339 object.ETag = trimEtag(object.ETag)
340 select {
341 // Send object content.
342 case objectStatCh <- object:
343 // If receives done from the caller, return here.
344 case <-ctx.Done():
345 return
346 }
347 }
348
349 // Send all common prefixes if any.
350 // NOTE: prefixes are only present if the request is delimited.
351 for _, obj := range result.CommonPrefixes {
352 select {
353 // Send object prefixes.
354 case objectStatCh <- ObjectInfo{Key: obj.Prefix}:
355 // If receives done from the caller, return here.
356 case <-ctx.Done():
357 return
358 }
359 }
360
361 // If next marker present, save it for next request.
362 if result.NextMarker != "" {
363 marker = result.NextMarker
364 }
365
366 // Listing ends result is not truncated, return right here.
367 if !result.IsTruncated {
368 return
369 }
370 }
371 }(objectStatCh)
372 return objectStatCh
373}
374
375func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
376 // Allocate new list objects channel.
377 resultCh := make(chan ObjectInfo, 1)
378 // Default listing is delimited at "/"
379 delimiter := "/"
380 if opts.Recursive {
381 // If recursive we do not delimit.
382 delimiter = ""
383 }
384
385 sendObjectInfo := func(info ObjectInfo) {
386 select {
387 case resultCh <- info:
388 case <-ctx.Done():
389 }
390 }
391
392 // Validate bucket name.
393 if err := s3utils.CheckValidBucketName(bucketName); err != nil {
394 defer close(resultCh)
395 sendObjectInfo(ObjectInfo{
396 Err: err,
397 })
398 return resultCh
399 }
400
401 // Validate incoming object prefix.
402 if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
403 defer close(resultCh)
404 sendObjectInfo(ObjectInfo{
405 Err: err,
406 })
407 return resultCh
408 }
409
410 // Initiate list objects goroutine here.
411 go func(resultCh chan<- ObjectInfo) {
412 defer func() {
413 if contextCanceled(ctx) {
414 resultCh <- ObjectInfo{
415 Err: ctx.Err(),
416 }
417 }
418 close(resultCh)
419 }()
420
421 var (
422 keyMarker = ""
423 versionIDMarker = ""
424 )
425
426 for {
427 // Get list of objects a maximum of 1000 per request.
428 result, err := c.listObjectVersionsQuery(ctx, bucketName, opts, keyMarker, versionIDMarker, delimiter)
429 if err != nil {
430 sendObjectInfo(ObjectInfo{
431 Err: err,
432 })
433 return
434 }
435
436 // If contents are available loop through and send over channel.
437 for _, version := range result.Versions {
438 info := ObjectInfo{
439 ETag: trimEtag(version.ETag),
440 Key: version.Key,
441 LastModified: version.LastModified.Truncate(time.Millisecond),
442 Size: version.Size,
443 Owner: version.Owner,
444 StorageClass: version.StorageClass,
445 IsLatest: version.IsLatest,
446 VersionID: version.VersionID,
447 IsDeleteMarker: version.isDeleteMarker,
448 UserTags: version.UserTags,
449 UserMetadata: version.UserMetadata,
450 Internal: version.Internal,
451 }
452 select {
453 // Send object version info.
454 case resultCh <- info:
455 // If receives done from the caller, return here.
456 case <-ctx.Done():
457 return
458 }
459 }
460
461 // Send all common prefixes if any.
462 // NOTE: prefixes are only present if the request is delimited.
463 for _, obj := range result.CommonPrefixes {
464 select {
465 // Send object prefixes.
466 case resultCh <- ObjectInfo{Key: obj.Prefix}:
467 // If receives done from the caller, return here.
468 case <-ctx.Done():
469 return
470 }
471 }
472
473 // If next key marker is present, save it for next request.
474 if result.NextKeyMarker != "" {
475 keyMarker = result.NextKeyMarker
476 }
477
478 // If next version id marker is present, save it for next request.
479 if result.NextVersionIDMarker != "" {
480 versionIDMarker = result.NextVersionIDMarker
481 }
482
483 // Listing ends result is not truncated, return right here.
484 if !result.IsTruncated {
485 return
486 }
487 }
488 }(resultCh)
489 return resultCh
490}
491
492// listObjectVersions - (List Object Versions) - List some or all (up to 1000) of the existing objects
493// and their versions in a bucket.
494//
495// You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
496// request parameters :-
497// ---------
498// ?key-marker - Specifies the key to start with when listing objects in a bucket.
499// ?version-id-marker - Specifies the version id marker to start with when listing objects with versions in a bucket.
500// ?delimiter - A delimiter is a character you use to group keys.
501// ?prefix - Limits the response to keys that begin with the specified prefix.
502// ?max-keys - Sets the maximum number of keys returned in the response body.
503func (c *Client) listObjectVersionsQuery(ctx context.Context, bucketName string, opts ListObjectsOptions, keyMarker, versionIDMarker, delimiter string) (ListVersionsResult, error) {
504 // Validate bucket name.
505 if err := s3utils.CheckValidBucketName(bucketName); err != nil {
506 return ListVersionsResult{}, err
507 }
508 // Validate object prefix.
509 if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
510 return ListVersionsResult{}, err
511 }
512 // Get resources properly escaped and lined up before
513 // using them in http request.
514 urlValues := make(url.Values)
515
516 // Set versions to trigger versioning API
517 urlValues.Set("versions", "")
518
519 // Set object prefix, prefix value to be set to empty is okay.
520 urlValues.Set("prefix", opts.Prefix)
521
522 // Set delimiter, delimiter value to be set to empty is okay.
523 urlValues.Set("delimiter", delimiter)
524
525 // Set object marker.
526 if keyMarker != "" {
527 urlValues.Set("key-marker", keyMarker)
528 }
529
530 // Set max keys.
531 if opts.MaxKeys > 0 {
532 urlValues.Set("max-keys", fmt.Sprintf("%d", opts.MaxKeys))
533 }
534
535 // Set version ID marker
536 if versionIDMarker != "" {
537 urlValues.Set("version-id-marker", versionIDMarker)
538 }
539
540 if opts.WithMetadata {
541 urlValues.Set("metadata", "true")
542 }
543
544 // Always set encoding-type
545 urlValues.Set("encoding-type", "url")
546
547 // Execute GET on bucket to list objects.
548 resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
549 bucketName: bucketName,
550 queryValues: urlValues,
551 contentSHA256Hex: emptySHA256Hex,
552 customHeader: opts.headers,
553 })
554 defer closeResponse(resp)
555 if err != nil {
556 return ListVersionsResult{}, err
557 }
558 if resp != nil {
559 if resp.StatusCode != http.StatusOK {
560 return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "")
561 }
562 }
563
564 // Decode ListVersionsResult XML.
565 listObjectVersionsOutput := ListVersionsResult{}
566 err = xmlDecoder(resp.Body, &listObjectVersionsOutput)
567 if err != nil {
568 return ListVersionsResult{}, err
569 }
570
571 for i, obj := range listObjectVersionsOutput.Versions {
572 listObjectVersionsOutput.Versions[i].Key, err = decodeS3Name(obj.Key, listObjectVersionsOutput.EncodingType)
573 if err != nil {
574 return listObjectVersionsOutput, err
575 }
576 }
577
578 for i, obj := range listObjectVersionsOutput.CommonPrefixes {
579 listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listObjectVersionsOutput.EncodingType)
580 if err != nil {
581 return listObjectVersionsOutput, err
582 }
583 }
584
585 if listObjectVersionsOutput.NextKeyMarker != "" {
586 listObjectVersionsOutput.NextKeyMarker, err = decodeS3Name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType)
587 if err != nil {
588 return listObjectVersionsOutput, err
589 }
590 }
591
592 return listObjectVersionsOutput, nil
593}
594
595// listObjects - (List Objects) - List some or all (up to 1000) of the objects in a bucket.
596//
597// You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
598// request parameters :-
599// ---------
600// ?marker - Specifies the key to start with when listing objects in a bucket.
601// ?delimiter - A delimiter is a character you use to group keys.
602// ?prefix - Limits the response to keys that begin with the specified prefix.
603// ?max-keys - Sets the maximum number of keys returned in the response body.
604func (c *Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int, headers http.Header) (ListBucketResult, error) {
605 // Validate bucket name.
606 if err := s3utils.CheckValidBucketName(bucketName); err != nil {
607 return ListBucketResult{}, err
608 }
609 // Validate object prefix.
610 if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil {
611 return ListBucketResult{}, err
612 }
613 // Get resources properly escaped and lined up before
614 // using them in http request.
615 urlValues := make(url.Values)
616
617 // Set object prefix, prefix value to be set to empty is okay.
618 urlValues.Set("prefix", objectPrefix)
619
620 // Set delimiter, delimiter value to be set to empty is okay.
621 urlValues.Set("delimiter", delimiter)
622
623 // Set object marker.
624 if objectMarker != "" {
625 urlValues.Set("marker", objectMarker)
626 }
627
628 // Set max keys.
629 if maxkeys > 0 {
630 urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys))
631 }
632
633 // Always set encoding-type
634 urlValues.Set("encoding-type", "url")
635
636 // Execute GET on bucket to list objects.
637 resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
638 bucketName: bucketName,
639 queryValues: urlValues,
640 contentSHA256Hex: emptySHA256Hex,
641 customHeader: headers,
642 })
643 defer closeResponse(resp)
644 if err != nil {
645 return ListBucketResult{}, err
646 }
647 if resp != nil {
648 if resp.StatusCode != http.StatusOK {
649 return ListBucketResult{}, httpRespToErrorResponse(resp, bucketName, "")
650 }
651 }
652 // Decode listBuckets XML.
653 listBucketResult := ListBucketResult{}
654 err = xmlDecoder(resp.Body, &listBucketResult)
655 if err != nil {
656 return listBucketResult, err
657 }
658
659 for i, obj := range listBucketResult.Contents {
660 listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType)
661 if err != nil {
662 return listBucketResult, err
663 }
664 listBucketResult.Contents[i].LastModified = listBucketResult.Contents[i].LastModified.Truncate(time.Millisecond)
665 }
666
667 for i, obj := range listBucketResult.CommonPrefixes {
668 listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType)
669 if err != nil {
670 return listBucketResult, err
671 }
672 }
673
674 if listBucketResult.NextMarker != "" {
675 listBucketResult.NextMarker, err = decodeS3Name(listBucketResult.NextMarker, listBucketResult.EncodingType)
676 if err != nil {
677 return listBucketResult, err
678 }
679 }
680
681 return listBucketResult, nil
682}
683
684// ListObjectsOptions holds all options of a list object request
685type ListObjectsOptions struct {
686 // Include objects versions in the listing
687 WithVersions bool
688 // Include objects metadata in the listing
689 WithMetadata bool
690 // Only list objects with the prefix
691 Prefix string
692 // Ignore '/' delimiter
693 Recursive bool
694 // The maximum number of objects requested per
695 // batch, advanced use-case not useful for most
696 // applications
697 MaxKeys int
698 // StartAfter start listing lexically at this
699 // object onwards, this value can also be set
700 // for Marker when `UseV1` is set to true.
701 StartAfter string
702
703 // Use the deprecated list objects V1 API
704 UseV1 bool
705
706 headers http.Header
707}
708
709// Set adds a key value pair to the options. The
710// key-value pair will be part of the HTTP GET request
711// headers.
712func (o *ListObjectsOptions) Set(key, value string) {
713 if o.headers == nil {
714 o.headers = make(http.Header)
715 }
716 o.headers.Set(key, value)
717}
718
719// ListObjects returns objects list after evaluating the passed options.
720//
721// api := client.New(....)
722// for object := range api.ListObjects(ctx, "mytestbucket", minio.ListObjectsOptions{Prefix: "starthere", Recursive:true}) {
723// fmt.Println(object)
724// }
725//
726// If caller cancels the context, then the last entry on the 'chan ObjectInfo' will be the context.Error()
727// caller must drain the channel entirely and wait until channel is closed before proceeding, without
728// waiting on the channel to be closed completely you might leak goroutines.
729func (c *Client) ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
730 if opts.WithVersions {
731 return c.listObjectVersions(ctx, bucketName, opts)
732 }
733
734 // Use legacy list objects v1 API
735 if opts.UseV1 {
736 return c.listObjects(ctx, bucketName, opts)
737 }
738
739 // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1.
740 if location, ok := c.bucketLocCache.Get(bucketName); ok {
741 if location == "snowball" {
742 return c.listObjects(ctx, bucketName, opts)
743 }
744 }
745
746 return c.listObjectsV2(ctx, bucketName, opts)
747}
748
749// ListIncompleteUploads - List incompletely uploaded multipart objects.
750//
751// ListIncompleteUploads lists all incompleted objects matching the
752// objectPrefix from the specified bucket. If recursion is enabled
753// it would list all subdirectories and all its contents.
754//
755// Your input parameters are just bucketName, objectPrefix, recursive.
756// If you enable recursive as 'true' this function will return back all
757// the multipart objects in a given bucket name.
758//
759// api := client.New(....)
760// // Recurively list all objects in 'mytestbucket'
761// recursive := true
762// for message := range api.ListIncompleteUploads(context.Background(), "mytestbucket", "starthere", recursive) {
763// fmt.Println(message)
764// }
765func (c *Client) ListIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo {
766 return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive)
767}
768
769// contextCanceled returns whether a context is canceled.
770func contextCanceled(ctx context.Context) bool {
771 select {
772 case <-ctx.Done():
773 return true
774 default:
775 return false
776 }
777}
778
779// listIncompleteUploads lists all incomplete uploads.
780func (c *Client) listIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo {
781 // Allocate channel for multipart uploads.
782 objectMultipartStatCh := make(chan ObjectMultipartInfo, 1)
783 // Delimiter is set to "/" by default.
784 delimiter := "/"
785 if recursive {
786 // If recursive do not delimit.
787 delimiter = ""
788 }
789 // Validate bucket name.
790 if err := s3utils.CheckValidBucketName(bucketName); err != nil {
791 defer close(objectMultipartStatCh)
792 objectMultipartStatCh <- ObjectMultipartInfo{
793 Err: err,
794 }
795 return objectMultipartStatCh
796 }
797 // Validate incoming object prefix.
798 if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil {
799 defer close(objectMultipartStatCh)
800 objectMultipartStatCh <- ObjectMultipartInfo{
801 Err: err,
802 }
803 return objectMultipartStatCh
804 }
805 go func(objectMultipartStatCh chan<- ObjectMultipartInfo) {
806 defer func() {
807 if contextCanceled(ctx) {
808 objectMultipartStatCh <- ObjectMultipartInfo{
809 Err: ctx.Err(),
810 }
811 }
812 close(objectMultipartStatCh)
813 }()
814
815 // object and upload ID marker for future requests.
816 var objectMarker string
817 var uploadIDMarker string
818 for {
819 // list all multipart uploads.
820 result, err := c.listMultipartUploadsQuery(ctx, bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0)
821 if err != nil {
822 objectMultipartStatCh <- ObjectMultipartInfo{
823 Err: err,
824 }
825 return
826 }
827 objectMarker = result.NextKeyMarker
828 uploadIDMarker = result.NextUploadIDMarker
829
830 // Send all multipart uploads.
831 for _, obj := range result.Uploads {
832 // Calculate total size of the uploaded parts if 'aggregateSize' is enabled.
833 select {
834 // Send individual uploads here.
835 case objectMultipartStatCh <- obj:
836 // If the context is canceled
837 case <-ctx.Done():
838 return
839 }
840 }
841 // Send all common prefixes if any.
842 // NOTE: prefixes are only present if the request is delimited.
843 for _, obj := range result.CommonPrefixes {
844 select {
845 // Send delimited prefixes here.
846 case objectMultipartStatCh <- ObjectMultipartInfo{Key: obj.Prefix, Size: 0}:
847 // If context is canceled.
848 case <-ctx.Done():
849 return
850 }
851 }
852 // Listing ends if result not truncated, return right here.
853 if !result.IsTruncated {
854 return
855 }
856 }
857 }(objectMultipartStatCh)
858 // return.
859 return objectMultipartStatCh
860}
861
862// listMultipartUploadsQuery - (List Multipart Uploads).
863// - Lists some or all (up to 1000) in-progress multipart uploads in a bucket.
864//
865// You can use the request parameters as selection criteria to return a subset of the uploads in a bucket.
866// request parameters. :-
867// ---------
868// ?key-marker - Specifies the multipart upload after which listing should begin.
869// ?upload-id-marker - Together with key-marker specifies the multipart upload after which listing should begin.
870// ?delimiter - A delimiter is a character you use to group keys.
871// ?prefix - Limits the response to keys that begin with the specified prefix.
872// ?max-uploads - Sets the maximum number of multipart uploads returned in the response body.
873func (c *Client) listMultipartUploadsQuery(ctx context.Context, bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (ListMultipartUploadsResult, error) {
874 // Get resources properly escaped and lined up before using them in http request.
875 urlValues := make(url.Values)
876 // Set uploads.
877 urlValues.Set("uploads", "")
878 // Set object key marker.
879 if keyMarker != "" {
880 urlValues.Set("key-marker", keyMarker)
881 }
882 // Set upload id marker.
883 if uploadIDMarker != "" {
884 urlValues.Set("upload-id-marker", uploadIDMarker)
885 }
886
887 // Set object prefix, prefix value to be set to empty is okay.
888 urlValues.Set("prefix", prefix)
889
890 // Set delimiter, delimiter value to be set to empty is okay.
891 urlValues.Set("delimiter", delimiter)
892
893 // Always set encoding-type
894 urlValues.Set("encoding-type", "url")
895
896 // maxUploads should be 1000 or less.
897 if maxUploads > 0 {
898 // Set max-uploads.
899 urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads))
900 }
901
902 // Execute GET on bucketName to list multipart uploads.
903 resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
904 bucketName: bucketName,
905 queryValues: urlValues,
906 contentSHA256Hex: emptySHA256Hex,
907 })
908 defer closeResponse(resp)
909 if err != nil {
910 return ListMultipartUploadsResult{}, err
911 }
912 if resp != nil {
913 if resp.StatusCode != http.StatusOK {
914 return ListMultipartUploadsResult{}, httpRespToErrorResponse(resp, bucketName, "")
915 }
916 }
917 // Decode response body.
918 listMultipartUploadsResult := ListMultipartUploadsResult{}
919 err = xmlDecoder(resp.Body, &listMultipartUploadsResult)
920 if err != nil {
921 return listMultipartUploadsResult, err
922 }
923
924 listMultipartUploadsResult.NextKeyMarker, err = decodeS3Name(listMultipartUploadsResult.NextKeyMarker, listMultipartUploadsResult.EncodingType)
925 if err != nil {
926 return listMultipartUploadsResult, err
927 }
928
929 listMultipartUploadsResult.NextUploadIDMarker, err = decodeS3Name(listMultipartUploadsResult.NextUploadIDMarker, listMultipartUploadsResult.EncodingType)
930 if err != nil {
931 return listMultipartUploadsResult, err
932 }
933
934 for i, obj := range listMultipartUploadsResult.Uploads {
935 listMultipartUploadsResult.Uploads[i].Key, err = decodeS3Name(obj.Key, listMultipartUploadsResult.EncodingType)
936 if err != nil {
937 return listMultipartUploadsResult, err
938 }
939 }
940
941 for i, obj := range listMultipartUploadsResult.CommonPrefixes {
942 listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listMultipartUploadsResult.EncodingType)
943 if err != nil {
944 return listMultipartUploadsResult, err
945 }
946 }
947
948 return listMultipartUploadsResult, nil
949}
950
951// listObjectParts list all object parts recursively.
952//
953//lint:ignore U1000 Keep this around
954func (c *Client) listObjectParts(ctx context.Context, bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) {
955 // Part number marker for the next batch of request.
956 var nextPartNumberMarker int
957 partsInfo = make(map[int]ObjectPart)
958 for {
959 // Get list of uploaded parts a maximum of 1000 per request.
960 listObjPartsResult, err := c.listObjectPartsQuery(ctx, bucketName, objectName, uploadID, nextPartNumberMarker, 1000)
961 if err != nil {
962 return nil, err
963 }
964 // Append to parts info.
965 for _, part := range listObjPartsResult.ObjectParts {
966 // Trim off the odd double quotes from ETag in the beginning and end.
967 part.ETag = trimEtag(part.ETag)
968 partsInfo[part.PartNumber] = part
969 }
970 // Keep part number marker, for the next iteration.
971 nextPartNumberMarker = listObjPartsResult.NextPartNumberMarker
972 // Listing ends result is not truncated, return right here.
973 if !listObjPartsResult.IsTruncated {
974 break
975 }
976 }
977
978 // Return all the parts.
979 return partsInfo, nil
980}
981
982// findUploadIDs lists all incomplete uploads and find the uploadIDs of the matching object name.
983func (c *Client) findUploadIDs(ctx context.Context, bucketName, objectName string) ([]string, error) {
984 var uploadIDs []string
985 // Make list incomplete uploads recursive.
986 isRecursive := true
987 // List all incomplete uploads.
988 for mpUpload := range c.listIncompleteUploads(ctx, bucketName, objectName, isRecursive) {
989 if mpUpload.Err != nil {
990 return nil, mpUpload.Err
991 }
992 if objectName == mpUpload.Key {
993 uploadIDs = append(uploadIDs, mpUpload.UploadID)
994 }
995 }
996 // Return the latest upload id.
997 return uploadIDs, nil
998}
999
1000// listObjectPartsQuery (List Parts query)
1001// - lists some or all (up to 1000) parts that have been uploaded
1002// for a specific multipart upload
1003//
1004// You can use the request parameters as selection criteria to return
1005// a subset of the uploads in a bucket, request parameters :-
1006// ---------
1007// ?part-number-marker - Specifies the part after which listing should
1008// begin.
1009// ?max-parts - Maximum parts to be listed per request.
1010func (c *Client) listObjectPartsQuery(ctx context.Context, bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (ListObjectPartsResult, error) {
1011 // Get resources properly escaped and lined up before using them in http request.
1012 urlValues := make(url.Values)
1013 // Set part number marker.
1014 urlValues.Set("part-number-marker", fmt.Sprintf("%d", partNumberMarker))
1015 // Set upload id.
1016 urlValues.Set("uploadId", uploadID)
1017
1018 // maxParts should be 1000 or less.
1019 if maxParts > 0 {
1020 // Set max parts.
1021 urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts))
1022 }
1023
1024 // Execute GET on objectName to get list of parts.
1025 resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
1026 bucketName: bucketName,
1027 objectName: objectName,
1028 queryValues: urlValues,
1029 contentSHA256Hex: emptySHA256Hex,
1030 })
1031 defer closeResponse(resp)
1032 if err != nil {
1033 return ListObjectPartsResult{}, err
1034 }
1035 if resp != nil {
1036 if resp.StatusCode != http.StatusOK {
1037 return ListObjectPartsResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
1038 }
1039 }
1040 // Decode list object parts XML.
1041 listObjectPartsResult := ListObjectPartsResult{}
1042 err = xmlDecoder(resp.Body, &listObjectPartsResult)
1043 if err != nil {
1044 return listObjectPartsResult, err
1045 }
1046 return listObjectPartsResult, nil
1047}
1048
1049// Decode an S3 object name according to the encoding type
1050func decodeS3Name(name, encodingType string) (string, error) {
1051 switch encodingType {
1052 case "url":
1053 return url.QueryUnescape(name)
1054 default:
1055 return name, nil
1056 }
1057}
generated by cgit v1.2.3 (git 2.45.2) at 2024-09-30 23:36:17 +0000