aboutsummaryrefslogtreecommitdiffstats
path: root/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go
diff options
context:
space:
mode:
authorLibravatar Rutger Broekhoff2023-12-29 21:31:53 +0100
committerLibravatar Rutger Broekhoff2023-12-29 21:31:53 +0100
commit404aeae4545d2426c089a5f8d5e82dae56f5212b (patch)
tree2d84e00af272b39fc04f3795ae06bc48970e57b5 /vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go
parent209d8b0187ed025dec9ac149ebcced3462877bff (diff)
downloadgitolfs3-404aeae4545d2426c089a5f8d5e82dae56f5212b.tar.gz
gitolfs3-404aeae4545d2426c089a5f8d5e82dae56f5212b.zip
Make Nix builds work
Diffstat (limited to 'vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go')
-rw-r--r--vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go246
1 files changed, 246 insertions, 0 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go b/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go
new file mode 100644
index 0000000..eb4da41
--- /dev/null
+++ b/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go
@@ -0,0 +1,246 @@
1/*
2 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
3 * Copyright 2021 MinIO, Inc.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18package minio
19
20import (
21 "archive/tar"
22 "bufio"
23 "bytes"
24 "context"
25 "fmt"
26 "io"
27 "net/http"
28 "os"
29 "strings"
30 "sync"
31 "time"
32
33 "github.com/klauspost/compress/s2"
34)
35
36// SnowballOptions contains options for PutObjectsSnowball calls.
37type SnowballOptions struct {
38 // Opts is options applied to all objects.
39 Opts PutObjectOptions
40
41 // Processing options:
42
43 // InMemory specifies that all objects should be collected in memory
44 // before they are uploaded.
45 // If false a temporary file will be created.
46 InMemory bool
47
48 // Compress enabled content compression before upload.
49 // Compression will typically reduce memory and network usage,
50 // Compression can safely be enabled with MinIO hosts.
51 Compress bool
52
53 // SkipErrs if enabled will skip any errors while reading the
54 // object content while creating the snowball archive
55 SkipErrs bool
56}
57
58// SnowballObject contains information about a single object to be added to the snowball.
59type SnowballObject struct {
60 // Key is the destination key, including prefix.
61 Key string
62
63 // Size is the content size of this object.
64 Size int64
65
66 // Modtime to apply to the object.
67 // If Modtime is the zero value current time will be used.
68 ModTime time.Time
69
70 // Content of the object.
71 // Exactly 'Size' number of bytes must be provided.
72 Content io.Reader
73
74 // VersionID of the object; if empty, a new versionID will be generated
75 VersionID string
76
77 // Headers contains more options for this object upload, the same as you
78 // would include in a regular PutObject operation, such as user metadata
79 // and content-disposition, expires, ..
80 Headers http.Header
81
82 // Close will be called when an object has finished processing.
83 // Note that if PutObjectsSnowball returns because of an error,
84 // objects not consumed from the input will NOT have been closed.
85 // Leave as nil for no callback.
86 Close func()
87}
88
89type nopReadSeekCloser struct {
90 io.ReadSeeker
91}
92
93func (n nopReadSeekCloser) Close() error {
94 return nil
95}
96
97// This is available as io.ReadSeekCloser from go1.16
98type readSeekCloser interface {
99 io.Reader
100 io.Closer
101 io.Seeker
102}
103
104// PutObjectsSnowball will put multiple objects with a single put call.
105// A (compressed) TAR file will be created which will contain multiple objects.
106// The key for each object will be used for the destination in the specified bucket.
107// Total size should be < 5TB.
108// This function blocks until 'objs' is closed and the content has been uploaded.
109func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts SnowballOptions, objs <-chan SnowballObject) (err error) {
110 err = opts.Opts.validate()
111 if err != nil {
112 return err
113 }
114 var tmpWriter io.Writer
115 var getTmpReader func() (rc readSeekCloser, sz int64, err error)
116 if opts.InMemory {
117 b := bytes.NewBuffer(nil)
118 tmpWriter = b
119 getTmpReader = func() (readSeekCloser, int64, error) {
120 return nopReadSeekCloser{bytes.NewReader(b.Bytes())}, int64(b.Len()), nil
121 }
122 } else {
123 f, err := os.CreateTemp("", "s3-putsnowballobjects-*")
124 if err != nil {
125 return err
126 }
127 name := f.Name()
128 tmpWriter = f
129 var once sync.Once
130 defer once.Do(func() {
131 f.Close()
132 })
133 defer os.Remove(name)
134 getTmpReader = func() (readSeekCloser, int64, error) {
135 once.Do(func() {
136 f.Close()
137 })
138 f, err := os.Open(name)
139 if err != nil {
140 return nil, 0, err
141 }
142 st, err := f.Stat()
143 if err != nil {
144 return nil, 0, err
145 }
146 return f, st.Size(), nil
147 }
148 }
149 flush := func() error { return nil }
150 if !opts.Compress {
151 if !opts.InMemory {
152 // Insert buffer for writes.
153 buf := bufio.NewWriterSize(tmpWriter, 1<<20)
154 flush = buf.Flush
155 tmpWriter = buf
156 }
157 } else {
158 s2c := s2.NewWriter(tmpWriter, s2.WriterBetterCompression())
159 flush = s2c.Close
160 defer s2c.Close()
161 tmpWriter = s2c
162 }
163 t := tar.NewWriter(tmpWriter)
164
165objectLoop:
166 for {
167 select {
168 case <-ctx.Done():
169 return ctx.Err()
170 case obj, ok := <-objs:
171 if !ok {
172 break objectLoop
173 }
174
175 closeObj := func() {}
176 if obj.Close != nil {
177 closeObj = obj.Close
178 }
179
180 // Trim accidental slash prefix.
181 obj.Key = strings.TrimPrefix(obj.Key, "/")
182 header := tar.Header{
183 Typeflag: tar.TypeReg,
184 Name: obj.Key,
185 Size: obj.Size,
186 ModTime: obj.ModTime,
187 Format: tar.FormatPAX,
188 }
189 if header.ModTime.IsZero() {
190 header.ModTime = time.Now().UTC()
191 }
192
193 header.PAXRecords = make(map[string]string)
194 if obj.VersionID != "" {
195 header.PAXRecords["minio.versionId"] = obj.VersionID
196 }
197 for k, vals := range obj.Headers {
198 header.PAXRecords["minio.metadata."+k] = strings.Join(vals, ",")
199 }
200
201 if err := t.WriteHeader(&header); err != nil {
202 closeObj()
203 return err
204 }
205 n, err := io.Copy(t, obj.Content)
206 if err != nil {
207 closeObj()
208 if opts.SkipErrs {
209 continue
210 }
211 return err
212 }
213 if n != obj.Size {
214 closeObj()
215 if opts.SkipErrs {
216 continue
217 }
218 return io.ErrUnexpectedEOF
219 }
220 closeObj()
221 }
222 }
223 // Flush tar
224 err = t.Flush()
225 if err != nil {
226 return err
227 }
228 // Flush compression
229 err = flush()
230 if err != nil {
231 return err
232 }
233 if opts.Opts.UserMetadata == nil {
234 opts.Opts.UserMetadata = map[string]string{}
235 }
236 opts.Opts.UserMetadata["X-Amz-Meta-Snowball-Auto-Extract"] = "true"
237 opts.Opts.DisableMultipart = true
238 rc, sz, err := getTmpReader()
239 if err != nil {
240 return err
241 }
242 defer rc.Close()
243 rand := c.random.Uint64()
244 _, err = c.PutObject(ctx, bucketName, fmt.Sprintf("snowball-upload-%x.tar", rand), rc, sz, opts.Opts)
245 return err
246}