diff options
author | Wim <wim@42.be> | 2022-04-01 00:23:19 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-04-01 00:23:19 +0200 |
commit | c6716e030c02f316b887c1d3ee4b443aa3ab6afd (patch) | |
tree | 470461fe2d29662e7a69834ed21fce30beed65ab /vendor/github.com/minio/minio-go | |
parent | 4ab72acec656dafd304f88359b509b1f27c06604 (diff) | |
download | matterbridge-msglm-c6716e030c02f316b887c1d3ee4b443aa3ab6afd.tar.gz matterbridge-msglm-c6716e030c02f316b887c1d3ee4b443aa3ab6afd.tar.bz2 matterbridge-msglm-c6716e030c02f316b887c1d3ee4b443aa3ab6afd.zip |
Update dependencies (#1784)
Diffstat (limited to 'vendor/github.com/minio/minio-go')
22 files changed, 500 insertions, 128 deletions
diff --git a/vendor/github.com/minio/minio-go/v7/CONTRIBUTING.md b/vendor/github.com/minio/minio-go/v7/CONTRIBUTING.md index 8b1ee86c..24522ef7 100644 --- a/vendor/github.com/minio/minio-go/v7/CONTRIBUTING.md +++ b/vendor/github.com/minio/minio-go/v7/CONTRIBUTING.md @@ -1,4 +1,3 @@ - ### Developer Guidelines ``minio-go`` welcomes your contribution. To make the process as seamless as possible, we ask for the following: diff --git a/vendor/github.com/minio/minio-go/v7/README.md b/vendor/github.com/minio/minio-go/v7/README.md index 3ba174f4..211dd5e8 100644 --- a/vendor/github.com/minio/minio-go/v7/README.md +++ b/vendor/github.com/minio/minio-go/v7/README.md @@ -8,7 +8,7 @@ This document assumes that you have a working [Go development environment](https ## Download from Github ```sh -GO111MODULE=on go get github.com/minio/minio-go/v7 +go get github.com/minio/minio-go/v7 ``` ## Initialize MinIO Client @@ -115,7 +115,6 @@ func main() { ### Run FileUploader ```sh -export GO111MODULE=on go run file-uploader.go 2016/08/13 17:03:28 Successfully created mymusic 2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 diff --git a/vendor/github.com/minio/minio-go/v7/api-error-response.go b/vendor/github.com/minio/minio-go/v7/api-error-response.go index 39df7eec..dd781cae 100644 --- a/vendor/github.com/minio/minio-go/v7/api-error-response.go +++ b/vendor/github.com/minio/minio-go/v7/api-error-response.go @@ -47,6 +47,7 @@ type ErrorResponse struct { Message string BucketName string Key string + Resource string RequestID string `xml:"RequestId"` HostID string `xml:"HostId"` diff --git a/vendor/github.com/minio/minio-go/v7/api-get-object-acl.go b/vendor/github.com/minio/minio-go/v7/api-get-object-acl.go index b1291b6b..9041d99e 100644 --- a/vendor/github.com/minio/minio-go/v7/api-get-object-acl.go +++ b/vendor/github.com/minio/minio-go/v7/api-get-object-acl.go @@ -47,8 +47,9 @@ type AccessControlList struct { } type accessControlPolicy struct { - Owner - AccessControlList + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner + AccessControlList AccessControlList } // GetObjectACL get object ACLs diff --git a/vendor/github.com/minio/minio-go/v7/api-get-object.go b/vendor/github.com/minio/minio-go/v7/api-get-object.go index b9b96025..2ce4b260 100644 --- a/vendor/github.com/minio/minio-go/v7/api-get-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-get-object.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/url" + "strconv" "sync" "github.com/minio/minio-go/v7/pkg/s3utils" @@ -652,6 +653,9 @@ func (c *Client) getObject(ctx context.Context, bucketName, objectName string, o if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } + if opts.PartNumber > 0 { + urlValues.Set("partNumber", strconv.Itoa(opts.PartNumber)) + } // Execute GET on objectName. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ diff --git a/vendor/github.com/minio/minio-go/v7/api-get-options.go b/vendor/github.com/minio/minio-go/v7/api-get-options.go index 0be858d1..184ef9f8 100644 --- a/vendor/github.com/minio/minio-go/v7/api-get-options.go +++ b/vendor/github.com/minio/minio-go/v7/api-get-options.go @@ -37,6 +37,7 @@ type GetObjectOptions struct { headers map[string]string ServerSideEncryption encrypt.ServerSide VersionID string + PartNumber int // To be not used by external applications Internal AdvancedGetOptions } diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object.go b/vendor/github.com/minio/minio-go/v7/api-put-object.go index fb61b407..e8a964e2 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object.go @@ -214,13 +214,20 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // // You must have WRITE permissions on a bucket to create an object. // -// - For size smaller than 128MiB PutObject automatically does a -// single atomic Put operation. -// - For size larger than 128MiB PutObject automatically does a -// multipart Put operation. +// - For size smaller than 16MiB PutObject automatically does a +// single atomic PUT operation. +// +// - For size larger than 16MiB PutObject automatically does a +// multipart upload operation. +// // - For size input as -1 PutObject does a multipart Put operation // until input stream reaches EOF. Maximum object size that can // be uploaded through this operation will be 5TiB. +// +// WARNING: Passing down '-1' will use memory and these cannot +// be reused for best outcomes for PutObject(), pass the size always. +// +// NOTE: Upon errors during upload multipart operation is entirely aborted. func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) (info UploadInfo, err error) { if objectSize < 0 && opts.DisableMultipart { diff --git a/vendor/github.com/minio/minio-go/v7/api-remove.go b/vendor/github.com/minio/minio-go/v7/api-remove.go index fd3f1e12..0fee9022 100644 --- a/vendor/github.com/minio/minio-go/v7/api-remove.go +++ b/vendor/github.com/minio/minio-go/v7/api-remove.go @@ -136,11 +136,11 @@ func (c *Client) RemoveObject(ctx context.Context, bucketName, objectName string return err } - return c.removeObject(ctx, bucketName, objectName, opts) + res := c.removeObject(ctx, bucketName, objectName, opts) + return res.Err } -func (c *Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { - +func (c *Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) RemoveObjectResult { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) @@ -181,19 +181,25 @@ func (c *Client) removeObject(ctx context.Context, bucketName, objectName string }) defer closeResponse(resp) if err != nil { - return err + return RemoveObjectResult{Err: err} } if resp != nil { // if some unexpected error happened and max retry is reached, we want to let client know if resp.StatusCode != http.StatusNoContent { - return httpRespToErrorResponse(resp, bucketName, objectName) + err := httpRespToErrorResponse(resp, bucketName, objectName) + return RemoveObjectResult{Err: err} } } // DeleteObject always responds with http '204' even for // objects which do not exist. So no need to handle them // specifically. - return nil + return RemoveObjectResult{ + ObjectName: objectName, + ObjectVersionID: opts.VersionID, + DeleteMarker: resp.Header.Get("x-amz-delete-marker") == "true", + DeleteMarkerVersionID: resp.Header.Get("x-amz-version-id"), + } } // RemoveObjectError - container of Multi Delete S3 API error @@ -203,6 +209,17 @@ type RemoveObjectError struct { Err error } +// RemoveObjectResult - container of Multi Delete S3 API result +type RemoveObjectResult struct { + ObjectName string + ObjectVersionID string + + DeleteMarker bool + DeleteMarkerVersionID string + + Err error +} + // generateRemoveMultiObjects - generate the XML request for remove multi objects request func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte { delObjects := []deleteObject{} @@ -212,21 +229,32 @@ func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte { VersionID: obj.VersionID, }) } - xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: true}) + xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: false}) return xmlBytes } // processRemoveMultiObjectsResponse - parse the remove multi objects web service // and return the success/failure result status for each object -func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, errorCh chan<- RemoveObjectError) { +func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, resultCh chan<- RemoveObjectResult) { // Parse multi delete XML response rmResult := &deleteMultiObjectsResult{} err := xmlDecoder(body, rmResult) if err != nil { - errorCh <- RemoveObjectError{ObjectName: "", Err: err} + resultCh <- RemoveObjectResult{ObjectName: "", Err: err} return } + // Fill deletion that returned success + for _, obj := range rmResult.DeletedObjects { + resultCh <- RemoveObjectResult{ + ObjectName: obj.Key, + // Only filled with versioned buckets + ObjectVersionID: obj.VersionID, + DeleteMarker: obj.DeleteMarker, + DeleteMarkerVersionID: obj.DeleteMarkerVersionID, + } + } + // Fill deletion that returned an error. for _, obj := range rmResult.UnDeletedObjects { // Version does not exist is not an error ignore and continue. @@ -234,9 +262,9 @@ func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, err case "InvalidArgument", "NoSuchVersion": continue } - errorCh <- RemoveObjectError{ - ObjectName: obj.Key, - VersionID: obj.VersionID, + resultCh <- RemoveObjectResult{ + ObjectName: obj.Key, + ObjectVersionID: obj.VersionID, Err: ErrorResponse{ Code: obj.Code, Message: obj.Message, @@ -273,10 +301,54 @@ func (c *Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh return errorCh } - go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts) + resultCh := make(chan RemoveObjectResult, 1) + go c.removeObjects(ctx, bucketName, objectsCh, resultCh, opts) + go func() { + defer close(errorCh) + for res := range resultCh { + // Send only errors to the error channel + if res.Err == nil { + continue + } + errorCh <- RemoveObjectError{ + ObjectName: res.ObjectName, + VersionID: res.ObjectVersionID, + Err: res.Err, + } + } + }() + return errorCh } +// RemoveObjectsWithResult removes multiple objects from a bucket while +// it is possible to specify objects versions which are received from +// objectsCh. Remove results, successes and failures are sent back via +// RemoveObjectResult channel +func (c *Client) RemoveObjectsWithResult(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectResult { + resultCh := make(chan RemoveObjectResult, 1) + + // Validate if bucket name is valid. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(resultCh) + resultCh <- RemoveObjectResult{ + Err: err, + } + return resultCh + } + // Validate objects channel to be properly allocated. + if objectsCh == nil { + defer close(resultCh) + resultCh <- RemoveObjectResult{ + Err: errInvalidArgument("Objects channel cannot be nil"), + } + return resultCh + } + + go c.removeObjects(ctx, bucketName, objectsCh, resultCh, opts) + return resultCh +} + // Return true if the character is within the allowed characters in an XML 1.0 document // The list of allowed characters can be found here: https://www.w3.org/TR/xml/#charsets func validXMLChar(r rune) (ok bool) { @@ -298,14 +370,14 @@ func hasInvalidXMLChar(str string) bool { } // Generate and call MultiDelete S3 requests based on entries received from objectsCh -func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) { +func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, resultCh chan<- RemoveObjectResult, opts RemoveObjectsOptions) { maxEntries := 1000 finish := false urlValues := make(url.Values) urlValues.Set("delete", "") - // Close error channel when Multi delete finishes. - defer close(errorCh) + // Close result channel when Multi delete finishes. + defer close(resultCh) // Loop over entries by 1000 and call MultiDelete requests for { @@ -319,22 +391,20 @@ func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh for object := range objectsCh { if hasInvalidXMLChar(object.Key) { // Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document. - err := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{ + removeResult := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{ VersionID: object.VersionID, GovernanceBypass: opts.GovernanceBypass, }) - if err != nil { + if err := removeResult.Err; err != nil { // Version does not exist is not an error ignore and continue. switch ToErrorResponse(err).Code { case "InvalidArgument", "NoSuchVersion": continue } - errorCh <- RemoveObjectError{ - ObjectName: object.Key, - VersionID: object.VersionID, - Err: err, - } + resultCh <- removeResult } + + resultCh <- removeResult continue } @@ -374,22 +444,22 @@ func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh if resp != nil { if resp.StatusCode != http.StatusOK { e := httpRespToErrorResponse(resp, bucketName, "") - errorCh <- RemoveObjectError{ObjectName: "", Err: e} + resultCh <- RemoveObjectResult{ObjectName: "", Err: e} } } if err != nil { for _, b := range batch { - errorCh <- RemoveObjectError{ - ObjectName: b.Key, - VersionID: b.VersionID, - Err: err, + resultCh <- RemoveObjectResult{ + ObjectName: b.Key, + ObjectVersionID: b.VersionID, + Err: err, } } continue } // Process multiobjects remove xml response - processRemoveMultiObjectsResponse(resp.Body, batch, errorCh) + processRemoveMultiObjectsResponse(resp.Body, batch, resultCh) closeResponse(resp) } diff --git a/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go b/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go index 948f8a74..592d4cdc 100644 --- a/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go +++ b/vendor/github.com/minio/minio-go/v7/api-s3-datatypes.go @@ -335,7 +335,7 @@ type deletedObject struct { VersionID string `xml:"VersionId,omitempty"` // These fields are ignored. DeleteMarker bool - DeleteMarkerVersionID string + DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"` } // nonDeletedObject container for Error element (failed deletion) in MultiObjects Delete XML response diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go index b5eaa690..357cd1b4 100644 --- a/vendor/github.com/minio/minio-go/v7/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -111,7 +111,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.16" + libraryVersion = "v7.0.21" ) // User Agent should always following the below style. @@ -182,67 +182,6 @@ func (r *lockedRandSource) Seed(seed int64) { r.lk.Unlock() } -// Redirect requests by re signing the request. -func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { - if len(via) >= 5 { - return errors.New("stopped after 5 redirects") - } - if len(via) == 0 { - return nil - } - lastRequest := via[len(via)-1] - var reAuth bool - for attr, val := range lastRequest.Header { - // if hosts do not match do not copy Authorization header - if attr == "Authorization" && req.Host != lastRequest.Host { - reAuth = true - continue - } - if _, ok := req.Header[attr]; !ok { - req.Header[attr] = val - } - } - - *c.endpointURL = *req.URL - - value, err := c.credsProvider.Get() - if err != nil { - return err - } - var ( - signerType = value.SignerType - accessKeyID = value.AccessKeyID - secretAccessKey = value.SecretAccessKey - sessionToken = value.SessionToken - region = c.region - ) - - // Custom signer set then override the behavior. - if c.overrideSignerType != credentials.SignatureDefault { - signerType = c.overrideSignerType - } - - // If signerType returned by credentials helper is anonymous, - // then do not sign regardless of signerType override. - if value.SignerType == credentials.SignatureAnonymous { - signerType = credentials.SignatureAnonymous - } - - if reAuth { - // Check if there is no region override, if not get it from the URL if possible. - if region == "" { - region = s3utils.GetRegionFromURL(*c.endpointURL) - } - switch { - case signerType.IsV2(): - return errors.New("signature V2 cannot support redirection") - case signerType.IsV4(): - signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) - } - } - return nil -} - func privateNew(endpoint string, opts *Options) (*Client, error) { // construct endpoint. endpointURL, err := getEndpointURL(endpoint, opts.Secure) @@ -279,9 +218,11 @@ func privateNew(endpoint string, opts *Options) (*Client, error) { // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ - Jar: jar, - Transport: transport, - CheckRedirect: clnt.redirectHeaders, + Jar: jar, + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, } // Sets custom region, if region is empty bucket location cache is used automatically. @@ -917,8 +858,8 @@ func (c *Client) makeTargetURL(bucketName, objectName, bucketLocation string, is // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html host = c.s3AccelerateEndpoint } else { - // Do not change the host if the endpoint URL is a FIPS S3 endpoint. - if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) { + // Do not change the host if the endpoint URL is a FIPS S3 endpoint or a S3 PrivateLink interface endpoint + if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) && !s3utils.IsAmazonPrivateLinkEndpoint(*c.endpointURL) { // Fetch new host based on the bucket location. host = getS3Endpoint(bucketLocation) } diff --git a/vendor/github.com/minio/minio-go/v7/functional_tests.go b/vendor/github.com/minio/minio-go/v7/functional_tests.go index b8950dd2..413b63e5 100644 --- a/vendor/github.com/minio/minio-go/v7/functional_tests.go +++ b/vendor/github.com/minio/minio-go/v7/functional_tests.go @@ -1,3 +1,4 @@ +//go:build mint // +build mint /* @@ -2627,6 +2628,138 @@ func testRemoveMultipleObjects() { successLogger(testName, function, args, startTime).Info() } +// Test removing multiple objects and check for results +func testRemoveMultipleObjectsWithResult() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "RemoveObjects(bucketName, objectsCh)" + args := map[string]interface{}{ + "bucketName": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + defer cleanupVersionedBucket(bucketName, c) + + r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) + + nrObjects := 10 + nrLockedObjects := 5 + + objectsCh := make(chan minio.ObjectInfo) + + go func() { + defer close(objectsCh) + // Upload objects and send them to objectsCh + for i := 0; i < nrObjects; i++ { + objectName := "sample" + strconv.Itoa(i) + ".txt" + info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8, + minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + if i < nrLockedObjects { + // t := time.Date(2130, time.April, 25, 14, 0, 0, 0, time.UTC) + t := time.Now().Add(5 * time.Minute) + m := minio.RetentionMode(minio.Governance) + opts := minio.PutObjectRetentionOptions{ + GovernanceBypass: false, + RetainUntilDate: &t, + Mode: &m, + VersionID: info.VersionID, + } + err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts) + if err != nil { + logError(testName, function, args, startTime, "", "Error setting retention", err) + return + } + } + + objectsCh <- minio.ObjectInfo{ + Key: info.Key, + VersionID: info.VersionID, + } + } + }() + + // Call RemoveObjects API + resultCh := c.RemoveObjectsWithResult(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) + + var foundNil, foundErr int + + for { + // Check if errorCh doesn't receive any error + select { + case deleteRes, ok := <-resultCh: + if !ok { + goto out + } + if deleteRes.ObjectName == "" { + logError(testName, function, args, startTime, "", "Unexpected object name", nil) + return + } + if deleteRes.ObjectVersionID == "" { + logError(testName, function, args, startTime, "", "Unexpected object version ID", nil) + return + } + + if deleteRes.Err == nil { + foundNil++ + } else { + foundErr++ + } + } + } +out: + if foundNil+foundErr != nrObjects { + logError(testName, function, args, startTime, "", "Unexpected number of results", nil) + return + } + + if foundNil != nrObjects-nrLockedObjects { + logError(testName, function, args, startTime, "", "Unexpected number of nil errors", nil) + return + } + + if foundErr != nrLockedObjects { + logError(testName, function, args, startTime, "", "Unexpected number of errors", nil) + return + } + + successLogger(testName, function, args, startTime).Info() +} + // Tests FPutObject of a big file to trigger multipart func testFPutObjectMultipart() { // initialize logging params @@ -11297,12 +11430,6 @@ func testGetObjectACLContext() { // Seed random based on current time. rand.Seed(time.Now().Unix()) - // skipping region functional tests for non s3 runs - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - // Instantiate new minio client object. c, err := minio.New(os.Getenv(serverEndpoint), &minio.Options{ @@ -11379,6 +11506,17 @@ func testGetObjectACLContext() { return } + // Do a very limited testing if this is not AWS S3 + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + if s[0] != "private" { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"private\" but got"+fmt.Sprintf("%q", s[0]), nil) + return + } + + successLogger(testName, function, args, startTime).Info() + return + } + if s[0] != "public-read-write" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) return @@ -11978,6 +12116,7 @@ func main() { // Default to KMS tests. kms = true } + // execute tests if isFullMode() { testMakeBucketErrorV2() @@ -12009,6 +12148,7 @@ func main() { testGetObjectClosedTwice() testGetObjectS3Zip() testRemoveMultipleObjects() + testRemoveMultipleObjectsWithResult() testFPutObjectMultipart() testFPutObject() testGetObjectReadSeekFunctional() diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go index 3b1b547b..107a11b1 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/assume_role.go @@ -18,6 +18,7 @@ package credentials import ( + "bytes" "encoding/hex" "encoding/xml" "errors" @@ -184,11 +185,26 @@ func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssume } defer closeResponse(resp) if resp.StatusCode != http.StatusOK { - return AssumeRoleResponse{}, errors.New(resp.Status) + var errResp ErrorResponse + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return AssumeRoleResponse{}, err + } + _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) + if err != nil { + var s3Err Error + if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { + return AssumeRoleResponse{}, err + } + errResp.RequestID = s3Err.RequestID + errResp.STSError.Code = s3Err.Code + errResp.STSError.Message = s3Err.Message + } + return AssumeRoleResponse{}, errResp } a := AssumeRoleResponse{} - if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { + if _, err = xmlDecodeAndBody(resp.Body, &a); err != nil { return AssumeRoleResponse{}, err } return a, nil diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/error_response.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/error_response.go new file mode 100644 index 00000000..f4b027a4 --- /dev/null +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/error_response.go @@ -0,0 +1,96 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "io/ioutil" +) + +// ErrorResponse - Is the typed error returned. +// ErrorResponse struct should be comparable since it is compared inside +// golang http API (https://github.com/golang/go/issues/29768) +type ErrorResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ ErrorResponse" json:"-"` + STSError struct { + Type string `xml:"Type"` + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` + RequestID string `xml:"RequestId"` +} + +// Error - Is the typed error returned by all API operations. +type Error struct { + XMLName xml.Name `xml:"Error" json:"-"` + Code string + Message string + BucketName string + Key string + Resource string + RequestID string `xml:"RequestId"` + HostID string `xml:"HostId"` + + // Region where the bucket is located. This header is returned + // only in HEAD bucket and ListObjects response. + Region string + + // Captures the server string returned in response header. + Server string + + // Underlying HTTP status code for the returned error + StatusCode int `xml:"-" json:"-"` +} + +// Error - Returns S3 error string. +func (e Error) Error() string { + if e.Message == "" { + return fmt.Sprintf("Error response code %s.", e.Code) + } + return e.Message +} + +// Error - Returns STS error string. +func (e ErrorResponse) Error() string { + if e.STSError.Message == "" { + return fmt.Sprintf("Error response code %s.", e.STSError.Code) + } + return e.STSError.Message +} + +// xmlDecoder provide decoded value in xml. +func xmlDecoder(body io.Reader, v interface{}) error { + d := xml.NewDecoder(body) + return d.Decode(v) +} + +// xmlDecodeAndBody reads the whole body up to 1MB and +// tries to XML decode it into v. +// The body that was read and any error from reading or decoding is returned. +func xmlDecodeAndBody(bodyReader io.Reader, v interface{}) ([]byte, error) { + // read the whole body (up to 1MB) + const maxBodyLength = 1 << 20 + body, err := ioutil.ReadAll(io.LimitReader(bodyReader, maxBodyLength)) + if err != nil { + return nil, err + } + return bytes.TrimSpace(body), xmlDecoder(bytes.NewReader(body), v) +} diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/signature-type.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/signature_type.go index b7943330..b7943330 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/signature-type.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/signature_type.go diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go index b79f920f..b6712b19 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_client_grants.go @@ -18,9 +18,11 @@ package credentials import ( + "bytes" "encoding/xml" "errors" "fmt" + "io/ioutil" "net/http" "net/url" "time" @@ -132,7 +134,23 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string, } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return AssumeRoleWithClientGrantsResponse{}, errors.New(resp.Status) + var errResp ErrorResponse + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return AssumeRoleWithClientGrantsResponse{}, err + + } + _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) + if err != nil { + var s3Err Error + if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { + return AssumeRoleWithClientGrantsResponse{}, err + } + errResp.RequestID = s3Err.RequestID + errResp.STSError.Code = s3Err.Code + errResp.STSError.Message = s3Err.Message + } + return AssumeRoleWithClientGrantsResponse{}, errResp } a := AssumeRoleWithClientGrantsResponse{} diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go index bdde1fa3..39c7892b 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go @@ -18,9 +18,10 @@ package credentials import ( + "bytes" "encoding/xml" - "errors" "fmt" + "io/ioutil" "net/http" "net/url" "time" @@ -169,7 +170,23 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return value, errors.New(resp.Status) + var errResp ErrorResponse + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return value, err + + } + _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) + if err != nil { + var s3Err Error + if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { + return value, err + } + errResp.RequestID = s3Err.RequestID + errResp.STSError.Code = s3Err.Code + errResp.STSError.Message = s3Err.Message + } + return value, errResp } r := AssumeRoleWithLDAPResponse{} diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts-tls-identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go index 2e37025a..7f485d63 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts-tls-identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_tls_identity.go @@ -16,10 +16,12 @@ package credentials import ( + "bytes" "crypto/tls" "encoding/xml" "errors" "io" + "io/ioutil" "net" "net/http" "net/url" @@ -149,7 +151,23 @@ func (i *STSCertificateIdentity) Retrieve() (Value, error) { defer resp.Body.Close() } if resp.StatusCode != http.StatusOK { - return Value{}, errors.New(resp.Status) + var errResp ErrorResponse + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return Value{}, err + + } + _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) + if err != nil { + var s3Err Error + if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { + return Value{}, err + } + errResp.RequestID = s3Err.RequestID + errResp.STSError.Code = s3Err.Code + errResp.STSError.Message = s3Err.Message + } + return Value{}, errResp } const MaxSize = 10 * 1 << 20 diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go index 25ca751d..98f6ea65 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go @@ -18,9 +18,11 @@ package credentials import ( + "bytes" "encoding/xml" "errors" "fmt" + "io/ioutil" "net/http" "net/url" "strconv" @@ -150,7 +152,23 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return AssumeRoleWithWebIdentityResponse{}, errors.New(resp.Status) + var errResp ErrorResponse + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return AssumeRoleWithWebIdentityResponse{}, err + + } + _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) + if err != nil { + var s3Err Error + if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { + return AssumeRoleWithWebIdentityResponse{}, err + } + errResp.RequestID = s3Err.RequestID + errResp.STSError.Code = s3Err.Code + errResp.STSError.Message = s3Err.Message + } + return AssumeRoleWithWebIdentityResponse{}, errResp } a := AssumeRoleWithWebIdentityResponse{} diff --git a/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go b/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go index 96f1101c..743d8eca 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go @@ -53,12 +53,12 @@ func (n AbortIncompleteMultipartUpload) MarshalXML(e *xml.Encoder, start xml.Sta // (or suspended) to request server delete noncurrent object versions at a // specific period in the object's lifetime. type NoncurrentVersionExpiration struct { - XMLName xml.Name `xml:"NoncurrentVersionExpiration" json:"-"` - NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"` - MaxNoncurrentVersions int `xml:"MaxNoncurrentVersions,omitempty"` + XMLName xml.Name `xml:"NoncurrentVersionExpiration" json:"-"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"` + NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty"` } -// MarshalXML if non-current days not set to non zero value +// MarshalXML if n is non-empty, i.e has a non-zero NoncurrentDays or NewerNoncurrentVersions. func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if n.isNull() { return nil @@ -73,16 +73,17 @@ func (n NoncurrentVersionExpiration) IsDaysNull() bool { } func (n NoncurrentVersionExpiration) isNull() bool { - return n.IsDaysNull() && n.MaxNoncurrentVersions == 0 + return n.IsDaysNull() && n.NewerNoncurrentVersions == 0 } // NoncurrentVersionTransition structure, set this action to request server to // transition noncurrent object versions to different set storage classes // at a specific period in the object's lifetime. type NoncurrentVersionTransition struct { - XMLName xml.Name `xml:"NoncurrentVersionTransition,omitempty" json:"-"` - StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` - NoncurrentDays ExpirationDays `xml:"NoncurrentDays" json:"NoncurrentDays"` + XMLName xml.Name `xml:"NoncurrentVersionTransition,omitempty" json:"-"` + StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays" json:"NoncurrentDays"` + NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty" json:"NewerNoncurrentVersions,omitempty"` } // IsDaysNull returns true if days field is null diff --git a/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go b/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go index 44945464..2f1a5a65 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/s3utils/utils.go @@ -104,6 +104,9 @@ var elbAmazonRegex = regexp.MustCompile(`elb(.*?).amazonaws.com$`) // Regular expression used to determine if the arg is elb host in china. var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`) +// amazonS3HostPrivateLink - regular expression used to determine if an arg is s3 host in AWS PrivateLink interface endpoints style +var amazonS3HostPrivateLink = regexp.MustCompile(`^(?:bucket|accesspoint).vpce-.*?.s3.(.*?).vpce.amazonaws.com$`) + // GetRegionFromURL - returns a region from url host. func GetRegionFromURL(endpointURL url.URL) string { if endpointURL == sentinelURL { @@ -139,6 +142,10 @@ func GetRegionFromURL(endpointURL url.URL) string { if len(parts) > 1 { return parts[1] } + parts = amazonS3HostPrivateLink.FindStringSubmatch(endpointURL.Host) + if len(parts) > 1 { + return parts[1] + } return "" } @@ -202,6 +209,15 @@ func IsAmazonFIPSEndpoint(endpointURL url.URL) bool { return IsAmazonFIPSUSEastWestEndpoint(endpointURL) || IsAmazonFIPSGovCloudEndpoint(endpointURL) } +// IsAmazonPrivateLinkEndpoint - Match if it is exactly Amazon S3 PrivateLink interface endpoint +// See https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html. +func IsAmazonPrivateLinkEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + return amazonS3HostPrivateLink.MatchString(endpointURL.Host) +} + // IsGoogleEndpoint - Match if it is exactly Google cloud storage endpoint. func IsGoogleEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { diff --git a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v2.go b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v2.go index b6ea78f7..cf7921d1 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v2.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v2.go @@ -243,10 +243,14 @@ func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) { // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign // Whitelist resource list that will be used in query string for signature-V2 calculation. -// The list should be alphabetically sorted +// +// This list should be kept alphabetically sorted, do not hastily edit. var resourceList = []string{ "acl", + "cors", "delete", + "encryption", + "legal-hold", "lifecycle", "location", "logging", @@ -261,6 +265,10 @@ var resourceList = []string{ "response-content-language", "response-content-type", "response-expires", + "retention", + "select", + "select-type", + "tagging", "torrent", "uploadId", "uploads", diff --git a/vendor/github.com/minio/minio-go/v7/transport.go b/vendor/github.com/minio/minio-go/v7/transport.go index d5ad15b8..a88477b7 100644 --- a/vendor/github.com/minio/minio-go/v7/transport.go +++ b/vendor/github.com/minio/minio-go/v7/transport.go @@ -1,3 +1,4 @@ +//go:build go1.7 || go1.8 // +build go1.7 go1.8 /* |