From 3893a035be347a7687a41d2054dd1b274d3a0504 Mon Sep 17 00:00:00 2001 From: Wim Date: Sun, 12 Dec 2021 00:05:15 +0100 Subject: Update dependencies/vendor (#1659) --- .../minio-go/v7/pkg/credentials/credentials.go | 20 +++- .../minio/minio-go/v7/pkg/credentials/iam_aws.go | 13 ++- .../v7/pkg/credentials/sts_ldap_identity.go | 96 +++++++++++++++--- .../v7/pkg/credentials/sts_web_identity.go | 9 +- .../minio/minio-go/v7/pkg/lifecycle/lifecycle.go | 107 +++++++++++++++++++++ .../minio-go/v7/pkg/replication/replication.go | 97 ++++++++++++------- 6 files changed, 278 insertions(+), 64 deletions(-) (limited to 'vendor/github.com/minio/minio-go/v7/pkg') diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go index 62d1701e..6b93a27f 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/credentials.go @@ -22,8 +22,13 @@ import ( "time" ) -// STSVersion sts version string -const STSVersion = "2011-06-15" +const ( + // STSVersion sts version string + STSVersion = "2011-06-15" + + // How much duration to slash from the given expiration duration + defaultExpiryWindow = 0.8 +) // A Value is the AWS credentials value for individual credential fields. type Value struct { @@ -82,10 +87,15 @@ type Expiry struct { // the expiration time given to ensure no requests are made with expired // tokens. func (e *Expiry) SetExpiration(expiration time.Time, window time.Duration) { - e.expiration = expiration - if window > 0 { - e.expiration = e.expiration.Add(-window) + if e.CurrentTime == nil { + e.CurrentTime = time.Now + } + cut := window + if cut < 0 { + expireIn := expiration.Sub(e.CurrentTime()) + cut = time.Duration(float64(expireIn) * (1 - defaultExpiryWindow)) } + e.expiration = expiration.Add(-cut) } // IsExpired returns if the credentials are expired. diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go index b532bcb6..bbd25ed8 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go @@ -38,7 +38,10 @@ import ( // prior to the credentials actually expiring. This is beneficial // so race conditions with expiring credentials do not cause // request to fail unexpectedly due to ExpiredTokenException exceptions. -const DefaultExpiryWindow = time.Second * 10 // 10 secs +// DefaultExpiryWindow can be used as parameter to (*Expiry).SetExpiration. +// When used the tokens refresh will be triggered when 80% of the elapsed +// time until the actual expiration time is passed. +const DefaultExpiryWindow = -1 // A IAM retrieves credentials from the EC2 service, and keeps track if // those credentials are expired. @@ -181,10 +184,6 @@ type ec2RoleCredRespBody struct { // be sent to fetch the rolling access credentials. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func getIAMRoleURL(endpoint string) (*url.URL, error) { - if endpoint == "" { - endpoint = defaultIAMRoleEndpoint - } - u, err := url.Parse(endpoint) if err != nil { return nil, err @@ -281,6 +280,10 @@ func fetchIMDSToken(client *http.Client, endpoint string) (string, error) { // If the credentials cannot be found, or there is an error // reading the response an error will be returned. func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { + if endpoint == "" { + endpoint = defaultIAMRoleEndpoint + } + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html token, _ := fetchIMDSToken(client, endpoint) diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go index bcb3c36a..0fa5b55f 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. + * Copyright 2019-2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package credentials import ( "encoding/xml" "errors" + "fmt" "net/http" "net/url" "time" @@ -60,26 +61,86 @@ type LDAPIdentity struct { // LDAP username/password used to fetch LDAP STS credentials. LDAPUsername, LDAPPassword string + + // Session policy to apply to the generated credentials. Leave empty to + // use the full access policy available to the user. + Policy string + + // RequestedExpiry is the configured expiry duration for credentials + // requested from LDAP. + RequestedExpiry time.Duration } // NewLDAPIdentity returns new credentials object that uses LDAP // Identity. -func NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword string) (*Credentials, error) { +func NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword string, optFuncs ...LDAPIdentityOpt) (*Credentials, error) { + l := LDAPIdentity{ + Client: &http.Client{Transport: http.DefaultTransport}, + STSEndpoint: stsEndpoint, + LDAPUsername: ldapUsername, + LDAPPassword: ldapPassword, + } + for _, optFunc := range optFuncs { + optFunc(&l) + } + return New(&l), nil +} + +// LDAPIdentityOpt is a function type used to configured the LDAPIdentity +// instance. +type LDAPIdentityOpt func(*LDAPIdentity) + +// LDAPIdentityPolicyOpt sets the session policy for requested credentials. +func LDAPIdentityPolicyOpt(policy string) LDAPIdentityOpt { + return func(k *LDAPIdentity) { + k.Policy = policy + } +} + +// LDAPIdentityExpiryOpt sets the expiry duration for requested credentials. +func LDAPIdentityExpiryOpt(d time.Duration) LDAPIdentityOpt { + return func(k *LDAPIdentity) { + k.RequestedExpiry = d + } +} + +func stripPassword(err error) error { + urlErr, ok := err.(*url.Error) + if ok { + u, _ := url.Parse(urlErr.URL) + if u == nil { + return urlErr + } + values := u.Query() + values.Set("LDAPPassword", "xxxxx") + u.RawQuery = values.Encode() + urlErr.URL = u.String() + return urlErr + } + return err +} + +// NewLDAPIdentityWithSessionPolicy returns new credentials object that uses +// LDAP Identity with a specified session policy. The `policy` parameter must be +// a JSON string specifying the policy document. +// +// DEPRECATED: Use the `LDAPIdentityPolicyOpt` with `NewLDAPIdentity` instead. +func NewLDAPIdentityWithSessionPolicy(stsEndpoint, ldapUsername, ldapPassword, policy string) (*Credentials, error) { return New(&LDAPIdentity{ Client: &http.Client{Transport: http.DefaultTransport}, STSEndpoint: stsEndpoint, LDAPUsername: ldapUsername, LDAPPassword: ldapPassword, + Policy: policy, }), nil } // Retrieve gets the credential by calling the MinIO STS API for // LDAP on the configured stsEndpoint. func (k *LDAPIdentity) Retrieve() (value Value, err error) { - u, kerr := url.Parse(k.STSEndpoint) - if kerr != nil { - err = kerr - return + u, err := url.Parse(k.STSEndpoint) + if err != nil { + return value, err } v := url.Values{} @@ -87,25 +148,28 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) { v.Set("Version", STSVersion) v.Set("LDAPUsername", k.LDAPUsername) v.Set("LDAPPassword", k.LDAPPassword) + if k.Policy != "" { + v.Set("Policy", k.Policy) + } + if k.RequestedExpiry != 0 { + v.Set("DurationSeconds", fmt.Sprintf("%d", int(k.RequestedExpiry.Seconds()))) + } u.RawQuery = v.Encode() - req, kerr := http.NewRequest(http.MethodPost, u.String(), nil) - if kerr != nil { - err = kerr - return + req, err := http.NewRequest(http.MethodPost, u.String(), nil) + if err != nil { + return value, stripPassword(err) } - resp, kerr := k.Client.Do(req) - if kerr != nil { - err = kerr - return + resp, err := k.Client.Do(req) + if err != nil { + return value, stripPassword(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - err = errors.New(resp.Status) - return + return value, errors.New(resp.Status) } r := AssumeRoleWithLDAPResponse{} diff --git a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go index 161ffd36..c1109140 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_web_identity.go @@ -54,8 +54,9 @@ type WebIdentityResult struct { // WebIdentityToken - web identity token with expiry. type WebIdentityToken struct { - Token string - Expiry int + Token string + AccessToken string + Expiry int } // A STSWebIdentity retrieves credentials from MinIO service, and keeps track if @@ -121,6 +122,10 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession v.Set("RoleSessionName", roleSessionName) } v.Set("WebIdentityToken", idToken.Token) + if idToken.AccessToken != "" { + // Usually set when server is using extended userInfo endpoint. + v.Set("WebIdentityAccessToken", idToken.AccessToken) + } if idToken.Expiry > 0 { v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry)) } diff --git a/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go b/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go index b6f9601b..83870a36 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go @@ -19,6 +19,7 @@ package lifecycle import ( + "encoding/json" "encoding/xml" "time" ) @@ -116,6 +117,26 @@ type Transition struct { Days ExpirationDays `xml:"Days,omitempty" json:"Days,omitempty"` } +// MarshalJSON customizes json encoding by omitting empty values +func (t Transition) MarshalJSON() ([]byte, error) { + type transition struct { + Date *ExpirationDate `json:"Date,omitempty"` + StorageClass string `json:"StorageClass,omitempty"` + Days *ExpirationDays `json:"Days,omitempty"` + } + + newt := transition{ + StorageClass: t.StorageClass, + } + if !t.IsDaysNull() { + newt.Days = &t.Days + } + if !t.IsDateNull() { + newt.Date = &t.Date + } + return json.Marshal(newt) +} + // IsDaysNull returns true if days field is null func (t Transition) IsDaysNull() bool { return t.Days == ExpirationDays(0) @@ -160,6 +181,31 @@ type Filter struct { Tag Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` } +// IsNull returns true if all Filter fields are empty. +func (f Filter) IsNull() bool { + return f.Tag.IsEmpty() && f.And.IsEmpty() && f.Prefix == "" +} + +// MarshalJSON customizes json encoding by removing empty values. +func (f Filter) MarshalJSON() ([]byte, error) { + type filter struct { + And *And `json:"And,omitempty"` + Prefix string `json:"Prefix,omitempty"` + Tag *Tag `json:"Tag,omitempty"` + } + + newf := filter{ + Prefix: f.Prefix, + } + if !f.Tag.IsEmpty() { + newf.Tag = &f.Tag + } + if !f.And.IsEmpty() { + newf.And = &f.And + } + return json.Marshal(newf) +} + // MarshalXML - produces the xml representation of the Filter struct // only one of Prefix, And and Tag should be present in the output. func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -238,6 +284,26 @@ type Expiration struct { DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker,omitempty"` } +// MarshalJSON customizes json encoding by removing empty day/date specification. +func (e Expiration) MarshalJSON() ([]byte, error) { + type expiration struct { + Date *ExpirationDate `json:"Date,omitempty"` + Days *ExpirationDays `json:"Days,omitempty"` + DeleteMarker ExpireDeleteMarker + } + + newexp := expiration{ + DeleteMarker: e.DeleteMarker, + } + if !e.IsDaysNull() { + newexp.Days = &e.Days + } + if !e.IsDateNull() { + newexp.Date = &e.Date + } + return json.Marshal(newexp) +} + // IsDaysNull returns true if days field is null func (e Expiration) IsDaysNull() bool { return e.Days == ExpirationDays(0) @@ -267,6 +333,47 @@ func (e Expiration) MarshalXML(en *xml.Encoder, startElement xml.StartElement) e return en.EncodeElement(expirationWrapper(e), startElement) } +// MarshalJSON customizes json encoding by omitting empty values +func (r Rule) MarshalJSON() ([]byte, error) { + type rule struct { + AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `json:"AbortIncompleteMultipartUpload,omitempty"` + Expiration *Expiration `json:"Expiration,omitempty"` + ID string `json:"ID"` + RuleFilter *Filter `json:"Filter,omitempty"` + NoncurrentVersionExpiration *NoncurrentVersionExpiration `json:"NoncurrentVersionExpiration,omitempty"` + NoncurrentVersionTransition *NoncurrentVersionTransition `json:"NoncurrentVersionTransition,omitempty"` + Prefix string `json:"Prefix,omitempty"` + Status string `json:"Status"` + Transition *Transition `json:"Transition,omitempty"` + } + newr := rule{ + Prefix: r.Prefix, + Status: r.Status, + ID: r.ID, + } + + if !r.RuleFilter.IsNull() { + newr.RuleFilter = &r.RuleFilter + } + if !r.AbortIncompleteMultipartUpload.IsDaysNull() { + newr.AbortIncompleteMultipartUpload = &r.AbortIncompleteMultipartUpload + } + if !r.Expiration.IsNull() { + newr.Expiration = &r.Expiration + } + if !r.Transition.IsNull() { + newr.Transition = &r.Transition + } + if !r.NoncurrentVersionExpiration.IsDaysNull() { + newr.NoncurrentVersionExpiration = &r.NoncurrentVersionExpiration + } + if !r.NoncurrentVersionTransition.IsDaysNull() { + newr.NoncurrentVersionTransition = &r.NoncurrentVersionTransition + } + + return json.Marshal(newr) +} + // Rule represents a single rule in lifecycle configuration type Rule struct { XMLName xml.Name `xml:"Rule,omitempty" json:"-"` diff --git a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go index beacc71f..0211f1fb 100644 --- a/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go +++ b/vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go @@ -47,13 +47,13 @@ const ( // Options represents options to set a replication configuration rule type Options struct { Op OptionType + RoleArn string ID string Prefix string RuleStatus string Priority string TagString string StorageClass string - RoleArn string DestBucket string IsTagSet bool IsSCSet bool @@ -103,9 +103,17 @@ func (c *Config) AddRule(opts Options) error { if err != nil { return err } - if opts.RoleArn != c.Role && c.Role != "" { - return fmt.Errorf("role ARN does not match existing configuration") + if opts.RoleArn != "" { + tokens := strings.Split(opts.RoleArn, ":") + if len(tokens) != 6 { + return fmt.Errorf("invalid format for replication Role Arn: %v", opts.RoleArn) + } + if !strings.HasPrefix(opts.RoleArn, "arn:aws:iam") { + return fmt.Errorf("RoleArn invalid for AWS replication configuration: %v", opts.RoleArn) + } + c.Role = opts.RoleArn } + var status Status // toggle rule status for edit option switch opts.RuleStatus { @@ -139,28 +147,11 @@ func (c *Config) AddRule(opts Options) error { if opts.ID == "" { opts.ID = xid.New().String() } - arnStr := opts.RoleArn - if opts.RoleArn == "" { - arnStr = c.Role - } - if arnStr == "" { - return fmt.Errorf("role ARN required") - } - tokens := strings.Split(arnStr, ":") - if len(tokens) != 6 { - return fmt.Errorf("invalid format for replication Arn") - } - if c.Role == "" { - c.Role = arnStr - } + destBucket := opts.DestBucket // ref https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html if btokens := strings.Split(destBucket, ":"); len(btokens) != 6 { - if len(btokens) == 1 { - destBucket = fmt.Sprintf("arn:aws:s3:::%s", destBucket) - } else { - return fmt.Errorf("destination bucket needs to be in Arn format") - } + return fmt.Errorf("destination bucket needs to be in Arn format") } dmStatus := Disabled if opts.ReplicateDeleteMarkers != "" { @@ -236,13 +227,18 @@ func (c *Config) AddRule(opts Options) error { if err := newRule.Validate(); err != nil { return err } + // if replication config uses RoleArn, migrate this to the destination element as target ARN for remote bucket for MinIO configuration + if c.Role != "" && !strings.HasPrefix(c.Role, "arn:aws:iam") { + for i := range c.Rules { + c.Rules[i].Destination.Bucket = c.Role + } + c.Role = "" + } + for _, rule := range c.Rules { if rule.Priority == newRule.Priority { return fmt.Errorf("priority must be unique. Replication configuration already has a rule with this priority") } - if rule.Destination.Bucket != newRule.Destination.Bucket { - return fmt.Errorf("the destination bucket must be same for all rules") - } if rule.ID == newRule.ID { return fmt.Errorf("a rule exists with this ID") } @@ -257,6 +253,14 @@ func (c *Config) EditRule(opts Options) error { if opts.ID == "" { return fmt.Errorf("rule ID missing") } + // if replication config uses RoleArn, migrate this to the destination element as target ARN for remote bucket for non AWS. + if c.Role != "" && !strings.HasPrefix(c.Role, "arn:aws:iam") { + for i := range c.Rules { + c.Rules[i].Destination.Bucket = c.Role + } + c.Role = "" + } + rIdx := -1 var newRule Rule for i, rule := range c.Rules { @@ -351,7 +355,7 @@ func (c *Config) EditRule(opts Options) error { return fmt.Errorf("replica metadata sync should be either [enable|disable]") } } - fmt.Println("opts.ExistingObjectReplicate>", opts.ExistingObjectReplicate) + if opts.ExistingObjectReplicate != "" { switch opts.ExistingObjectReplicate { case "enable": @@ -376,11 +380,7 @@ func (c *Config) EditRule(opts Options) error { destBucket := opts.DestBucket // ref https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html if btokens := strings.Split(opts.DestBucket, ":"); len(btokens) != 6 { - if len(btokens) == 1 { - destBucket = fmt.Sprintf("arn:aws:s3:::%s", destBucket) - } else { - return fmt.Errorf("destination bucket needs to be in Arn format") - } + return fmt.Errorf("destination bucket needs to be in Arn format") } newRule.Destination.Bucket = destBucket } @@ -393,8 +393,8 @@ func (c *Config) EditRule(opts Options) error { if rule.Priority == newRule.Priority && rIdx != idx { return fmt.Errorf("priority must be unique. Replication configuration already has a rule with this priority") } - if rule.Destination.Bucket != newRule.Destination.Bucket { - return fmt.Errorf("the destination bucket must be same for all rules") + if rule.Destination.Bucket != newRule.Destination.Bucket && rule.ID == newRule.ID { + return fmt.Errorf("invalid destination bucket for this rule") } } @@ -678,9 +678,9 @@ func (e ExistingObjectReplication) Validate() error { return nil } -// Metrics represents inline replication metrics -// such as pending, failed and completed bytes in total for a bucket -type Metrics struct { +// TargetMetrics represents inline replication metrics +// such as pending, failed and completed bytes in total for a bucket remote target +type TargetMetrics struct { // Pending size in bytes PendingSize uint64 `json:"pendingReplicationSize"` // Completed size in bytes @@ -694,3 +694,28 @@ type Metrics struct { // Total number of failed operations including metadata updates FailedCount uint64 `json:"failedReplicationCount"` } + +// Metrics represents inline replication metrics for a bucket. +type Metrics struct { + Stats map[string]TargetMetrics + // Total Pending size in bytes across targets + PendingSize uint64 `json:"pendingReplicationSize"` + // Completed size in bytes across targets + ReplicatedSize uint64 `json:"completedReplicationSize"` + // Total Replica size in bytes across targets + ReplicaSize uint64 `json:"replicaSize"` + // Failed size in bytes across targets + FailedSize uint64 `json:"failedReplicationSize"` + // Total number of pending operations including metadata updates across targets + PendingCount uint64 `json:"pendingReplicationCount"` + // Total number of failed operations including metadata updates across targets + FailedCount uint64 `json:"failedReplicationCount"` +} + +type ResyncTargetsInfo struct { + Targets []ResyncTarget `json:"target,omitempty"` +} +type ResyncTarget struct { + Arn string `json:"arn"` + ResetID string `json:"resetid"` +} -- cgit v1.2.3