bump go-store version (includes minio) (#1657)

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
2023-03-30 22:39:55 +01:00
committed by GitHub
parent 0746ef741a
commit a5c920a50b
33 changed files with 14876 additions and 8512 deletions

View File

@ -43,7 +43,7 @@ type StringMap map[string]string
// if m is nil it can be initialized, which is often the case if m is
// nested in another xml structural. This is also why the first thing done
// on the first line is initialize it.
func (m *StringMap) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
func (m *StringMap) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) error {
*m = StringMap{}
type Item struct {
Key string

View File

@ -387,6 +387,12 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
return UploadInfo{}, err
}
headers := opts.Header()
if s3utils.IsAmazonEndpoint(*c.endpointURL) {
headers.Del(encrypt.SseKmsKeyID) // Remove X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id not supported in CompleteMultipartUpload
headers.Del(encrypt.SseGenericHeader) // Remove X-Amz-Server-Side-Encryption not supported in CompleteMultipartUpload
}
// Instantiate all the complete multipart buffer.
completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes)
reqMetadata := requestMetadata{
@ -396,7 +402,7 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
contentBody: completeMultipartUploadBuffer,
contentLength: int64(len(completeMultipartUploadBytes)),
contentSHA256Hex: sum256Hex(completeMultipartUploadBytes),
customHeader: opts.Header(),
customHeader: headers,
}
// Execute POST to complete multipart upload for an objectName.

View File

@ -235,7 +235,7 @@ func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte {
// processRemoveMultiObjectsResponse - parse the remove multi objects web service
// and return the success/failure result status for each object
func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, resultCh chan<- RemoveObjectResult) {
func processRemoveMultiObjectsResponse(body io.Reader, resultCh chan<- RemoveObjectResult) {
// Parse multi delete XML response
rmResult := &deleteMultiObjectsResult{}
err := xmlDecoder(body, rmResult)
@ -459,7 +459,7 @@ func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh
}
// Process multiobjects remove xml response
processRemoveMultiObjectsResponse(resp.Body, batch, resultCh)
processRemoveMultiObjectsResponse(resp.Body, resultCh)
closeResponse(resp)
}

View File

@ -110,7 +110,7 @@ type ListVersionsResult struct {
// UnmarshalXML is a custom unmarshal code for the response of ListObjectVersions, the custom
// code will unmarshal <Version> and <DeleteMarker> tags and save them in Versions field to
// preserve the lexical order of the listing.
func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) (err error) {
for {
// Read tokens from the XML document in a stream.
t, err := d.Token()

View File

@ -106,6 +106,12 @@ type Options struct {
Region string
BucketLookup BucketLookupType
// Allows setting a custom region lookup based on URL pattern
// not all URL patterns are covered by this library so if you
// have a custom endpoints with many regions you can use this
// function to perform region lookups appropriately.
CustomRegionViaURL func(u url.URL) string
// TrailingHeaders indicates server support of trailing headers.
// Only supported for v4 signatures.
TrailingHeaders bool
@ -118,7 +124,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v7.0.49"
libraryVersion = "v7.0.50"
)
// User Agent should always following the below style.
@ -234,7 +240,11 @@ func privateNew(endpoint string, opts *Options) (*Client, error) {
// Sets custom region, if region is empty bucket location cache is used automatically.
if opts.Region == "" {
opts.Region = s3utils.GetRegionFromURL(*clnt.endpointURL)
if opts.CustomRegionViaURL != nil {
opts.Region = opts.CustomRegionViaURL(*clnt.endpointURL)
} else {
opts.Region = s3utils.GetRegionFromURL(*clnt.endpointURL)
}
}
clnt.region = opts.Region

View File

@ -190,12 +190,11 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
}
}
isVirtualHost := s3utils.IsVirtualHostSupported(targetURL, bucketName)
isVirtualStyle := c.isVirtualHostStyleRequest(targetURL, bucketName)
var urlStr string
// only support Aliyun OSS for virtual hosted path, compatible Amazon & Google Endpoint
if isVirtualHost && s3utils.IsAliyunOSSEndpoint(targetURL) {
if isVirtualStyle {
urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location"
} else {
targetURL.Path = path.Join(bucketName, "") + "/"

View File

@ -86,19 +86,30 @@ func (c Core) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarke
return c.listMultipartUploadsQuery(ctx, bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads)
}
// PutObjectPartOptions contains options for PutObjectPart API
type PutObjectPartOptions struct {
Md5Base64, Sha256Hex string
SSE encrypt.ServerSide
CustomHeader, Trailer http.Header
}
// PutObjectPart - Upload an object part.
func (c Core) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) {
func (c Core) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int,
data io.Reader, size int64, opts PutObjectPartOptions,
) (ObjectPart, error) {
p := uploadPartParams{
bucketName: bucket,
objectName: object,
uploadID: uploadID,
reader: data,
partNumber: partID,
md5Base64: md5Base64,
sha256Hex: sha256Hex,
md5Base64: opts.Md5Base64,
sha256Hex: opts.Sha256Hex,
size: size,
sse: sse,
sse: opts.SSE,
streamSha256: true,
customHeader: opts.CustomHeader,
trailer: opts.Trailer,
}
return c.uploadPart(ctx, p)
}
@ -109,11 +120,11 @@ func (c Core) ListObjectParts(ctx context.Context, bucket, object, uploadID stri
}
// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object.
func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart, opts PutObjectOptions) (string, error) {
func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart, opts PutObjectOptions) (UploadInfo, error) {
res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{
Parts: parts,
}, opts)
return res.ETag, err
return res, err
}
// AbortMultipartUpload - Abort an incomplete upload.

View File

@ -2053,7 +2053,7 @@ func testPutObjectWithChecksums() {
}
// Enable tracing, write to stderr.
//c.TraceOn(os.Stderr)
// c.TraceOn(os.Stderr)
// Set user agent.
c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0")
@ -8414,14 +8414,20 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() {
var completeParts []minio.CompletePart
part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption)
part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1,
bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024,
minio.PutObjectPartOptions{SSE: srcencryption},
)
if err != nil {
logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
return
}
completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag})
part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption)
part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2,
bytes.NewReader(buf[5*1024*1024:]), 1024*1024,
minio.PutObjectPartOptions{SSE: srcencryption},
)
if err != nil {
logError(testName, function, args, startTime, "", "PutObjectPart call failed", err)
return

View File

@ -28,27 +28,27 @@ import (
)
const (
// sseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS.
sseGenericHeader = "X-Amz-Server-Side-Encryption"
// SseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS.
SseGenericHeader = "X-Amz-Server-Side-Encryption"
// sseKmsKeyID is the AWS SSE-KMS key id.
sseKmsKeyID = sseGenericHeader + "-Aws-Kms-Key-Id"
// sseEncryptionContext is the AWS SSE-KMS Encryption Context data.
sseEncryptionContext = sseGenericHeader + "-Context"
// SseKmsKeyID is the AWS SSE-KMS key id.
SseKmsKeyID = SseGenericHeader + "-Aws-Kms-Key-Id"
// SseEncryptionContext is the AWS SSE-KMS Encryption Context data.
SseEncryptionContext = SseGenericHeader + "-Context"
// sseCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
sseCustomerAlgorithm = sseGenericHeader + "-Customer-Algorithm"
// sseCustomerKey is the AWS SSE-C encryption key HTTP header key.
sseCustomerKey = sseGenericHeader + "-Customer-Key"
// sseCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key.
sseCustomerKeyMD5 = sseGenericHeader + "-Customer-Key-MD5"
// SseCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
SseCustomerAlgorithm = SseGenericHeader + "-Customer-Algorithm"
// SseCustomerKey is the AWS SSE-C encryption key HTTP header key.
SseCustomerKey = SseGenericHeader + "-Customer-Key"
// SseCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key.
SseCustomerKeyMD5 = SseGenericHeader + "-Customer-Key-MD5"
// sseCopyCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key for CopyObject API.
sseCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
// sseCopyCustomerKey is the AWS SSE-C encryption key HTTP header key for CopyObject API.
sseCopyCustomerKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
// sseCopyCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key for CopyObject API.
sseCopyCustomerKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5"
// SseCopyCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key for CopyObject API.
SseCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
// SseCopyCustomerKey is the AWS SSE-C encryption key HTTP header key for CopyObject API.
SseCopyCustomerKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
// SseCopyCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key for CopyObject API.
SseCopyCustomerKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5"
)
// PBKDF creates a SSE-C key from the provided password and salt.
@ -157,9 +157,9 @@ func (s ssec) Type() Type { return SSEC }
func (s ssec) Marshal(h http.Header) {
keyMD5 := md5.Sum(s[:])
h.Set(sseCustomerAlgorithm, "AES256")
h.Set(sseCustomerKey, base64.StdEncoding.EncodeToString(s[:]))
h.Set(sseCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:]))
h.Set(SseCustomerAlgorithm, "AES256")
h.Set(SseCustomerKey, base64.StdEncoding.EncodeToString(s[:]))
h.Set(SseCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:]))
}
type ssecCopy [32]byte
@ -168,16 +168,16 @@ func (s ssecCopy) Type() Type { return SSEC }
func (s ssecCopy) Marshal(h http.Header) {
keyMD5 := md5.Sum(s[:])
h.Set(sseCopyCustomerAlgorithm, "AES256")
h.Set(sseCopyCustomerKey, base64.StdEncoding.EncodeToString(s[:]))
h.Set(sseCopyCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:]))
h.Set(SseCopyCustomerAlgorithm, "AES256")
h.Set(SseCopyCustomerKey, base64.StdEncoding.EncodeToString(s[:]))
h.Set(SseCopyCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:]))
}
type s3 struct{}
func (s s3) Type() Type { return S3 }
func (s s3) Marshal(h http.Header) { h.Set(sseGenericHeader, "AES256") }
func (s s3) Marshal(h http.Header) { h.Set(SseGenericHeader, "AES256") }
type kms struct {
key string
@ -188,11 +188,11 @@ type kms struct {
func (s kms) Type() Type { return KMS }
func (s kms) Marshal(h http.Header) {
h.Set(sseGenericHeader, "aws:kms")
h.Set(SseGenericHeader, "aws:kms")
if s.key != "" {
h.Set(sseKmsKeyID, s.key)
h.Set(SseKmsKeyID, s.key)
}
if s.hasContext {
h.Set(sseEncryptionContext, base64.StdEncoding.EncodeToString(s.context))
h.Set(SseEncryptionContext, base64.StdEncoding.EncodeToString(s.context))
}
}

View File

@ -339,12 +339,12 @@ func EncodePath(pathName string) string {
encodedPathname.WriteRune(s)
continue
default:
len := utf8.RuneLen(s)
if len < 0 {
l := utf8.RuneLen(s)
if l < 0 {
// if utf8 cannot convert return the same string as is
return pathName
}
u := make([]byte, len)
u := make([]byte, l)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})

View File

@ -34,6 +34,7 @@ var awsS3EndpointMap = map[string]string{
"eu-south-2": "s3.dualstack.eu-south-2.amazonaws.com",
"ap-east-1": "s3.dualstack.ap-east-1.amazonaws.com",
"ap-south-1": "s3.dualstack.ap-south-1.amazonaws.com",
"ap-south-2": "s3.dualstack.ap-south-2.amazonaws.com",
"ap-southeast-1": "s3.dualstack.ap-southeast-1.amazonaws.com",
"ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com",
"ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com",