[chore]: Bump github.com/minio/minio-go/v7 from 7.0.44 to 7.0.47 (#1348)
Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.44 to 7.0.47. - [Release notes](https://github.com/minio/minio-go/releases) - [Commits](https://github.com/minio/minio-go/compare/v7.0.44...v7.0.47) --- updated-dependencies: - dependency-name: github.com/minio/minio-go/v7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
dependabot[bot]
GitHub
parent
2a46980082
commit
fe3e9ede52
2
go.mod
2
go.mod
|
|
@ -32,7 +32,7 @@ require (
|
|||
github.com/jackc/pgx/v4 v4.17.2
|
||||
github.com/microcosm-cc/bluemonday v1.0.21
|
||||
github.com/miekg/dns v1.1.50
|
||||
github.com/minio/minio-go/v7 v7.0.44
|
||||
github.com/minio/minio-go/v7 v7.0.47
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/oklog/ulid v1.3.1
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -418,8 +418,8 @@ github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
|
|||
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.44 h1:9zUJ7iU7ax2P1jOvTp6nVrgzlZq3AZlFm0XfRFDKstM=
|
||||
github.com/minio/minio-go/v7 v7.0.44/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
|
||||
github.com/minio/minio-go/v7 v7.0.47 h1:sLiuCKGSIcn/MI6lREmTzX91DX/oRau4ia0j6e6eOSs=
|
||||
github.com/minio/minio-go/v7 v7.0.47/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
|
||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ lint:
|
|||
|
||||
vet:
|
||||
@GO111MODULE=on go vet ./...
|
||||
@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
|
||||
${GOPATH}/bin/staticcheck -tests=false -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-ST1023,-ST1005"
|
||||
|
||||
test:
|
||||
@GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./...
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
|
|
@ -143,5 +143,5 @@ func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]b
|
|||
}
|
||||
}
|
||||
|
||||
return ioutil.ReadAll(resp.Body)
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ package minio
|
|||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
|
@ -137,7 +137,7 @@ func (c *Client) getBucketPolicy(ctx context.Context, bucketName string) (string
|
|||
}
|
||||
}
|
||||
|
||||
bucketPolicyBuf, err := ioutil.ReadAll(resp.Body)
|
||||
bucketPolicyBuf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
|
@ -180,7 +180,7 @@ func (c *Client) GetBucketReplicationMetrics(ctx context.Context, bucketName str
|
|||
if resp.StatusCode != http.StatusOK {
|
||||
return s, httpRespToErrorResponse(resp, bucketName, "")
|
||||
}
|
||||
respBytes, err := ioutil.ReadAll(resp.Body)
|
||||
respBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import (
|
|||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
|
|
@ -58,7 +57,7 @@ func (c *Client) GetBucketTagging(ctx context.Context, bucketName string) (*tags
|
|||
return nil, httpRespToErrorResponse(resp, bucketName, "")
|
||||
}
|
||||
|
||||
defer io.Copy(ioutil.Discard, resp.Body)
|
||||
defer io.Copy(io.Discard, resp.Body)
|
||||
return tags.ParseBucketXML(resp.Body)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
|
@ -516,7 +515,7 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ..
|
|||
return UploadInfo{}, err
|
||||
}
|
||||
if dst.Progress != nil {
|
||||
io.CopyN(ioutil.Discard, dst.Progress, end-start+1)
|
||||
io.CopyN(io.Discard, dst.Progress, end-start+1)
|
||||
}
|
||||
objParts = append(objParts, complPart)
|
||||
partIndex++
|
||||
|
|
@ -525,7 +524,7 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ..
|
|||
|
||||
// 4. Make final complete-multipart request.
|
||||
uploadInfo, err := c.completeMultipartUpload(ctx, dst.Bucket, dst.Object, uploadID,
|
||||
completeMultipartUpload{Parts: objParts}, PutObjectOptions{})
|
||||
completeMultipartUpload{Parts: objParts}, PutObjectOptions{ServerSideEncryption: dst.Encryption})
|
||||
if err != nil {
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ package minio
|
|||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
|
|
@ -54,7 +53,7 @@ func (c *Client) CopyObject(ctx context.Context, dst CopyDestOptions, src CopySr
|
|||
|
||||
// Update the progress properly after successful copy.
|
||||
if dst.Progress != nil {
|
||||
io.Copy(ioutil.Discard, io.LimitReader(dst.Progress, dst.Size))
|
||||
io.Copy(io.Discard, io.LimitReader(dst.Progress, dst.Size))
|
||||
}
|
||||
|
||||
cpObjRes := copyObjectResult{}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import (
|
|||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
|
|
@ -108,7 +107,7 @@ const (
|
|||
func xmlDecodeAndBody(bodyReader io.Reader, v interface{}) ([]byte, error) {
|
||||
// read the whole body (up to 1MB)
|
||||
const maxBodyLength = 1 << 20
|
||||
body, err := ioutil.ReadAll(io.LimitReader(bodyReader, maxBodyLength))
|
||||
body, err := io.ReadAll(io.LimitReader(bodyReader, maxBodyLength))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -253,26 +252,6 @@ func errUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string)
|
|||
}
|
||||
}
|
||||
|
||||
// errInvalidBucketName - Invalid bucket name response.
|
||||
func errInvalidBucketName(message string) error {
|
||||
return ErrorResponse{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Code: "InvalidBucketName",
|
||||
Message: message,
|
||||
RequestID: "minio",
|
||||
}
|
||||
}
|
||||
|
||||
// errInvalidObjectName - Invalid object name response.
|
||||
func errInvalidObjectName(message string) error {
|
||||
return ErrorResponse{
|
||||
StatusCode: http.StatusNotFound,
|
||||
Code: "NoSuchKey",
|
||||
Message: message,
|
||||
RequestID: "minio",
|
||||
}
|
||||
}
|
||||
|
||||
// errInvalidArgument - Invalid argument response.
|
||||
func errInvalidArgument(message string) error {
|
||||
return ErrorResponse{
|
||||
|
|
|
|||
|
|
@ -897,6 +897,8 @@ func (c *Client) listMultipartUploadsQuery(ctx context.Context, bucketName, keyM
|
|||
}
|
||||
|
||||
// listObjectParts list all object parts recursively.
|
||||
//
|
||||
//lint:ignore U1000 Keep this around
|
||||
func (c *Client) listObjectParts(ctx context.Context, bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) {
|
||||
// Part number marker for the next batch of request.
|
||||
var nextPartNumberMarker int
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import (
|
|||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
|
|
@ -201,7 +200,9 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
|
|||
|
||||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(complMultipartUpload.Parts))
|
||||
opts = PutObjectOptions{}
|
||||
opts = PutObjectOptions{
|
||||
ServerSideEncryption: opts.ServerSideEncryption,
|
||||
}
|
||||
if len(crcBytes) > 0 {
|
||||
// Add hash of hashes.
|
||||
crc.Reset()
|
||||
|
|
@ -412,7 +413,7 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
|
|||
|
||||
// Read resp.Body into a []bytes to parse for Error response inside the body
|
||||
var b []byte
|
||||
b, err = ioutil.ReadAll(resp.Body)
|
||||
b, err = io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import (
|
|||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
|
|
@ -44,7 +45,9 @@ import (
|
|||
func (c *Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string,
|
||||
reader io.Reader, size int64, opts PutObjectOptions,
|
||||
) (info UploadInfo, err error) {
|
||||
if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 {
|
||||
if opts.ConcurrentStreamParts && opts.NumThreads > 1 {
|
||||
info, err = c.putObjectMultipartStreamParallel(ctx, bucketName, objectName, reader, opts)
|
||||
} else if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 {
|
||||
// Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader.
|
||||
info, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts)
|
||||
} else {
|
||||
|
|
@ -266,6 +269,9 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
|
|||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(complMultipartUpload.Parts))
|
||||
|
||||
opts = PutObjectOptions{
|
||||
ServerSideEncryption: opts.ServerSideEncryption,
|
||||
}
|
||||
if withChecksum {
|
||||
// Add hash of hashes.
|
||||
crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
|
||||
|
|
@ -278,7 +284,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
|
|||
opts.UserMetadata = map[string]string{"X-Amz-Checksum-Crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))}
|
||||
}
|
||||
|
||||
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{})
|
||||
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
|
||||
if err != nil {
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
|
|
@ -425,6 +431,211 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
|
|||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(complMultipartUpload.Parts))
|
||||
|
||||
opts = PutObjectOptions{
|
||||
ServerSideEncryption: opts.ServerSideEncryption,
|
||||
}
|
||||
if len(crcBytes) > 0 {
|
||||
// Add hash of hashes.
|
||||
crc.Reset()
|
||||
crc.Write(crcBytes)
|
||||
opts.UserMetadata = map[string]string{"X-Amz-Checksum-Crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))}
|
||||
}
|
||||
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts)
|
||||
if err != nil {
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
|
||||
uploadInfo.Size = totalUploadedSize
|
||||
return uploadInfo, nil
|
||||
}
|
||||
|
||||
// putObjectMultipartStreamParallel uploads opts.NumThreads parts in parallel.
|
||||
// This is expected to take opts.PartSize * opts.NumThreads * (GOGC / 100) bytes of buffer.
|
||||
func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketName, objectName string,
|
||||
reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) {
|
||||
// Input validation.
|
||||
if err = s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
|
||||
if err = s3utils.CheckValidObjectName(objectName); err != nil {
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
|
||||
if !opts.SendContentMd5 {
|
||||
if opts.UserMetadata == nil {
|
||||
opts.UserMetadata = make(map[string]string, 1)
|
||||
}
|
||||
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = "CRC32C"
|
||||
}
|
||||
|
||||
// Cancel all when an error occurs.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
// Calculate the optimal parts info for a given size.
|
||||
totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize)
|
||||
if err != nil {
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
|
||||
// Initiates a new multipart request
|
||||
uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
|
||||
if err != nil {
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
|
||||
|
||||
// Aborts the multipart upload if the function returns
|
||||
// any error, since we do not resume we should purge
|
||||
// the parts which have been uploaded to relinquish
|
||||
// storage space.
|
||||
defer func() {
|
||||
if err != nil {
|
||||
c.abortMultipartUpload(ctx, bucketName, objectName, uploadID)
|
||||
}
|
||||
}()
|
||||
|
||||
// Create checksums
|
||||
// CRC32C is ~50% faster on AMD64 @ 30GB/s
|
||||
var crcBytes []byte
|
||||
crc := crc32.New(crc32.MakeTable(crc32.Castagnoli))
|
||||
md5Hash := c.md5Hasher()
|
||||
defer md5Hash.Close()
|
||||
|
||||
// Total data read and written to server. should be equal to 'size' at the end of the call.
|
||||
var totalUploadedSize int64
|
||||
|
||||
// Initialize parts uploaded map.
|
||||
partsInfo := make(map[int]ObjectPart)
|
||||
|
||||
// Create a buffer.
|
||||
nBuffers := int64(opts.NumThreads)
|
||||
bufs := make(chan []byte, nBuffers)
|
||||
all := make([]byte, nBuffers*partSize)
|
||||
for i := int64(0); i < nBuffers; i++ {
|
||||
bufs <- all[i*partSize : i*partSize+partSize]
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
errCh := make(chan error, opts.NumThreads)
|
||||
|
||||
reader = newHook(reader, opts.Progress)
|
||||
|
||||
// Part number always starts with '1'.
|
||||
var partNumber int
|
||||
for partNumber = 1; partNumber <= totalPartsCount; partNumber++ {
|
||||
// Proceed to upload the part.
|
||||
var buf []byte
|
||||
select {
|
||||
case buf = <-bufs:
|
||||
case err = <-errCh:
|
||||
cancel()
|
||||
wg.Wait()
|
||||
return UploadInfo{}, err
|
||||
}
|
||||
|
||||
if int64(len(buf)) != partSize {
|
||||
return UploadInfo{}, fmt.Errorf("read buffer < %d than expected partSize: %d", len(buf), partSize)
|
||||
}
|
||||
|
||||
length, rerr := readFull(reader, buf)
|
||||
if rerr == io.EOF && partNumber > 1 {
|
||||
// Done
|
||||
break
|
||||
}
|
||||
|
||||
if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF {
|
||||
cancel()
|
||||
wg.Wait()
|
||||
return UploadInfo{}, rerr
|
||||
}
|
||||
|
||||
// Calculate md5sum.
|
||||
customHeader := make(http.Header)
|
||||
if !opts.SendContentMd5 {
|
||||
// Add CRC32C instead.
|
||||
crc.Reset()
|
||||
crc.Write(buf[:length])
|
||||
cSum := crc.Sum(nil)
|
||||
customHeader.Set("x-amz-checksum-crc32c", base64.StdEncoding.EncodeToString(cSum))
|
||||
crcBytes = append(crcBytes, cSum...)
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(partNumber int) {
|
||||
// Avoid declaring variables in the for loop
|
||||
var md5Base64 string
|
||||
|
||||
if opts.SendContentMd5 {
|
||||
md5Hash.Reset()
|
||||
md5Hash.Write(buf[:length])
|
||||
md5Base64 = base64.StdEncoding.EncodeToString(md5Hash.Sum(nil))
|
||||
}
|
||||
|
||||
defer wg.Done()
|
||||
p := uploadPartParams{
|
||||
bucketName: bucketName,
|
||||
objectName: objectName,
|
||||
uploadID: uploadID,
|
||||
reader: bytes.NewReader(buf[:length]),
|
||||
partNumber: partNumber,
|
||||
md5Base64: md5Base64,
|
||||
size: int64(length),
|
||||
sse: opts.ServerSideEncryption,
|
||||
streamSha256: !opts.DisableContentSha256,
|
||||
customHeader: customHeader,
|
||||
}
|
||||
objPart, uerr := c.uploadPart(ctx, p)
|
||||
if uerr != nil {
|
||||
errCh <- uerr
|
||||
}
|
||||
|
||||
// Save successfully uploaded part metadata.
|
||||
mu.Lock()
|
||||
partsInfo[partNumber] = objPart
|
||||
mu.Unlock()
|
||||
|
||||
// Send buffer back so it can be reused.
|
||||
bufs <- buf
|
||||
}(partNumber)
|
||||
|
||||
// Save successfully uploaded size.
|
||||
totalUploadedSize += int64(length)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Collect any error
|
||||
select {
|
||||
case err = <-errCh:
|
||||
return UploadInfo{}, err
|
||||
default:
|
||||
}
|
||||
|
||||
// Complete multipart upload.
|
||||
var complMultipartUpload completeMultipartUpload
|
||||
|
||||
// Loop over total uploaded parts to save them in
|
||||
// Parts array before completing the multipart request.
|
||||
for i := 1; i < partNumber; i++ {
|
||||
part, ok := partsInfo[i]
|
||||
if !ok {
|
||||
return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i))
|
||||
}
|
||||
complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{
|
||||
ETag: part.ETag,
|
||||
PartNumber: part.PartNumber,
|
||||
ChecksumCRC32: part.ChecksumCRC32,
|
||||
ChecksumCRC32C: part.ChecksumCRC32C,
|
||||
ChecksumSHA1: part.ChecksumSHA1,
|
||||
ChecksumSHA256: part.ChecksumSHA256,
|
||||
})
|
||||
}
|
||||
|
||||
// Sort all completed parts.
|
||||
sort.Sort(completedParts(complMultipartUpload.Parts))
|
||||
|
||||
opts = PutObjectOptions{}
|
||||
if len(crcBytes) > 0 {
|
||||
// Add hash of hashes.
|
||||
|
|
|
|||
|
|
@ -87,7 +87,12 @@ type PutObjectOptions struct {
|
|||
SendContentMd5 bool
|
||||
DisableContentSha256 bool
|
||||
DisableMultipart bool
|
||||
Internal AdvancedPutOptions
|
||||
|
||||
// ConcurrentStreamParts will create NumThreads buffers of PartSize bytes,
|
||||
// fill them serially and upload them in parallel.
|
||||
// This can be used for faster uploads on non-seekable or slow-to-seek input.
|
||||
ConcurrentStreamParts bool
|
||||
Internal AdvancedPutOptions
|
||||
}
|
||||
|
||||
// getNumThreads - gets the number of threads to be used in the multipart
|
||||
|
|
@ -272,6 +277,9 @@ func (c *Client) putObjectCommon(ctx context.Context, bucketName, objectName str
|
|||
if opts.DisableMultipart {
|
||||
return UploadInfo{}, errors.New("no length provided and multipart disabled")
|
||||
}
|
||||
if opts.ConcurrentStreamParts && opts.NumThreads > 1 {
|
||||
return c.putObjectMultipartStreamParallel(ctx, bucketName, objectName, reader, opts)
|
||||
}
|
||||
return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -107,7 +106,7 @@ func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts
|
|||
return nopReadSeekCloser{bytes.NewReader(b.Bytes())}, int64(b.Len()), nil
|
||||
}
|
||||
} else {
|
||||
f, err := ioutil.TempFile("", "s3-putsnowballobjects-*")
|
||||
f, err := os.CreateTemp("", "s3-putsnowballobjects-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -316,8 +316,6 @@ type completeMultipartUploadResult struct {
|
|||
// CompletePart sub container lists individual part numbers and their
|
||||
// md5sum, part of completeMultipartUpload.
|
||||
type CompletePart struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"`
|
||||
|
||||
// Part number identifies the part.
|
||||
PartNumber int
|
||||
ETag string
|
||||
|
|
|
|||
|
|
@ -41,8 +41,8 @@ type CSVFileHeaderInfo string
|
|||
// Constants for file header info.
|
||||
const (
|
||||
CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE"
|
||||
CSVFileHeaderInfoIgnore = "IGNORE"
|
||||
CSVFileHeaderInfoUse = "USE"
|
||||
CSVFileHeaderInfoIgnore CSVFileHeaderInfo = "IGNORE"
|
||||
CSVFileHeaderInfoUse CSVFileHeaderInfo = "USE"
|
||||
)
|
||||
|
||||
// SelectCompressionType - is the parameter for what type of compression is
|
||||
|
|
@ -52,15 +52,15 @@ type SelectCompressionType string
|
|||
// Constants for compression types under select API.
|
||||
const (
|
||||
SelectCompressionNONE SelectCompressionType = "NONE"
|
||||
SelectCompressionGZIP = "GZIP"
|
||||
SelectCompressionBZIP = "BZIP2"
|
||||
SelectCompressionGZIP SelectCompressionType = "GZIP"
|
||||
SelectCompressionBZIP SelectCompressionType = "BZIP2"
|
||||
|
||||
// Non-standard compression schemes, supported by MinIO hosts:
|
||||
|
||||
SelectCompressionZSTD = "ZSTD" // Zstandard compression.
|
||||
SelectCompressionLZ4 = "LZ4" // LZ4 Stream
|
||||
SelectCompressionS2 = "S2" // S2 Stream
|
||||
SelectCompressionSNAPPY = "SNAPPY" // Snappy stream
|
||||
SelectCompressionZSTD SelectCompressionType = "ZSTD" // Zstandard compression.
|
||||
SelectCompressionLZ4 SelectCompressionType = "LZ4" // LZ4 Stream
|
||||
SelectCompressionS2 SelectCompressionType = "S2" // S2 Stream
|
||||
SelectCompressionSNAPPY SelectCompressionType = "SNAPPY" // Snappy stream
|
||||
)
|
||||
|
||||
// CSVQuoteFields - is the parameter for how CSV fields are quoted.
|
||||
|
|
@ -69,7 +69,7 @@ type CSVQuoteFields string
|
|||
// Constants for csv quote styles.
|
||||
const (
|
||||
CSVQuoteFieldsAlways CSVQuoteFields = "Always"
|
||||
CSVQuoteFieldsAsNeeded = "AsNeeded"
|
||||
CSVQuoteFieldsAsNeeded CSVQuoteFields = "AsNeeded"
|
||||
)
|
||||
|
||||
// QueryExpressionType - is of what syntax the expression is, this should only
|
||||
|
|
@ -87,7 +87,7 @@ type JSONType string
|
|||
// Constants for JSONTypes.
|
||||
const (
|
||||
JSONDocumentType JSONType = "DOCUMENT"
|
||||
JSONLinesType = "LINES"
|
||||
JSONLinesType JSONType = "LINES"
|
||||
)
|
||||
|
||||
// ParquetInputOptions parquet input specific options
|
||||
|
|
@ -378,8 +378,8 @@ type SelectObjectType string
|
|||
// Constants for input data types.
|
||||
const (
|
||||
SelectObjectTypeCSV SelectObjectType = "CSV"
|
||||
SelectObjectTypeJSON = "JSON"
|
||||
SelectObjectTypeParquet = "Parquet"
|
||||
SelectObjectTypeJSON SelectObjectType = "JSON"
|
||||
SelectObjectTypeParquet SelectObjectType = "Parquet"
|
||||
)
|
||||
|
||||
// preludeInfo is used for keeping track of necessary information from the
|
||||
|
|
@ -416,7 +416,7 @@ type messageType string
|
|||
|
||||
const (
|
||||
errorMsg messageType = "error"
|
||||
commonMsg = "event"
|
||||
commonMsg messageType = "event"
|
||||
)
|
||||
|
||||
// eventType represents the type of event.
|
||||
|
|
@ -425,9 +425,9 @@ type eventType string
|
|||
// list of event-types returned by Select API.
|
||||
const (
|
||||
endEvent eventType = "End"
|
||||
recordsEvent = "Records"
|
||||
progressEvent = "Progress"
|
||||
statsEvent = "Stats"
|
||||
recordsEvent eventType = "Records"
|
||||
progressEvent eventType = "Progress"
|
||||
statsEvent eventType = "Stats"
|
||||
)
|
||||
|
||||
// contentType represents content type of event.
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ import (
|
|||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
|
|
@ -119,7 +118,7 @@ type Options struct {
|
|||
// Global constants.
|
||||
const (
|
||||
libraryName = "minio-go"
|
||||
libraryVersion = "v7.0.44"
|
||||
libraryVersion = "v7.0.47"
|
||||
)
|
||||
|
||||
// User Agent should always following the below style.
|
||||
|
|
@ -635,7 +634,7 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ
|
|||
}
|
||||
|
||||
// Read the body to be saved later.
|
||||
errBodyBytes, err := ioutil.ReadAll(res.Body)
|
||||
errBodyBytes, err := io.ReadAll(res.Body)
|
||||
// res.Body should be closed
|
||||
closeResponse(res)
|
||||
if err != nil {
|
||||
|
|
@ -644,14 +643,14 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ
|
|||
|
||||
// Save the body.
|
||||
errBodySeeker := bytes.NewReader(errBodyBytes)
|
||||
res.Body = ioutil.NopCloser(errBodySeeker)
|
||||
res.Body = io.NopCloser(errBodySeeker)
|
||||
|
||||
// For errors verify if its retryable otherwise fail quickly.
|
||||
errResponse := ToErrorResponse(httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName))
|
||||
|
||||
// Save the body back again.
|
||||
errBodySeeker.Seek(0, 0) // Seek back to starting point.
|
||||
res.Body = ioutil.NopCloser(errBodySeeker)
|
||||
res.Body = io.NopCloser(errBodySeeker)
|
||||
|
||||
// Bucket region if set in error response and the error
|
||||
// code dictates invalid region, we can retry the request
|
||||
|
|
@ -814,7 +813,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
|
|||
if metadata.contentLength == 0 {
|
||||
req.Body = nil
|
||||
} else {
|
||||
req.Body = ioutil.NopCloser(metadata.contentBody)
|
||||
req.Body = io.NopCloser(metadata.contentBody)
|
||||
}
|
||||
|
||||
// Set incoming content-length.
|
||||
|
|
@ -846,7 +845,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
|
|||
// Additionally, we also look if the initialized client is secure,
|
||||
// if yes then we don't need to perform streaming signature.
|
||||
req = signer.StreamingSignV4(req, accessKeyID,
|
||||
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC())
|
||||
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
|
||||
default:
|
||||
// Set sha256 sum for signature calculation only with signature version '4'.
|
||||
shaHeader := unsignedPayload
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ import (
|
|||
"hash"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
|
|
@ -346,7 +345,7 @@ func getDataReader(fileName string) io.ReadCloser {
|
|||
if _, ok := dataFileCRC32[fileName]; !ok {
|
||||
dataFileCRC32[fileName] = mustCrcReader(newRandomReader(size, size))
|
||||
}
|
||||
return ioutil.NopCloser(newRandomReader(size, size))
|
||||
return io.NopCloser(newRandomReader(size, size))
|
||||
}
|
||||
reader, _ := os.Open(getMintDataDirFilePath(fileName))
|
||||
if _, ok := dataFileCRC32[fileName]; !ok {
|
||||
|
|
@ -989,7 +988,7 @@ func testGetObjectWithVersioning() {
|
|||
|
||||
for _, testFile := range testFiles {
|
||||
r := getDataReader(testFile)
|
||||
buf, err := ioutil.ReadAll(r)
|
||||
buf, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
||||
return
|
||||
|
|
@ -1131,7 +1130,7 @@ func testPutObjectWithVersioning() {
|
|||
var errs [n]error
|
||||
for i := 0; i < n; i++ {
|
||||
r := newRandomReader(int64((1<<20)*i+i), int64(i))
|
||||
buf, err := ioutil.ReadAll(r)
|
||||
buf, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
||||
return
|
||||
|
|
@ -1271,7 +1270,7 @@ func testCopyObjectWithVersioning() {
|
|||
testFiles := []string{"datafile-1-b", "datafile-10-kB"}
|
||||
for _, testFile := range testFiles {
|
||||
r := getDataReader(testFile)
|
||||
buf, err := ioutil.ReadAll(r)
|
||||
buf, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
||||
return
|
||||
|
|
@ -1304,7 +1303,7 @@ func testCopyObjectWithVersioning() {
|
|||
return
|
||||
}
|
||||
|
||||
oldestContent, err := ioutil.ReadAll(reader)
|
||||
oldestContent, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err)
|
||||
return
|
||||
|
|
@ -1338,7 +1337,7 @@ func testCopyObjectWithVersioning() {
|
|||
}
|
||||
defer readerCopy.Close()
|
||||
|
||||
newestContent, err := ioutil.ReadAll(readerCopy)
|
||||
newestContent, err := io.ReadAll(readerCopy)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err)
|
||||
return
|
||||
|
|
@ -1408,7 +1407,7 @@ func testConcurrentCopyObjectWithVersioning() {
|
|||
testFiles := []string{"datafile-10-kB"}
|
||||
for _, testFile := range testFiles {
|
||||
r := getDataReader(testFile)
|
||||
buf, err := ioutil.ReadAll(r)
|
||||
buf, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
||||
return
|
||||
|
|
@ -1441,7 +1440,7 @@ func testConcurrentCopyObjectWithVersioning() {
|
|||
return
|
||||
}
|
||||
|
||||
oldestContent, err := ioutil.ReadAll(reader)
|
||||
oldestContent, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err)
|
||||
return
|
||||
|
|
@ -1491,7 +1490,7 @@ func testConcurrentCopyObjectWithVersioning() {
|
|||
}
|
||||
defer readerCopy.Close()
|
||||
|
||||
newestContent, err := ioutil.ReadAll(readerCopy)
|
||||
newestContent, err := io.ReadAll(readerCopy)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err)
|
||||
return
|
||||
|
|
@ -1571,7 +1570,7 @@ func testComposeObjectWithVersioning() {
|
|||
|
||||
for _, testFile := range testFiles {
|
||||
r := getDataReader(testFile)
|
||||
buf, err := ioutil.ReadAll(r)
|
||||
buf, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "unexpected failure", err)
|
||||
return
|
||||
|
|
@ -1633,7 +1632,7 @@ func testComposeObjectWithVersioning() {
|
|||
}
|
||||
defer readerCopy.Close()
|
||||
|
||||
copyContentBytes, err := ioutil.ReadAll(readerCopy)
|
||||
copyContentBytes, err := io.ReadAll(readerCopy)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err)
|
||||
return
|
||||
|
|
@ -1733,12 +1732,39 @@ func testRemoveObjectWithVersioning() {
|
|||
logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err)
|
||||
return
|
||||
}
|
||||
|
||||
err = c.RemoveBucket(context.Background(), bucketName)
|
||||
// test delete marker version id is non-null
|
||||
_, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "CleanupBucket failed", err)
|
||||
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
||||
return
|
||||
}
|
||||
// create delete marker
|
||||
err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "DeleteObject failed", err)
|
||||
return
|
||||
}
|
||||
objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true})
|
||||
idx := 0
|
||||
for info := range objectsInfo {
|
||||
if info.Err != nil {
|
||||
logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err)
|
||||
return
|
||||
}
|
||||
if idx == 0 {
|
||||
if !info.IsDeleteMarker {
|
||||
logError(testName, function, args, startTime, "", "Unexpected error - expected delete marker to have been created", err)
|
||||
return
|
||||
}
|
||||
if info.VersionID == "" {
|
||||
logError(testName, function, args, startTime, "", "Unexpected error - expected delete marker to be versioned", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
idx++
|
||||
}
|
||||
|
||||
defer cleanupBucket(bucketName, c)
|
||||
|
||||
successLogger(testName, function, args, startTime).Info()
|
||||
}
|
||||
|
|
@ -2461,7 +2487,7 @@ func testGetObjectSeekEnd() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -2982,7 +3008,7 @@ func testFPutObjectMultipart() {
|
|||
fileName := getMintDataDirFilePath("datafile-129-MB")
|
||||
if fileName == "" {
|
||||
// Make a temp file with minPartSize bytes of data.
|
||||
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
|
||||
file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest")
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
||||
return
|
||||
|
|
@ -3091,7 +3117,7 @@ func testFPutObject() {
|
|||
fName := getMintDataDirFilePath("datafile-129-MB")
|
||||
if fName == "" {
|
||||
// Make a temp file with minPartSize bytes of data.
|
||||
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
|
||||
file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest")
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
||||
return
|
||||
|
|
@ -3257,7 +3283,7 @@ func testFPutObjectContext() {
|
|||
fName := getMintDataDirFilePath("datafile-1-MB")
|
||||
if fName == "" {
|
||||
// Make a temp file with 1 MiB bytes of data.
|
||||
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest")
|
||||
file, err := os.CreateTemp(os.TempDir(), "FPutObjectContextTest")
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
||||
return
|
||||
|
|
@ -3357,7 +3383,7 @@ func testFPutObjectContextV2() {
|
|||
fName := getMintDataDirFilePath("datafile-1-MB")
|
||||
if fName == "" {
|
||||
// Make a temp file with 1 MiB bytes of data.
|
||||
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectContextTest")
|
||||
file, err := os.CreateTemp(os.TempDir(), "FPutObjectContextTest")
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "Temp file creation failed", err)
|
||||
return
|
||||
|
|
@ -3621,7 +3647,7 @@ func testGetObjectS3Zip() {
|
|||
logError(testName, function, args, startTime, "", "file.Open failed", err)
|
||||
return
|
||||
}
|
||||
want, err := ioutil.ReadAll(zfr)
|
||||
want, err := io.ReadAll(zfr)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "fzip file read failed", err)
|
||||
return
|
||||
|
|
@ -3638,7 +3664,7 @@ func testGetObjectS3Zip() {
|
|||
}
|
||||
return
|
||||
}
|
||||
got, err := ioutil.ReadAll(r)
|
||||
got, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -3722,7 +3748,7 @@ func testGetObjectReadSeekFunctional() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -3885,7 +3911,7 @@ func testGetObjectReadAtFunctional() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -4062,7 +4088,7 @@ func testGetObjectReadAtWhenEOFWasReached() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -4181,7 +4207,7 @@ func testPresignedPostPolicy() {
|
|||
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user")
|
||||
metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -4245,7 +4271,7 @@ func testPresignedPostPolicy() {
|
|||
filePath := getMintDataDirFilePath("datafile-33-kB")
|
||||
if filePath == "" {
|
||||
// Make a temp file with 33 KB data.
|
||||
file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest")
|
||||
file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest")
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
||||
return
|
||||
|
|
@ -4588,7 +4614,7 @@ func testSSECEncryptedGetObjectReadSeekFunctional() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -4770,7 +4796,7 @@ func testSSES3EncryptedGetObjectReadSeekFunctional() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -4944,7 +4970,7 @@ func testSSECEncryptedGetObjectReadAtFunctional() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -5127,7 +5153,7 @@ func testSSES3EncryptedGetObjectReadAtFunctional() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -6138,7 +6164,7 @@ func testFunctional() {
|
|||
return
|
||||
}
|
||||
|
||||
newReadBytes, err := ioutil.ReadAll(newReader)
|
||||
newReadBytes, err := io.ReadAll(newReader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -6269,7 +6295,7 @@ func testFunctional() {
|
|||
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err)
|
||||
return
|
||||
}
|
||||
newPresignedBytes, err := ioutil.ReadAll(resp.Body)
|
||||
newPresignedBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
||||
return
|
||||
|
|
@ -6312,7 +6338,7 @@ func testFunctional() {
|
|||
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err)
|
||||
return
|
||||
}
|
||||
newPresignedBytes, err = ioutil.ReadAll(resp.Body)
|
||||
newPresignedBytes, err = io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err)
|
||||
return
|
||||
|
|
@ -6372,7 +6398,7 @@ func testFunctional() {
|
|||
return
|
||||
}
|
||||
|
||||
newReadBytes, err = ioutil.ReadAll(newReader)
|
||||
newReadBytes, err = io.ReadAll(newReader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err)
|
||||
return
|
||||
|
|
@ -6428,7 +6454,7 @@ func testFunctional() {
|
|||
return
|
||||
}
|
||||
|
||||
newReadBytes, err = ioutil.ReadAll(newReader)
|
||||
newReadBytes, err = io.ReadAll(newReader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed during get on custom-presigned put object", err)
|
||||
return
|
||||
|
|
@ -6652,7 +6678,7 @@ func testPutObjectUploadSeekedObject() {
|
|||
}
|
||||
args["fileToUpload"] = fileName
|
||||
} else {
|
||||
tempfile, err = ioutil.TempFile("", "minio-go-upload-test-")
|
||||
tempfile, err = os.CreateTemp("", "minio-go-upload-test-")
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "TempFile create failed", err)
|
||||
return
|
||||
|
|
@ -6916,7 +6942,7 @@ func testFPutObjectV2() {
|
|||
defer cleanupBucket(bucketName, c)
|
||||
|
||||
// Make a temp file with 11*1024*1024 bytes of data.
|
||||
file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest")
|
||||
file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest")
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
|
||||
return
|
||||
|
|
@ -7145,7 +7171,7 @@ func testGetObjectReadSeekFunctionalV2() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -7299,7 +7325,7 @@ func testGetObjectReadAtFunctionalV2() {
|
|||
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
|
||||
args["objectName"] = objectName
|
||||
|
||||
buf, err := ioutil.ReadAll(reader)
|
||||
buf, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -7837,7 +7863,7 @@ func testEncryptedEmptyObject() {
|
|||
}
|
||||
defer reader.Close()
|
||||
|
||||
decBytes, err := ioutil.ReadAll(reader)
|
||||
decBytes, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -7915,7 +7941,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
|
|||
return
|
||||
}
|
||||
|
||||
decBytes, err := ioutil.ReadAll(reader)
|
||||
decBytes, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -7955,7 +7981,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
|
|||
return
|
||||
}
|
||||
|
||||
decBytes, err = ioutil.ReadAll(reader)
|
||||
decBytes, err = io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -7994,7 +8020,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
|
|||
}
|
||||
defer reader.Close()
|
||||
|
||||
decBytes, err = ioutil.ReadAll(reader)
|
||||
decBytes, err = io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -11040,7 +11066,7 @@ func testFunctionalV2() {
|
|||
return
|
||||
}
|
||||
|
||||
newReadBytes, err := ioutil.ReadAll(newReader)
|
||||
newReadBytes, err := io.ReadAll(newReader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -11146,7 +11172,7 @@ func testFunctionalV2() {
|
|||
logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err)
|
||||
return
|
||||
}
|
||||
newPresignedBytes, err := ioutil.ReadAll(resp.Body)
|
||||
newPresignedBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -11185,7 +11211,7 @@ func testFunctionalV2() {
|
|||
logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err)
|
||||
return
|
||||
}
|
||||
newPresignedBytes, err = ioutil.ReadAll(resp.Body)
|
||||
newPresignedBytes, err = io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed", err)
|
||||
return
|
||||
|
|
@ -11239,7 +11265,7 @@ func testFunctionalV2() {
|
|||
return
|
||||
}
|
||||
|
||||
newReadBytes, err = ioutil.ReadAll(newReader)
|
||||
newReadBytes, err = io.ReadAll(newReader)
|
||||
if err != nil {
|
||||
logError(testName, function, args, startTime, "", "ReadAll failed during get on presigned put object", err)
|
||||
return
|
||||
|
|
@ -11553,7 +11579,7 @@ func testGetObjectRanges() {
|
|||
}
|
||||
for _, test := range tests {
|
||||
wantRC := getDataReader("datafile-129-MB")
|
||||
io.CopyN(ioutil.Discard, wantRC, test.start)
|
||||
io.CopyN(io.Discard, wantRC, test.start)
|
||||
want := mustCrcReader(io.LimitReader(wantRC, test.end-test.start+1))
|
||||
opts := minio.GetObjectOptions{}
|
||||
opts.SetRange(test.start, test.end)
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ import (
|
|||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
|
@ -139,7 +138,7 @@ func closeResponse(resp *http.Response) {
|
|||
// Without this closing connection would disallow re-using
|
||||
// the same connection for future uses.
|
||||
// - http://stackoverflow.com/a/17961593/4465767
|
||||
io.Copy(ioutil.Discard, resp.Body)
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
}
|
||||
}
|
||||
|
|
@ -191,7 +190,7 @@ func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssume
|
|||
defer closeResponse(resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var errResp ErrorResponse
|
||||
buf, err := ioutil.ReadAll(resp.Body)
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return AssumeRoleResponse{}, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import (
|
|||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
// ErrorResponse - Is the typed error returned.
|
||||
|
|
@ -88,7 +87,7 @@ func xmlDecoder(body io.Reader, v interface{}) error {
|
|||
func xmlDecodeAndBody(bodyReader io.Reader, v interface{}) ([]byte, error) {
|
||||
// read the whole body (up to 1MB)
|
||||
const maxBodyLength = 1 << 20
|
||||
body, err := ioutil.ReadAll(io.LimitReader(bodyReader, maxBodyLength))
|
||||
body, err := io.ReadAll(io.LimitReader(bodyReader, maxBodyLength))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
package credentials
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
|
|
@ -114,6 +113,7 @@ type hostConfig struct {
|
|||
type config struct {
|
||||
Version string `json:"version"`
|
||||
Hosts map[string]hostConfig `json:"hosts"`
|
||||
Aliases map[string]hostConfig `json:"aliases"`
|
||||
}
|
||||
|
||||
// loadAliass loads from the file pointed to by shared credentials filename for alias.
|
||||
|
|
@ -123,12 +123,17 @@ func loadAlias(filename, alias string) (hostConfig, error) {
|
|||
cfg := &config{}
|
||||
json := jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
|
||||
configBytes, err := ioutil.ReadFile(filename)
|
||||
configBytes, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return hostConfig{}, err
|
||||
}
|
||||
if err = json.Unmarshal(configBytes, cfg); err != nil {
|
||||
return hostConfig{}, err
|
||||
}
|
||||
|
||||
if cfg.Version == "10" {
|
||||
return cfg.Aliases[alias], nil
|
||||
}
|
||||
|
||||
return cfg.Hosts[alias], nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -106,7 +106,7 @@ func (m *IAM) Retrieve() (Value, error) {
|
|||
Client: m.Client,
|
||||
STSEndpoint: endpoint,
|
||||
GetWebIDTokenExpiry: func() (*WebIdentityToken, error) {
|
||||
token, err := ioutil.ReadFile(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE"))
|
||||
token, err := os.ReadFile(os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -268,7 +268,7 @@ func fetchIMDSToken(client *http.Client, endpoint string) (string, error) {
|
|||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import (
|
|||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
|
@ -138,7 +138,7 @@ func getClientGrantsCredentials(clnt *http.Client, endpoint string,
|
|||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var errResp ErrorResponse
|
||||
buf, err := ioutil.ReadAll(resp.Body)
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return AssumeRoleWithClientGrantsResponse{}, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
|
@ -156,7 +156,7 @@ func (k *LDAPIdentity) Retrieve() (value Value, err error) {
|
|||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var errResp ErrorResponse
|
||||
buf, err := ioutil.ReadAll(resp.Body)
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return value, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -152,7 +151,7 @@ func (i *STSCertificateIdentity) Retrieve() (Value, error) {
|
|||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var errResp ErrorResponse
|
||||
buf, err := ioutil.ReadAll(resp.Body)
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return Value{}, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import (
|
|||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
|
@ -155,7 +155,7 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession
|
|||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var errResp ErrorResponse
|
||||
buf, err := ioutil.ReadAll(resp.Body)
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return AssumeRoleWithWebIdentityResponse{}, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,19 +34,19 @@ type EventType string
|
|||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-event-types-and-destinations
|
||||
const (
|
||||
ObjectCreatedAll EventType = "s3:ObjectCreated:*"
|
||||
ObjectCreatedPut = "s3:ObjectCreated:Put"
|
||||
ObjectCreatedPost = "s3:ObjectCreated:Post"
|
||||
ObjectCreatedCopy = "s3:ObjectCreated:Copy"
|
||||
ObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload"
|
||||
ObjectAccessedGet = "s3:ObjectAccessed:Get"
|
||||
ObjectAccessedHead = "s3:ObjectAccessed:Head"
|
||||
ObjectAccessedAll = "s3:ObjectAccessed:*"
|
||||
ObjectRemovedAll = "s3:ObjectRemoved:*"
|
||||
ObjectRemovedDelete = "s3:ObjectRemoved:Delete"
|
||||
ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated"
|
||||
ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject"
|
||||
BucketCreatedAll = "s3:BucketCreated:*"
|
||||
BucketRemovedAll = "s3:BucketRemoved:*"
|
||||
ObjectCreatedPut EventType = "s3:ObjectCreated:Put"
|
||||
ObjectCreatedPost EventType = "s3:ObjectCreated:Post"
|
||||
ObjectCreatedCopy EventType = "s3:ObjectCreated:Copy"
|
||||
ObjectCreatedCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload"
|
||||
ObjectAccessedGet EventType = "s3:ObjectAccessed:Get"
|
||||
ObjectAccessedHead EventType = "s3:ObjectAccessed:Head"
|
||||
ObjectAccessedAll EventType = "s3:ObjectAccessed:*"
|
||||
ObjectRemovedAll EventType = "s3:ObjectRemoved:*"
|
||||
ObjectRemovedDelete EventType = "s3:ObjectRemoved:Delete"
|
||||
ObjectRemovedDeleteMarkerCreated EventType = "s3:ObjectRemoved:DeleteMarkerCreated"
|
||||
ObjectReducedRedundancyLostObject EventType = "s3:ReducedRedundancyLostObject"
|
||||
BucketCreatedAll EventType = "s3:BucketCreated:*"
|
||||
BucketRemovedAll EventType = "s3:BucketRemoved:*"
|
||||
)
|
||||
|
||||
// FilterRule - child of S3Key, a tag in the notification xml which
|
||||
|
|
|
|||
|
|
@ -700,6 +700,10 @@ type TargetMetrics struct {
|
|||
PendingCount uint64 `json:"pendingReplicationCount"`
|
||||
// Total number of failed operations including metadata updates
|
||||
FailedCount uint64 `json:"failedReplicationCount"`
|
||||
// Bandwidth limit in bytes/sec for this target
|
||||
BandWidthLimitInBytesPerSecond int64 `json:"limitInBits"`
|
||||
// Current bandwidth used in bytes/sec for this target
|
||||
CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"`
|
||||
}
|
||||
|
||||
// Metrics represents inline replication metrics for a bucket.
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
|
@ -132,7 +131,7 @@ func StreamingUnsignedV4(req *http.Request, sessionToken string, dataLen int64,
|
|||
prepareUSStreamingRequest(req, sessionToken, dataLen, reqTime)
|
||||
|
||||
if req.Body == nil {
|
||||
req.Body = ioutil.NopCloser(bytes.NewReader([]byte("")))
|
||||
req.Body = io.NopCloser(bytes.NewReader([]byte("")))
|
||||
}
|
||||
|
||||
stReader := &StreamingUSReader{
|
||||
|
|
|
|||
|
|
@ -22,11 +22,12 @@ import (
|
|||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
md5simd "github.com/minio/md5-simd"
|
||||
)
|
||||
|
||||
// Reference for constants used below -
|
||||
|
|
@ -91,14 +92,14 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
|
|||
|
||||
// buildChunkStringToSign - returns the string to sign given chunk data
|
||||
// and previous signature.
|
||||
func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string {
|
||||
func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
|
||||
stringToSignParts := []string{
|
||||
streamingPayloadHdr,
|
||||
t.Format(iso8601DateFormat),
|
||||
getScope(region, t, ServiceTypeS3),
|
||||
previousSig,
|
||||
emptySHA256,
|
||||
hex.EncodeToString(sum256(chunkData)),
|
||||
chunkChecksum,
|
||||
}
|
||||
|
||||
return strings.Join(stringToSignParts, "\n")
|
||||
|
|
@ -106,13 +107,13 @@ func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData [
|
|||
|
||||
// buildTrailerChunkStringToSign - returns the string to sign given chunk data
|
||||
// and previous signature.
|
||||
func buildTrailerChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string {
|
||||
func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
|
||||
stringToSignParts := []string{
|
||||
streamingTrailerHdr,
|
||||
t.Format(iso8601DateFormat),
|
||||
getScope(region, t, ServiceTypeS3),
|
||||
previousSig,
|
||||
hex.EncodeToString(sum256(chunkData)),
|
||||
chunkChecksum,
|
||||
}
|
||||
|
||||
return strings.Join(stringToSignParts, "\n")
|
||||
|
|
@ -149,21 +150,21 @@ func buildChunkHeader(chunkLen int64, signature string) []byte {
|
|||
}
|
||||
|
||||
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
|
||||
func buildChunkSignature(chunkData []byte, reqTime time.Time, region,
|
||||
func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region,
|
||||
previousSignature, secretAccessKey string,
|
||||
) string {
|
||||
chunkStringToSign := buildChunkStringToSign(reqTime, region,
|
||||
previousSignature, chunkData)
|
||||
previousSignature, chunkCheckSum)
|
||||
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
|
||||
return getSignature(signingKey, chunkStringToSign)
|
||||
}
|
||||
|
||||
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
|
||||
func buildTrailerChunkSignature(chunkData []byte, reqTime time.Time, region,
|
||||
func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region,
|
||||
previousSignature, secretAccessKey string,
|
||||
) string {
|
||||
chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region,
|
||||
previousSignature, chunkData)
|
||||
previousSignature, chunkChecksum)
|
||||
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
|
||||
return getSignature(signingKey, chunkStringToSign)
|
||||
}
|
||||
|
|
@ -203,12 +204,17 @@ type StreamingReader struct {
|
|||
totalChunks int
|
||||
lastChunkSize int
|
||||
trailer http.Header
|
||||
sh256 md5simd.Hasher
|
||||
}
|
||||
|
||||
// signChunk - signs a chunk read from s.baseReader of chunkLen size.
|
||||
func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) {
|
||||
// Compute chunk signature for next header
|
||||
signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime,
|
||||
s.sh256.Reset()
|
||||
s.sh256.Write(s.chunkBuf[:chunkLen])
|
||||
chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil))
|
||||
|
||||
signature := buildChunkSignature(chunckChecksum, s.reqTime,
|
||||
s.region, s.prevSignature, s.secretAccessKey)
|
||||
|
||||
// For next chunk signature computation
|
||||
|
|
@ -240,8 +246,11 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) {
|
|||
s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...)
|
||||
}
|
||||
|
||||
s.sh256.Reset()
|
||||
s.sh256.Write(s.chunkBuf)
|
||||
chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil))
|
||||
// Compute chunk signature
|
||||
signature := buildTrailerChunkSignature(s.chunkBuf, s.reqTime,
|
||||
signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime,
|
||||
s.region, s.prevSignature, s.secretAccessKey)
|
||||
|
||||
// For next chunk signature computation
|
||||
|
|
@ -274,13 +283,13 @@ func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) {
|
|||
// StreamingSignV4 - provides chunked upload signatureV4 support by
|
||||
// implementing io.Reader.
|
||||
func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken,
|
||||
region string, dataLen int64, reqTime time.Time,
|
||||
region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher,
|
||||
) *http.Request {
|
||||
// Set headers needed for streaming signature.
|
||||
prepareStreamingRequest(req, sessionToken, dataLen, reqTime)
|
||||
|
||||
if req.Body == nil {
|
||||
req.Body = ioutil.NopCloser(bytes.NewReader([]byte("")))
|
||||
req.Body = io.NopCloser(bytes.NewReader([]byte("")))
|
||||
}
|
||||
|
||||
stReader := &StreamingReader{
|
||||
|
|
@ -295,6 +304,7 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok
|
|||
chunkNum: 1,
|
||||
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
|
||||
lastChunkSize: int(dataLen % payloadChunkSize),
|
||||
sh256: sh256,
|
||||
}
|
||||
if len(req.Trailer) > 0 {
|
||||
stReader.trailer = req.Trailer
|
||||
|
|
@ -385,5 +395,9 @@ func (s *StreamingReader) Read(buf []byte) (int, error) {
|
|||
|
||||
// Close - this method makes underlying io.ReadCloser's Close method available.
|
||||
func (s *StreamingReader) Close() error {
|
||||
if s.sh256 != nil {
|
||||
s.sh256.Close()
|
||||
s.sh256 = nil
|
||||
}
|
||||
return s.baseReadCloser.Close()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import (
|
|||
)
|
||||
|
||||
// expirationDateFormat date format for expiration key in json policy.
|
||||
const expirationDateFormat = "2006-01-02T15:04:05.999Z"
|
||||
const expirationDateFormat = "2006-01-02T15:04:05.000Z"
|
||||
|
||||
// policyCondition explanation:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ package minio
|
|||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
|
|
@ -73,7 +72,7 @@ var DefaultTransport = func(secure bool) (*http.Transport, error) {
|
|||
}
|
||||
if f := os.Getenv("SSL_CERT_FILE"); f != "" {
|
||||
rootCAs := mustGetSystemCertPool()
|
||||
data, err := ioutil.ReadFile(f)
|
||||
data, err := os.ReadFile(f)
|
||||
if err == nil {
|
||||
rootCAs.AppendCertsFromPEM(data)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import (
|
|||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
|
|
@ -142,7 +141,7 @@ func closeResponse(resp *http.Response) {
|
|||
// Without this closing connection would disallow re-using
|
||||
// the same connection for future uses.
|
||||
// - http://stackoverflow.com/a/17961593/4465767
|
||||
io.Copy(ioutil.Discard, resp.Body)
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -297,7 +297,7 @@ github.com/miekg/dns
|
|||
# github.com/minio/md5-simd v1.1.2
|
||||
## explicit; go 1.14
|
||||
github.com/minio/md5-simd
|
||||
# github.com/minio/minio-go/v7 v7.0.44
|
||||
# github.com/minio/minio-go/v7 v7.0.47
|
||||
## explicit; go 1.17
|
||||
github.com/minio/minio-go/v7
|
||||
github.com/minio/minio-go/v7/pkg/credentials
|
||||
|
|
|
|||
Loading…
Reference in New Issue