/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2025 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "errors" "fmt" "io" "net/http" "strconv" "github.com/minio/minio-go/v7/pkg/s3utils" ) // AppendObjectOptions https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-buckets-objects-append.html type AppendObjectOptions struct { // Provide a progress reader to indicate the current append() progress. Progress io.Reader // ChunkSize indicates the maximum append() size, // it is useful when you want to control how much data // per append() you are interested in sending to server // while keeping the input io.Reader of a longer length. ChunkSize uint64 // Aggressively disable sha256 payload, it is automatically // turned-off for TLS supporting endpoints, useful in benchmarks // where you are interested in the peak() numbers. DisableContentSha256 bool customHeaders http.Header checksumType ChecksumType } // Header returns the custom header for AppendObject API func (opts AppendObjectOptions) Header() (header http.Header) { header = make(http.Header) for k, v := range opts.customHeaders { header[k] = v } return header } func (opts *AppendObjectOptions) setWriteOffset(offset int64) { if len(opts.customHeaders) == 0 { opts.customHeaders = make(http.Header) } opts.customHeaders["x-amz-write-offset-bytes"] = []string{strconv.FormatInt(offset, 10)} } func (opts *AppendObjectOptions) setChecksumParams(info ObjectInfo) { if len(opts.customHeaders) == 0 { opts.customHeaders = make(http.Header) } fullObject := info.ChecksumMode == ChecksumFullObjectMode.String() switch { case info.ChecksumCRC32 != "": if fullObject { opts.checksumType = ChecksumFullObjectCRC32 } case info.ChecksumCRC32C != "": if fullObject { opts.checksumType = ChecksumFullObjectCRC32C } case info.ChecksumCRC64NVME != "": // CRC64NVME only has a full object variant // so it does not carry any special full object // modifier opts.checksumType = ChecksumCRC64NVME } } func (opts AppendObjectOptions) validate(c *Client) (err error) { if opts.ChunkSize > maxPartSize { return errInvalidArgument("Append chunkSize cannot be larger than max part size allowed") } switch { case !c.trailingHeaderSupport: return errInvalidArgument("AppendObject() requires Client with TrailingHeaders enabled") case c.overrideSignerType.IsV2(): return errInvalidArgument("AppendObject() cannot be used with v2 signatures") case s3utils.IsGoogleEndpoint(*c.endpointURL): return errInvalidArgument("AppendObject() cannot be used with GCS endpoints") } return nil } // appendObjectDo - executes the append object http operation. // NOTE: You must have WRITE permissions on a bucket to add an object to it. func (c *Client) appendObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts AppendObjectOptions) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Set headers. customHeader := opts.Header() // Populate request metadata. reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, customHeader: customHeader, contentBody: reader, contentLength: size, streamSha256: !opts.DisableContentSha256, } if opts.checksumType.IsSet() { reqMetadata.addCrc = &opts.checksumType } // Execute PUT an objectName. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return UploadInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } h := resp.Header // When AppendObject() is used, S3 Express will return final object size as x-amz-object-size if amzSize := h.Get("x-amz-object-size"); amzSize != "" { size, err = strconv.ParseInt(amzSize, 10, 64) if err != nil { return UploadInfo{}, err } } return UploadInfo{ Bucket: bucketName, Key: objectName, ETag: trimEtag(h.Get("ETag")), Size: size, // Checksum values ChecksumCRC32: h.Get(ChecksumCRC32.Key()), ChecksumCRC32C: h.Get(ChecksumCRC32C.Key()), ChecksumSHA1: h.Get(ChecksumSHA1.Key()), ChecksumSHA256: h.Get(ChecksumSHA256.Key()), ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()), ChecksumMode: h.Get(ChecksumFullObjectMode.Key()), }, nil } // AppendObject - S3 Express Zone https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-buckets-objects-append.html func (c *Client) AppendObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts AppendObjectOptions, ) (info UploadInfo, err error) { if objectSize < 0 && opts.ChunkSize == 0 { return UploadInfo{}, errors.New("object size must be provided when no chunk size is provided") } if err = opts.validate(c); err != nil { return UploadInfo{}, err } oinfo, err := c.StatObject(ctx, bucketName, objectName, StatObjectOptions{Checksum: true}) if err != nil { return UploadInfo{}, err } if oinfo.ChecksumMode != ChecksumFullObjectMode.String() { return UploadInfo{}, fmt.Errorf("append API is not allowed on objects that are not full_object checksum type: %s", oinfo.ChecksumMode) } opts.setChecksumParams(oinfo) // set the appropriate checksum params based on the existing object checksum metadata. opts.setWriteOffset(oinfo.Size) // First append must set the current object size as the offset. if opts.ChunkSize > 0 { finalObjSize := int64(-1) if objectSize > 0 { finalObjSize = info.Size + objectSize } totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(finalObjSize, opts.ChunkSize) if err != nil { return UploadInfo{}, err } buf := make([]byte, partSize) var partNumber int for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { // Proceed to upload the part. if partNumber == totalPartsCount { partSize = lastPartSize } n, err := readFull(reader, buf) if err != nil { return info, err } if n != int(partSize) { return info, io.ErrUnexpectedEOF } rd := newHook(bytes.NewReader(buf[:n]), opts.Progress) uinfo, err := c.appendObjectDo(ctx, bucketName, objectName, rd, partSize, opts) if err != nil { return info, err } opts.setWriteOffset(uinfo.Size) } } rd := newHook(reader, opts.Progress) return c.appendObjectDo(ctx, bucketName, objectName, rd, objectSize, opts) }