//go:build !plan9 && !solaris && !js // +build !plan9,!solaris,!js package oracleobjectstorage import ( "bytes" "context" "encoding/base64" "encoding/hex" "fmt" "io" "net/http" "os" "regexp" "strconv" "strings" "time" "golang.org/x/net/http/httpguts" "github.com/ncw/swift/v2" "github.com/oracle/oci-go-sdk/v65/common" "github.com/oracle/oci-go-sdk/v65/objectstorage" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/hash" ) // ------------------------------------------------------------ // Object Interface Implementation // ------------------------------------------------------------ const ( metaMtime = "mtime" // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime metaMD5Hash = "md5chksum" // the meta key to store md5hash in // StandardTier object storage tier ociMetaPrefix = "opc-meta-" ) var archive = "archive" var infrequentAccess = "infrequentaccess" var standard = "standard" var storageTierMap = map[string]*string{ archive: &archive, infrequentAccess: &infrequentAccess, standard: &standard, } var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`) // Object describes a oci bucket object type Object struct { fs *Fs // what this object is part of remote string // The remote path md5 string // MD5 hash if known bytes int64 // Size of the object lastModified time.Time // The modified time of the object if known meta map[string]string // The object metadata if known - may be nil mimeType string // Content-Type of the object // Metadata as pointers to strings as they often won't be present storageTier *string // e.g. Standard } // split returns bucket and bucketPath from the object func (o *Object) split() (bucket, bucketPath string) { return o.fs.split(o.remote) } // readMetaData gets the metadata if it hasn't already been fetched func (o *Object) readMetaData(ctx context.Context) (err error) { fs.Debugf(o, "trying to read metadata %v", o.remote) if o.meta != nil { return nil } info, err := o.headObject(ctx) if err != nil { return err } return o.decodeMetaDataHead(info) } // headObject gets the metadata from the object unconditionally func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObjectResponse, err error) { bucketName, objectPath := o.split() req := objectstorage.HeadObjectRequest{ NamespaceName: common.String(o.fs.opt.Namespace), BucketName: common.String(bucketName), ObjectName: common.String(objectPath), } useBYOKHeadObject(o.fs, &req) var response objectstorage.HeadObjectResponse err = o.fs.pacer.Call(func() (bool, error) { var err error response, err = o.fs.srv.HeadObject(ctx, req) return shouldRetry(ctx, response.HTTPResponse(), err) }) if err != nil { if svcErr, ok := err.(common.ServiceError); ok { if svcErr.GetHTTPStatusCode() == http.StatusNotFound { return nil, fs.ErrorObjectNotFound } } fs.Errorf(o, "Failed to head object: %v", err) return nil, err } o.fs.cache.MarkOK(bucketName) return &response, err } func (o *Object) decodeMetaDataHead(info *objectstorage.HeadObjectResponse) (err error) { return o.setMetaData( info.ContentLength, info.ContentMd5, info.ContentType, info.LastModified, info.StorageTier, info.OpcMeta) } func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (err error) { return o.setMetaData( info.ContentLength, info.ContentMd5, info.ContentType, info.LastModified, info.StorageTier, info.OpcMeta) } func (o *Object) setMetaData( contentLength *int64, contentMd5 *string, contentType *string, lastModified *common.SDKTime, storageTier interface{}, meta map[string]string) error { if contentLength != nil { o.bytes = *contentLength } if contentMd5 != nil { md5, err := o.base64ToMd5(*contentMd5) if err == nil { o.md5 = md5 } } o.meta = meta if o.meta == nil { o.meta = map[string]string{} } // Read MD5 from metadata if present if md5sumBase64, ok := o.meta[metaMD5Hash]; ok { md5, err := o.base64ToMd5(md5sumBase64) if err != nil { o.md5 = md5 } } if lastModified == nil { o.lastModified = time.Now() fs.Logf(o, "Failed to read last modified") } else { o.lastModified = lastModified.Time } if contentType != nil { o.mimeType = *contentType } if storageTier == nil || storageTier == "" { o.storageTier = storageTierMap[standard] } else { tier := strings.ToLower(fmt.Sprintf("%v", storageTier)) o.storageTier = storageTierMap[tier] } return nil } func (o *Object) base64ToMd5(md5sumBase64 string) (md5 string, err error) { md5sumBytes, err := base64.StdEncoding.DecodeString(md5sumBase64) if err != nil { fs.Debugf(o, "Failed to read md5sum from metadata %q: %v", md5sumBase64, err) return "", err } else if len(md5sumBytes) != 16 { fs.Debugf(o, "failed to read md5sum from metadata %q: wrong length", md5sumBase64) return "", fmt.Errorf("failed to read md5sum from metadata %q: wrong length", md5sumBase64) } return hex.EncodeToString(md5sumBytes), nil } // Fs returns the parent Fs func (o *Object) Fs() fs.Info { return o.fs } // Remote returns the remote path func (o *Object) Remote() string { return o.remote } // Return a string version func (o *Object) String() string { if o == nil { return "" } return o.remote } // Size returns the size of an object in bytes func (o *Object) Size() int64 { return o.bytes } // GetTier returns storage class as string func (o *Object) GetTier() string { if o.storageTier == nil || *o.storageTier == "" { return standard } return *o.storageTier } // SetTier performs changing storage class func (o *Object) SetTier(tier string) (err error) { ctx := context.TODO() tier = strings.ToLower(tier) bucketName, bucketPath := o.split() tierEnum, ok := objectstorage.GetMappingStorageTierEnum(tier) if !ok { return fmt.Errorf("not a valid storage tier %v ", tier) } req := objectstorage.UpdateObjectStorageTierRequest{ NamespaceName: common.String(o.fs.opt.Namespace), BucketName: common.String(bucketName), UpdateObjectStorageTierDetails: objectstorage.UpdateObjectStorageTierDetails{ ObjectName: common.String(bucketPath), StorageTier: tierEnum, }, } _, err = o.fs.srv.UpdateObjectStorageTier(ctx, req) if err != nil { return err } o.storageTier = storageTierMap[tier] return err } // MimeType of an Object if known, "" otherwise func (o *Object) MimeType(ctx context.Context) string { err := o.readMetaData(ctx) if err != nil { fs.Logf(o, "Failed to read metadata: %v", err) return "" } return o.mimeType } // Hash returns the MD5 of an object returning a lowercase hex string func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { if t != hash.MD5 { return "", hash.ErrUnsupported } // Convert base64 encoded md5 into lower case hex if o.md5 == "" { err := o.readMetaData(ctx) if err != nil { return "", err } } return o.md5, nil } // ModTime returns the modification time of the object // // It attempts to read the objects mtime and if that isn't present the // LastModified returned to the http headers func (o *Object) ModTime(ctx context.Context) (result time.Time) { if o.fs.ci.UseServerModTime { return o.lastModified } err := o.readMetaData(ctx) if err != nil { fs.Logf(o, "Failed to read metadata: %v", err) return time.Now() } // read mtime out of metadata if available d, ok := o.meta[metaMtime] if !ok || d == "" { return o.lastModified } modTime, err := swift.FloatStringToTime(d) if err != nil { fs.Logf(o, "Failed to read mtime from object: %v", err) return o.lastModified } return modTime } // SetModTime sets the modification time of the local fs object func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { err := o.readMetaData(ctx) if err != nil { return err } o.meta[metaMtime] = swift.TimeToFloatString(modTime) _, err = o.fs.Copy(ctx, o, o.remote) return err } // Storable returns if this object is storable func (o *Object) Storable() bool { return true } // Remove an object func (o *Object) Remove(ctx context.Context) error { bucketName, bucketPath := o.split() req := objectstorage.DeleteObjectRequest{ NamespaceName: common.String(o.fs.opt.Namespace), BucketName: common.String(bucketName), ObjectName: common.String(bucketPath), } err := o.fs.pacer.Call(func() (bool, error) { resp, err := o.fs.srv.DeleteObject(ctx, req) return shouldRetry(ctx, resp.HTTPResponse(), err) }) return err } // Open object file func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { bucketName, bucketPath := o.split() req := objectstorage.GetObjectRequest{ NamespaceName: common.String(o.fs.opt.Namespace), BucketName: common.String(bucketName), ObjectName: common.String(bucketPath), } o.applyGetObjectOptions(&req, options...) useBYOKGetObject(o.fs, &req) var resp objectstorage.GetObjectResponse err := o.fs.pacer.Call(func() (bool, error) { var err error resp, err = o.fs.srv.GetObject(ctx, req) return shouldRetry(ctx, resp.HTTPResponse(), err) }) if err != nil { return nil, err } // read size from ContentLength or ContentRange bytes := resp.ContentLength if resp.ContentRange != nil { var contentRange = *resp.ContentRange slash := strings.IndexRune(contentRange, '/') if slash >= 0 { i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64) if err == nil { bytes = &i } else { fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err) } } else { fs.Debugf(o, "Failed to find length in %q", contentRange) } } err = o.decodeMetaDataObject(&resp) if err != nil { return nil, err } o.bytes = *bytes return resp.HTTPResponse().Body, nil } func isZeroLength(streamReader io.Reader) bool { switch v := streamReader.(type) { case *bytes.Buffer: return v.Len() == 0 case *bytes.Reader: return v.Len() == 0 case *strings.Reader: return v.Len() == 0 case *os.File: fi, err := v.Stat() if err != nil { return false } return fi.Size() == 0 default: return false } } // Update an object if it has changed func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { bucketName, bucketPath := o.split() err = o.fs.makeBucket(ctx, bucketName) if err != nil { return err } // determine if we like upload single or multipart. size := src.Size() multipart := size >= int64(o.fs.opt.UploadCutoff) if isZeroLength(in) { multipart = false } req := objectstorage.PutObjectRequest{ NamespaceName: common.String(o.fs.opt.Namespace), BucketName: common.String(bucketName), ObjectName: common.String(bucketPath), } // Set the mtime in the metadata modTime := src.ModTime(ctx) // Fetch metadata if --metadata is in use meta, err := fs.GetMetadataOptions(ctx, src, options) if err != nil { return fmt.Errorf("failed to read metadata from source object: %w", err) } req.OpcMeta = make(map[string]string, len(meta)+2) // merge metadata into request and user metadata for k, v := range meta { pv := common.String(v) k = strings.ToLower(k) switch k { case "cache-control": req.CacheControl = pv case "content-disposition": req.ContentDisposition = pv case "content-encoding": req.ContentEncoding = pv case "content-language": req.ContentLanguage = pv case "content-type": req.ContentType = pv case "tier": // ignore case "mtime": // mtime in meta overrides source ModTime metaModTime, err := time.Parse(time.RFC3339Nano, v) if err != nil { fs.Debugf(o, "failed to parse metadata %s: %q: %v", k, v, err) } else { modTime = metaModTime } case "btime": // write as metadata since we can't set it req.OpcMeta[k] = v default: req.OpcMeta[k] = v } } // Set the mtime in the metadata req.OpcMeta[metaMtime] = swift.TimeToFloatString(modTime) // read the md5sum if available // - for non-multipart // - so we can add a ContentMD5 // - so we can add the md5sum in the metadata as metaMD5Hash if using SSE/SSE-C // - for multipart provided checksums aren't disabled // - so we can add the md5sum in the metadata as metaMD5Hash var md5sumBase64 string var md5sumHex string if !multipart || !o.fs.opt.DisableChecksum { md5sumHex, err = src.Hash(ctx, hash.MD5) if err == nil && matchMd5.MatchString(md5sumHex) { hashBytes, err := hex.DecodeString(md5sumHex) if err == nil { md5sumBase64 = base64.StdEncoding.EncodeToString(hashBytes) if multipart && !o.fs.opt.DisableChecksum { // Set the md5sum as metadata on the object if // - a multipart upload // - the ETag is not an MD5, e.g. when using SSE/SSE-C // provided checksums aren't disabled req.OpcMeta[metaMD5Hash] = md5sumBase64 } } } } // Set the content type if it isn't set already if req.ContentType == nil { req.ContentType = common.String(fs.MimeType(ctx, src)) } if size >= 0 { req.ContentLength = common.Int64(size) } if md5sumBase64 != "" { req.ContentMD5 = &md5sumBase64 } o.applyPutOptions(&req, options...) useBYOKPutObject(o.fs, &req) if o.fs.opt.StorageTier != "" { storageTier, ok := objectstorage.GetMappingPutObjectStorageTierEnum(o.fs.opt.StorageTier) if !ok { return fmt.Errorf("not a valid storage tier: %v", o.fs.opt.StorageTier) } req.StorageTier = storageTier } // Check metadata keys and values are valid for key, value := range req.OpcMeta { if !httpguts.ValidHeaderFieldName(key) { fs.Errorf(o, "Dropping invalid metadata key %q", key) delete(req.OpcMeta, key) } else if value == "" { fs.Errorf(o, "Dropping nil metadata value for key %q", key) delete(req.OpcMeta, key) } else if !httpguts.ValidHeaderFieldValue(value) { fs.Errorf(o, "Dropping invalid metadata value %q for key %q", value, key) delete(req.OpcMeta, key) } } if multipart { err = o.uploadMultipart(ctx, &req, in, src) if err != nil { return err } } else { var resp objectstorage.PutObjectResponse err = o.fs.pacer.CallNoRetry(func() (bool, error) { req.PutObjectBody = io.NopCloser(in) resp, err = o.fs.srv.PutObject(ctx, req) return shouldRetry(ctx, resp.HTTPResponse(), err) }) if err != nil { fs.Errorf(o, "put object failed %v", err) return err } } // Read the metadata from the newly created object o.meta = nil // wipe old metadata return o.readMetaData(ctx) } func (o *Object) applyPutOptions(req *objectstorage.PutObjectRequest, options ...fs.OpenOption) { // Apply upload options for _, option := range options { key, value := option.Header() lowerKey := strings.ToLower(key) switch lowerKey { case "": // ignore case "cache-control": req.CacheControl = common.String(value) case "content-disposition": req.ContentDisposition = common.String(value) case "content-encoding": req.ContentEncoding = common.String(value) case "content-language": req.ContentLanguage = common.String(value) case "content-type": req.ContentType = common.String(value) default: if strings.HasPrefix(lowerKey, ociMetaPrefix) { req.OpcMeta[lowerKey] = value } else { fs.Errorf(o, "Don't know how to set key %q on upload", key) } } } } func (o *Object) applyGetObjectOptions(req *objectstorage.GetObjectRequest, options ...fs.OpenOption) { fs.FixRangeOption(options, o.bytes) for _, option := range options { switch option.(type) { case *fs.RangeOption, *fs.SeekOption: _, value := option.Header() req.Range = &value default: if option.Mandatory() { fs.Logf(o, "Unsupported mandatory option: %v", option) } } } // Apply upload options for _, option := range options { key, value := option.Header() lowerKey := strings.ToLower(key) switch lowerKey { case "": // ignore case "cache-control": req.HttpResponseCacheControl = common.String(value) case "content-disposition": req.HttpResponseContentDisposition = common.String(value) case "content-encoding": req.HttpResponseContentEncoding = common.String(value) case "content-language": req.HttpResponseContentLanguage = common.String(value) case "content-type": req.HttpResponseContentType = common.String(value) case "range": // do nothing default: fs.Errorf(o, "Don't know how to set key %q on upload", key) } } } func (o *Object) applyMultipartUploadOptions(putReq *objectstorage.PutObjectRequest, req *objectstorage.CreateMultipartUploadRequest) { req.ContentType = putReq.ContentType req.ContentLanguage = putReq.ContentLanguage req.ContentEncoding = putReq.ContentEncoding req.ContentDisposition = putReq.ContentDisposition req.CacheControl = putReq.CacheControl req.Metadata = metadataWithOpcPrefix(putReq.OpcMeta) req.OpcSseCustomerAlgorithm = putReq.OpcSseCustomerAlgorithm req.OpcSseCustomerKey = putReq.OpcSseCustomerKey req.OpcSseCustomerKeySha256 = putReq.OpcSseCustomerKeySha256 req.OpcSseKmsKeyId = putReq.OpcSseKmsKeyId } func (o *Object) applyPartUploadOptions(putReq *objectstorage.PutObjectRequest, req *objectstorage.UploadPartRequest) { req.OpcSseCustomerAlgorithm = putReq.OpcSseCustomerAlgorithm req.OpcSseCustomerKey = putReq.OpcSseCustomerKey req.OpcSseCustomerKeySha256 = putReq.OpcSseCustomerKeySha256 req.OpcSseKmsKeyId = putReq.OpcSseKmsKeyId } func metadataWithOpcPrefix(src map[string]string) map[string]string { dst := make(map[string]string) for lowerKey, value := range src { if !strings.HasPrefix(lowerKey, ociMetaPrefix) { dst[ociMetaPrefix+lowerKey] = value } } return dst }