mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 09:32:29 +08:00
fs: implement --metadata-mapper to transform metatadata with a user supplied program
This commit is contained in:
parent
54196f34e3
commit
47ca0c326e
|
@ -596,7 +596,7 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
|
||||||
|
|
||||||
// Fetch metadata and update updateInfo if --metadata is in use
|
// Fetch metadata and update updateInfo if --metadata is in use
|
||||||
func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *drive.File, update bool) (callback updateMetadataFn, err error) {
|
func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *drive.File, update bool) (callback updateMetadataFn, err error) {
|
||||||
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
meta, err := fs.GetMetadataOptions(ctx, f, src, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read metadata from source object: %w", err)
|
return nil, fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -802,7 +802,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||||
headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
|
headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
|
||||||
}
|
}
|
||||||
var mdata fs.Metadata
|
var mdata fs.Metadata
|
||||||
mdata, err = fs.GetMetadataOptions(ctx, src, options)
|
mdata, err = fs.GetMetadataOptions(ctx, o.fs, src, options)
|
||||||
if err == nil && mdata != nil {
|
if err == nil && mdata != nil {
|
||||||
for mk, mv := range mdata {
|
for mk, mv := range mdata {
|
||||||
mk = strings.ToLower(mk)
|
mk = strings.ToLower(mk)
|
||||||
|
|
|
@ -1944,7 +1944,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||||
in = wrap(in)
|
in = wrap(in)
|
||||||
}
|
}
|
||||||
// Fetch metadata if --metadata is in use
|
// Fetch metadata if --metadata is in use
|
||||||
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to read metadata from source object: %w", err)
|
return fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1298,7 +1298,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch and set metadata if --metadata is in use
|
// Fetch and set metadata if --metadata is in use
|
||||||
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to read metadata from source object: %w", err)
|
return fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -295,7 +295,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||||
// Set the mtime in the metadata
|
// Set the mtime in the metadata
|
||||||
modTime := src.ModTime(ctx)
|
modTime := src.ModTime(ctx)
|
||||||
// Fetch metadata if --metadata is in use
|
// Fetch metadata if --metadata is in use
|
||||||
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ui, fmt.Errorf("failed to read metadata from source object: %w", err)
|
return ui, fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -6008,7 +6008,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch metadata if --metadata is in use
|
// Fetch metadata if --metadata is in use
|
||||||
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ui, fmt.Errorf("failed to read metadata from source object: %w", err)
|
return ui, fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||||
}
|
}
|
||||||
|
|
24
bin/test_metadata_mapper.py
Executable file
24
bin/test_metadata_mapper.py
Executable file
|
@ -0,0 +1,24 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
A demo metadata mapper
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
|
||||||
|
def main():
|
||||||
|
i = json.load(sys.stdin)
|
||||||
|
# Add tag to description
|
||||||
|
metadata = i["Metadata"]
|
||||||
|
if "description" in metadata:
|
||||||
|
metadata["description"] += " [migrated from domain1]"
|
||||||
|
else:
|
||||||
|
metadata["description"] = "[migrated from domain1]"
|
||||||
|
# Modify owner
|
||||||
|
if "owner" in metadata:
|
||||||
|
metadata["owner"] = metadata["owner"].replace("domain1.com", "domain2.com")
|
||||||
|
o = { "Metadata": metadata }
|
||||||
|
json.dump(o, sys.stdout, indent="\t")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
|
@ -475,6 +475,10 @@ Note that arbitrary metadata may be added to objects using the
|
||||||
`--metadata-set key=value` flag when the object is first uploaded.
|
`--metadata-set key=value` flag when the object is first uploaded.
|
||||||
This flag can be repeated as many times as necessary.
|
This flag can be repeated as many times as necessary.
|
||||||
|
|
||||||
|
The [--metadata-mapper](#metadata-mapper) flag can be used to pass the
|
||||||
|
name of a program in which can transform metadata when it is being
|
||||||
|
copied from source to destination.
|
||||||
|
|
||||||
### Types of metadata
|
### Types of metadata
|
||||||
|
|
||||||
Metadata is divided into two type. System metadata and User metadata.
|
Metadata is divided into two type. System metadata and User metadata.
|
||||||
|
@ -1504,12 +1508,123 @@ from reaching the limit. Only applicable for `--max-transfer`
|
||||||
|
|
||||||
Setting this flag enables rclone to copy the metadata from the source
|
Setting this flag enables rclone to copy the metadata from the source
|
||||||
to the destination. For local backends this is ownership, permissions,
|
to the destination. For local backends this is ownership, permissions,
|
||||||
xattr etc. See the [#metadata](metadata section) for more info.
|
xattr etc. See the [metadata section](#metadata) for more info.
|
||||||
|
|
||||||
|
### --metadata-mapper SpaceSepList {#metadata-mapper}
|
||||||
|
|
||||||
|
If you supply the parameter `--metadata-mapper /path/to/program` then
|
||||||
|
rclone will use that program to map metadata from source object to
|
||||||
|
destination object.
|
||||||
|
|
||||||
|
The argument to this flag should be a command with an optional space separated
|
||||||
|
list of arguments. If one of the arguments has a space in then enclose
|
||||||
|
it in `"`, if you want a literal `"` in an argument then enclose the
|
||||||
|
argument in `"` and double the `"`. See [CSV encoding](https://godoc.org/encoding/csv)
|
||||||
|
for more info.
|
||||||
|
|
||||||
|
--metadata-mapper "python bin/test_metadata_mapper.py"
|
||||||
|
--metadata-mapper 'python bin/test_metadata_mapper.py "argument with a space"'
|
||||||
|
--metadata-mapper 'python bin/test_metadata_mapper.py "argument with ""two"" quotes"'
|
||||||
|
|
||||||
|
This uses a simple JSON based protocol with input on STDIN and output
|
||||||
|
on STDOUT. This will be called for every file and directory copied and
|
||||||
|
may be called concurrently.
|
||||||
|
|
||||||
|
The program's job is to take a metadata blob on the input and turn it
|
||||||
|
into a metadata blob on the output suitable for the destination
|
||||||
|
backend.
|
||||||
|
|
||||||
|
Input to the program (via STDIN) might look like this. This provides
|
||||||
|
some context for the `Metadata` which may be important.
|
||||||
|
|
||||||
|
- `SrcFs` is the config string for the remote that the object is currently on.
|
||||||
|
- `SrcFsType` is the name of the source backend.
|
||||||
|
- `DstFs` is the config string for the remote that the object is being copied to
|
||||||
|
- `DstFsType` is the name of the destination backend.
|
||||||
|
- `Remote` is the path of the file relative to the root.
|
||||||
|
- `Size`, `MimeType`, `ModTime` are attributes of the file.
|
||||||
|
- `IsDir` is `true` if this is a directory (not yet implemented).
|
||||||
|
- `ID` is the source `ID` of the file if known.
|
||||||
|
- `Metadata` is the backend specific metadata as described in the backend docs.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"SrcFs": "gdrive:",
|
||||||
|
"SrcFsType": "drive",
|
||||||
|
"DstFs": "newdrive:user",
|
||||||
|
"DstFsType": "onedrive",
|
||||||
|
"Remote": "test.txt",
|
||||||
|
"Size": 6,
|
||||||
|
"MimeType": "text/plain; charset=utf-8",
|
||||||
|
"ModTime": "2022-10-11T17:53:10.286745272+01:00",
|
||||||
|
"IsDir": false,
|
||||||
|
"ID": "xyz",
|
||||||
|
"Metadata": {
|
||||||
|
"btime": "2022-10-11T16:53:11Z",
|
||||||
|
"content-type": "text/plain; charset=utf-8",
|
||||||
|
"mtime": "2022-10-11T17:53:10.286745272+01:00",
|
||||||
|
"owner": "user1@domain1.com",
|
||||||
|
"permissions": "...",
|
||||||
|
"description": "my nice file",
|
||||||
|
"starred": "false"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The program should then modify the input as desired and send it to
|
||||||
|
STDOUT. The returned `Metadata` field will be used in its entirety for
|
||||||
|
the destination object. Any other fields will be ignored. Note in this
|
||||||
|
example we translate user names and permissions and add something to
|
||||||
|
the description:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"Metadata": {
|
||||||
|
"btime": "2022-10-11T16:53:11Z",
|
||||||
|
"content-type": "text/plain; charset=utf-8",
|
||||||
|
"mtime": "2022-10-11T17:53:10.286745272+01:00",
|
||||||
|
"owner": "user1@domain2.com",
|
||||||
|
"permissions": "...",
|
||||||
|
"description": "my nice file [migrated from domain1]",
|
||||||
|
"starred": "false"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Metadata can be removed here too.
|
||||||
|
|
||||||
|
An example python program might look something like this to implement
|
||||||
|
the above transformations.
|
||||||
|
|
||||||
|
```python
|
||||||
|
import sys, json
|
||||||
|
|
||||||
|
i = json.load(sys.stdin)
|
||||||
|
metadata = i["Metadata"]
|
||||||
|
# Add tag to description
|
||||||
|
if "description" in metadata:
|
||||||
|
metadata["description"] += " [migrated from domain1]"
|
||||||
|
else:
|
||||||
|
metadata["description"] = "[migrated from domain1]"
|
||||||
|
# Modify owner
|
||||||
|
if "owner" in metadata:
|
||||||
|
metadata["owner"] = metadata["owner"].replace("domain1.com", "domain2.com")
|
||||||
|
o = { "Metadata": metadata }
|
||||||
|
json.dump(o, sys.stdout, indent="\t")
|
||||||
|
```
|
||||||
|
|
||||||
|
You can find this example (slightly expanded) in the rclone source code at
|
||||||
|
[bin/test_metadata_mapper.py](https://github.com/rclone/rclone/blob/master/test_metadata_mapper.py).
|
||||||
|
|
||||||
|
If you want to see the input to the metadata mapper and the output
|
||||||
|
returned from it in the log you can use `-vv --dump mapper`.
|
||||||
|
|
||||||
|
See the [metadata section](#metadata) for more info.
|
||||||
|
|
||||||
### --metadata-set key=value
|
### --metadata-set key=value
|
||||||
|
|
||||||
Add metadata `key` = `value` when uploading. This can be repeated as
|
Add metadata `key` = `value` when uploading. This can be repeated as
|
||||||
many times as required. See the [#metadata](metadata section) for more
|
many times as required. See the [metadata section](#metadata) for more
|
||||||
info.
|
info.
|
||||||
|
|
||||||
### --modify-window=TIME ###
|
### --modify-window=TIME ###
|
||||||
|
@ -1752,9 +1867,9 @@ for more info.
|
||||||
|
|
||||||
Eg
|
Eg
|
||||||
|
|
||||||
--password-command echo hello
|
--password-command "echo hello"
|
||||||
--password-command echo "hello with space"
|
--password-command 'echo "hello with space"'
|
||||||
--password-command echo "hello with ""quotes"" and space"
|
--password-command 'echo "hello with ""quotes"" and space"'
|
||||||
|
|
||||||
See the [Configuration Encryption](#configuration-encryption) for more info.
|
See the [Configuration Encryption](#configuration-encryption) for more info.
|
||||||
|
|
||||||
|
@ -2503,6 +2618,12 @@ This dumps a list of the open files at the end of the command. It
|
||||||
uses the `lsof` command to do that so you'll need that installed to
|
uses the `lsof` command to do that so you'll need that installed to
|
||||||
use it.
|
use it.
|
||||||
|
|
||||||
|
#### --dump mapper ####
|
||||||
|
|
||||||
|
This shows the JSON blobs being sent to the program supplied with
|
||||||
|
`--metadata-mapper` and received from it. It can be useful for
|
||||||
|
debugging the metadata mapper interface.
|
||||||
|
|
||||||
### --memprofile=FILE ###
|
### --memprofile=FILE ###
|
||||||
|
|
||||||
Write memory profile to file. This can be analysed with `go tool pprof`.
|
Write memory profile to file. This can be analysed with `go tool pprof`.
|
||||||
|
|
|
@ -149,6 +149,7 @@ type ConfigInfo struct {
|
||||||
DefaultTime Time // time that directories with no time should display
|
DefaultTime Time // time that directories with no time should display
|
||||||
Inplace bool // Download directly to destination file instead of atomic download to temp/rename
|
Inplace bool // Download directly to destination file instead of atomic download to temp/rename
|
||||||
PartialSuffix string
|
PartialSuffix string
|
||||||
|
MetadataMapper SpaceSepList
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewConfig creates a new config with everything set to the default
|
// NewConfig creates a new config with everything set to the default
|
||||||
|
|
|
@ -150,6 +150,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
|
||||||
flags.FVarP(flagSet, &ci.DefaultTime, "default-time", "", "Time to show if modtime is unknown for files and directories", "Config,Listing")
|
flags.FVarP(flagSet, &ci.DefaultTime, "default-time", "", "Time to show if modtime is unknown for files and directories", "Config,Listing")
|
||||||
flags.BoolVarP(flagSet, &ci.Inplace, "inplace", "", ci.Inplace, "Download directly to destination file instead of atomic download to temp/rename", "Copy")
|
flags.BoolVarP(flagSet, &ci.Inplace, "inplace", "", ci.Inplace, "Download directly to destination file instead of atomic download to temp/rename", "Copy")
|
||||||
flags.StringVarP(flagSet, &partialSuffix, "partial-suffix", "", ci.PartialSuffix, "Add partial-suffix to temporary file name when --inplace is not used", "Copy")
|
flags.StringVarP(flagSet, &partialSuffix, "partial-suffix", "", ci.PartialSuffix, "Add partial-suffix to temporary file name when --inplace is not used", "Copy")
|
||||||
|
flags.FVarP(flagSet, &ci.MetadataMapper, "metadata-mapper", "", "Program to run to transforming metadata before upload", "Metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseHeaders converts the strings passed in via the header flags into HTTPOptions
|
// ParseHeaders converts the strings passed in via the header flags into HTTPOptions
|
||||||
|
|
|
@ -13,6 +13,7 @@ const (
|
||||||
DumpFilters
|
DumpFilters
|
||||||
DumpGoRoutines
|
DumpGoRoutines
|
||||||
DumpOpenFiles
|
DumpOpenFiles
|
||||||
|
DumpMapper
|
||||||
)
|
)
|
||||||
|
|
||||||
type dumpChoices struct{}
|
type dumpChoices struct{}
|
||||||
|
@ -27,6 +28,7 @@ func (dumpChoices) Choices() []BitsChoicesInfo {
|
||||||
{uint64(DumpFilters), "filters"},
|
{uint64(DumpFilters), "filters"},
|
||||||
{uint64(DumpGoRoutines), "goroutines"},
|
{uint64(DumpGoRoutines), "goroutines"},
|
||||||
{uint64(DumpOpenFiles), "openfiles"},
|
{uint64(DumpOpenFiles), "openfiles"},
|
||||||
|
{uint64(DumpMapper), "mapper"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,14 @@
|
||||||
package fs
|
package fs
|
||||||
|
|
||||||
import "context"
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
// Metadata represents Object metadata in a standardised form
|
// Metadata represents Object metadata in a standardised form
|
||||||
//
|
//
|
||||||
|
@ -66,12 +74,85 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro
|
||||||
return do.Metadata(ctx)
|
return do.Metadata(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mapItem descripts the item to be mapped
|
||||||
|
type mapItem struct {
|
||||||
|
SrcFs string
|
||||||
|
SrcFsType string
|
||||||
|
DstFs string
|
||||||
|
DstFsType string
|
||||||
|
Remote string
|
||||||
|
Size int64
|
||||||
|
MimeType string `json:",omitempty"`
|
||||||
|
ModTime time.Time
|
||||||
|
IsDir bool
|
||||||
|
ID string `json:",omitempty"`
|
||||||
|
Metadata Metadata `json:",omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// This runs an external program on the metadata which can be used to
|
||||||
|
// map it from one form to another.
|
||||||
|
func metadataMapper(ctx context.Context, cmdLine SpaceSepList, dstFs Fs, o ObjectInfo, metadata Metadata) (newMetadata Metadata, err error) {
|
||||||
|
ci := GetConfig(ctx)
|
||||||
|
cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
|
||||||
|
in := mapItem{
|
||||||
|
DstFs: ConfigString(dstFs),
|
||||||
|
DstFsType: Type(dstFs),
|
||||||
|
Remote: o.Remote(),
|
||||||
|
Size: o.Size(),
|
||||||
|
MimeType: MimeType(ctx, o),
|
||||||
|
ModTime: o.ModTime(ctx),
|
||||||
|
IsDir: false,
|
||||||
|
Metadata: metadata,
|
||||||
|
}
|
||||||
|
fInfo := o.Fs()
|
||||||
|
if f, ok := fInfo.(Fs); ok {
|
||||||
|
in.SrcFs = ConfigString(f)
|
||||||
|
in.SrcFsType = Type(f)
|
||||||
|
} else {
|
||||||
|
in.SrcFs = fInfo.Name() + ":" + fInfo.Root()
|
||||||
|
in.SrcFsType = "unknown"
|
||||||
|
}
|
||||||
|
if do, ok := o.(IDer); ok {
|
||||||
|
in.ID = do.ID()
|
||||||
|
}
|
||||||
|
inBytes, err := json.MarshalIndent(in, "", "\t")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("metadata mapper: failed to marshal input: %w", err)
|
||||||
|
}
|
||||||
|
if ci.Dump.IsSet(DumpMapper) {
|
||||||
|
Debugf(nil, "Metadata mapper sent: \n%s\n", string(inBytes))
|
||||||
|
}
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
cmd.Stdin = bytes.NewBuffer(inBytes)
|
||||||
|
cmd.Stdout = &stdout
|
||||||
|
cmd.Stderr = &stderr
|
||||||
|
start := time.Now()
|
||||||
|
err = cmd.Run()
|
||||||
|
Debugf(o, "Calling metadata mapper %v", cmdLine)
|
||||||
|
duration := time.Since(start)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("metadata mapper: failed on %v: %q: %w", cmdLine, strings.TrimSpace(stderr.String()), err)
|
||||||
|
}
|
||||||
|
if ci.Dump.IsSet(DumpMapper) {
|
||||||
|
Debugf(nil, "Metadata mapper received: \n%s\n", stdout.String())
|
||||||
|
}
|
||||||
|
var out mapItem
|
||||||
|
err = json.Unmarshal(stdout.Bytes(), &out)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("metadata mapper: failed to read output: %q: %w", stdout.String(), err)
|
||||||
|
}
|
||||||
|
Debugf(o, "Metadata mapper returned in %v", duration)
|
||||||
|
return out.Metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetMetadataOptions from an ObjectInfo and merge it with any in options
|
// GetMetadataOptions from an ObjectInfo and merge it with any in options
|
||||||
//
|
//
|
||||||
// If --metadata isn't in use it will return nil
|
// If --metadata isn't in use it will return nil.
|
||||||
//
|
//
|
||||||
// If the object has no metadata then metadata will be nil
|
// If the object has no metadata then metadata will be nil.
|
||||||
func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) {
|
//
|
||||||
|
// This should be passed the destination Fs for the metadata mapper
|
||||||
|
func GetMetadataOptions(ctx context.Context, dstFs Fs, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) {
|
||||||
ci := GetConfig(ctx)
|
ci := GetConfig(ctx)
|
||||||
if !ci.Metadata {
|
if !ci.Metadata {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
@ -81,5 +162,11 @@ func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
metadata.MergeOptions(options)
|
metadata.MergeOptions(options)
|
||||||
|
if len(ci.MetadataMapper) != 0 {
|
||||||
|
metadata, err = metadataMapper(ctx, ci.MetadataMapper, dstFs, o, metadata)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
return metadata, nil
|
return metadata, nil
|
||||||
}
|
}
|
||||||
|
|
75
fs/metadata_mapper_code.go
Normal file
75
fs/metadata_mapper_code.go
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
//go:build ignore
|
||||||
|
// +build ignore
|
||||||
|
|
||||||
|
// A simple metadata mapper for testing purposes
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
func check[T comparable](in map[string]any, key string, want T) {
|
||||||
|
value, ok := in[key]
|
||||||
|
if !ok {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s key not found\n", key)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
if value.(T) != want {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s wrong - expecting %s but got %s\n", key, want, value)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// Read the input
|
||||||
|
var in map[string]any
|
||||||
|
err := json.NewDecoder(os.Stdin).Decode(&in)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check the input
|
||||||
|
metadata, ok := in["Metadata"]
|
||||||
|
if !ok {
|
||||||
|
fmt.Fprintf(os.Stderr, "Metadata key not found\n")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
check(in, "Size", 5.0)
|
||||||
|
check(in, "SrcFs", "memory:")
|
||||||
|
check(in, "SrcFsType", "object.memoryFs")
|
||||||
|
check(in, "DstFs", "dstFs:dstFsRoot")
|
||||||
|
check(in, "DstFsType", "mockfs")
|
||||||
|
check(in, "Remote", "file.txt")
|
||||||
|
check(in, "MimeType", "text/plain; charset=utf-8")
|
||||||
|
check(in, "ModTime", "2001-02-03T04:05:06.000000007Z")
|
||||||
|
check(in, "IsDir", false)
|
||||||
|
//check(in, "ID", "Potato")
|
||||||
|
|
||||||
|
// Map the metadata
|
||||||
|
metadataOut := map[string]string{}
|
||||||
|
var out = map[string]any{
|
||||||
|
"Metadata": metadataOut,
|
||||||
|
}
|
||||||
|
for k, v := range metadata.(map[string]any) {
|
||||||
|
switch k {
|
||||||
|
case "error":
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: %s\n", v)
|
||||||
|
os.Exit(1)
|
||||||
|
case "key1":
|
||||||
|
v = "two " + v.(string)
|
||||||
|
case "key3":
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
metadataOut[k] = v.(string)
|
||||||
|
}
|
||||||
|
metadataOut["key0"] = "cabbage"
|
||||||
|
|
||||||
|
// Write the output
|
||||||
|
json.NewEncoder(os.Stdout).Encode(&out)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,14 +1,20 @@
|
||||||
package fs
|
package fs_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rclone/rclone/fs"
|
||||||
|
"github.com/rclone/rclone/fs/object"
|
||||||
|
"github.com/rclone/rclone/fstest/mockfs"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMetadataSet(t *testing.T) {
|
func TestMetadataSet(t *testing.T) {
|
||||||
var m Metadata
|
var m fs.Metadata
|
||||||
assert.Nil(t, m)
|
assert.Nil(t, m)
|
||||||
m.Set("key", "value")
|
m.Set("key", "value")
|
||||||
assert.NotNil(t, m)
|
assert.NotNil(t, m)
|
||||||
|
@ -19,34 +25,34 @@ func TestMetadataSet(t *testing.T) {
|
||||||
|
|
||||||
func TestMetadataMerge(t *testing.T) {
|
func TestMetadataMerge(t *testing.T) {
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
in Metadata
|
in fs.Metadata
|
||||||
merge Metadata
|
merge fs.Metadata
|
||||||
want Metadata
|
want fs.Metadata
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
in: Metadata{},
|
in: fs.Metadata{},
|
||||||
merge: Metadata{},
|
merge: fs.Metadata{},
|
||||||
want: Metadata{},
|
want: fs.Metadata{},
|
||||||
}, {
|
}, {
|
||||||
in: nil,
|
in: nil,
|
||||||
merge: nil,
|
merge: nil,
|
||||||
want: nil,
|
want: nil,
|
||||||
}, {
|
}, {
|
||||||
in: nil,
|
in: nil,
|
||||||
merge: Metadata{},
|
merge: fs.Metadata{},
|
||||||
want: nil,
|
want: nil,
|
||||||
}, {
|
}, {
|
||||||
in: nil,
|
in: nil,
|
||||||
merge: Metadata{"a": "1", "b": "2"},
|
merge: fs.Metadata{"a": "1", "b": "2"},
|
||||||
want: Metadata{"a": "1", "b": "2"},
|
want: fs.Metadata{"a": "1", "b": "2"},
|
||||||
}, {
|
}, {
|
||||||
in: Metadata{"a": "1", "b": "2"},
|
in: fs.Metadata{"a": "1", "b": "2"},
|
||||||
merge: nil,
|
merge: nil,
|
||||||
want: Metadata{"a": "1", "b": "2"},
|
want: fs.Metadata{"a": "1", "b": "2"},
|
||||||
}, {
|
}, {
|
||||||
in: Metadata{"a": "1", "b": "2"},
|
in: fs.Metadata{"a": "1", "b": "2"},
|
||||||
merge: Metadata{"b": "B", "c": "3"},
|
merge: fs.Metadata{"b": "B", "c": "3"},
|
||||||
want: Metadata{"a": "1", "b": "B", "c": "3"},
|
want: fs.Metadata{"a": "1", "b": "B", "c": "3"},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge)
|
what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge)
|
||||||
|
@ -57,36 +63,36 @@ func TestMetadataMerge(t *testing.T) {
|
||||||
|
|
||||||
func TestMetadataMergeOptions(t *testing.T) {
|
func TestMetadataMergeOptions(t *testing.T) {
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
in Metadata
|
in fs.Metadata
|
||||||
opts []OpenOption
|
opts []fs.OpenOption
|
||||||
want Metadata
|
want fs.Metadata
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
opts: []OpenOption{},
|
opts: []fs.OpenOption{},
|
||||||
want: nil,
|
want: nil,
|
||||||
}, {
|
}, {
|
||||||
opts: []OpenOption{&HTTPOption{}},
|
opts: []fs.OpenOption{&fs.HTTPOption{}},
|
||||||
want: nil,
|
want: nil,
|
||||||
}, {
|
}, {
|
||||||
opts: []OpenOption{MetadataOption{"a": "1", "b": "2"}},
|
opts: []fs.OpenOption{fs.MetadataOption{"a": "1", "b": "2"}},
|
||||||
want: Metadata{"a": "1", "b": "2"},
|
want: fs.Metadata{"a": "1", "b": "2"},
|
||||||
}, {
|
}, {
|
||||||
opts: []OpenOption{
|
opts: []fs.OpenOption{
|
||||||
&HTTPOption{},
|
&fs.HTTPOption{},
|
||||||
MetadataOption{"a": "1", "b": "2"},
|
fs.MetadataOption{"a": "1", "b": "2"},
|
||||||
MetadataOption{"b": "B", "c": "3"},
|
fs.MetadataOption{"b": "B", "c": "3"},
|
||||||
&HTTPOption{},
|
&fs.HTTPOption{},
|
||||||
},
|
},
|
||||||
want: Metadata{"a": "1", "b": "B", "c": "3"},
|
want: fs.Metadata{"a": "1", "b": "B", "c": "3"},
|
||||||
}, {
|
}, {
|
||||||
in: Metadata{"a": "first", "z": "OK"},
|
in: fs.Metadata{"a": "first", "z": "OK"},
|
||||||
opts: []OpenOption{
|
opts: []fs.OpenOption{
|
||||||
&HTTPOption{},
|
&fs.HTTPOption{},
|
||||||
MetadataOption{"a": "1", "b": "2"},
|
fs.MetadataOption{"a": "1", "b": "2"},
|
||||||
MetadataOption{"b": "B", "c": "3"},
|
fs.MetadataOption{"b": "B", "c": "3"},
|
||||||
&HTTPOption{},
|
&fs.HTTPOption{},
|
||||||
},
|
},
|
||||||
want: Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"},
|
want: fs.Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts)
|
what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts)
|
||||||
|
@ -94,3 +100,58 @@ func TestMetadataMergeOptions(t *testing.T) {
|
||||||
assert.Equal(t, test.want, test.in, what)
|
assert.Equal(t, test.want, test.in, what)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMetadataMapper(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
ctx, ci := fs.AddConfig(ctx)
|
||||||
|
ci.Metadata = true
|
||||||
|
require.NoError(t, ci.MetadataMapper.Set("go run metadata_mapper_code.go"))
|
||||||
|
now := time.Date(2001, 2, 3, 4, 5, 6, 7, time.UTC)
|
||||||
|
f, err := mockfs.NewFs(ctx, "dstFs", "dstFsRoot", nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Run("Normal", func(t *testing.T) {
|
||||||
|
o := object.NewMemoryObject("file.txt", now, []byte("hello")).WithMetadata(fs.Metadata{
|
||||||
|
"key1": "potato",
|
||||||
|
"key2": "sausage",
|
||||||
|
"key3": "gravy",
|
||||||
|
})
|
||||||
|
metadata, err := fs.GetMetadataOptions(ctx, f, o, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, fs.Metadata{
|
||||||
|
"key0": "cabbage",
|
||||||
|
"key1": "two potato",
|
||||||
|
"key2": "sausage",
|
||||||
|
}, metadata)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("Error", func(t *testing.T) {
|
||||||
|
o := object.NewMemoryObject("file.txt", now, []byte("hello")).WithMetadata(fs.Metadata{
|
||||||
|
"error": "Red Alert",
|
||||||
|
})
|
||||||
|
metadata, err := fs.GetMetadataOptions(ctx, f, o, nil)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.ErrorContains(t, err, "Red Alert")
|
||||||
|
require.Nil(t, metadata)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("Merge", func(t *testing.T) {
|
||||||
|
o := object.NewMemoryObject("file.txt", now, []byte("hello")).WithMetadata(fs.Metadata{
|
||||||
|
"key1": "potato",
|
||||||
|
"key2": "sausage",
|
||||||
|
"key3": "gravy",
|
||||||
|
})
|
||||||
|
metadata, err := fs.GetMetadataOptions(ctx, f, o, []fs.OpenOption{fs.MetadataOption(fs.Metadata{
|
||||||
|
"option": "optionValue",
|
||||||
|
"key1": "new potato",
|
||||||
|
"key2": "salami",
|
||||||
|
})})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, fs.Metadata{
|
||||||
|
"key0": "cabbage",
|
||||||
|
"key1": "two new potato",
|
||||||
|
"key2": "salami",
|
||||||
|
"option": "optionValue",
|
||||||
|
}, metadata)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user