jottacloud: add support for reading and writing metadata
Most useful is the addition of the file created timestamp, but also a timestamp for when the file was uploaded. Currently supporting a rather minimalistic set of metadata items, see PR #6359 for some thoughts about possible extensions.
This commit is contained in:
albertony
parent
b296f37801
commit
19ad39fa1c
5 changed files with 158 additions and 26 deletions
|
|
@ -395,7 +395,7 @@ type JottaFile struct {
|
|||
State string `xml:"currentRevision>state"`
|
||||
CreatedAt JottaTime `xml:"currentRevision>created"`
|
||||
ModifiedAt JottaTime `xml:"currentRevision>modified"`
|
||||
Updated JottaTime `xml:"currentRevision>updated"`
|
||||
UpdatedAt JottaTime `xml:"currentRevision>updated"`
|
||||
Size int64 `xml:"currentRevision>size"`
|
||||
MimeType string `xml:"currentRevision>mime"`
|
||||
MD5 string `xml:"currentRevision>md5"`
|
||||
|
|
|
|||
|
|
@ -92,6 +92,33 @@ func init() {
|
|||
Description: "Jottacloud",
|
||||
NewFs: NewFs,
|
||||
Config: Config,
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
Help: `Jottacloud has limited support for metadata, currently an extended set of timestamps.`,
|
||||
System: map[string]fs.MetadataHelp{
|
||||
"btime": {
|
||||
Help: "Time of file birth (creation), read from rclone metadata",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||
},
|
||||
"mtime": {
|
||||
Help: "Time of last modification, read from rclone metadata",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||
},
|
||||
"utime": {
|
||||
Help: "Time of last upload, when current revision was created, generated by backend",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"content-type": {
|
||||
Help: "MIME type, also known as media type",
|
||||
Type: "string",
|
||||
Example: "text/plain",
|
||||
ReadOnly: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "md5_memory_limit",
|
||||
Help: "Files bigger than this will be cached on disk to calculate the MD5 if required.",
|
||||
|
|
@ -497,7 +524,9 @@ type Object struct {
|
|||
remote string
|
||||
hasMetaData bool
|
||||
size int64
|
||||
createTime time.Time
|
||||
modTime time.Time
|
||||
updateTime time.Time
|
||||
md5 string
|
||||
mimeType string
|
||||
}
|
||||
|
|
@ -972,6 +1001,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
|||
CanHaveEmptyDirectories: true,
|
||||
ReadMimeType: true,
|
||||
WriteMimeType: false,
|
||||
ReadMetadata: true,
|
||||
WriteMetadata: true,
|
||||
UserMetadata: false,
|
||||
}).Fill(ctx, f)
|
||||
f.jfsSrv.SetErrorHandler(errorHandler)
|
||||
if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
|
||||
|
|
@ -1158,6 +1190,7 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback
|
|||
remote: filesystem.opt.Enc.ToStandardPath(path.Join(f.Path, f.Name)),
|
||||
size: f.Size,
|
||||
md5: f.Checksum,
|
||||
createTime: time.Time(f.Created),
|
||||
modTime: time.Time(f.Modified),
|
||||
})
|
||||
}
|
||||
|
|
@ -1387,7 +1420,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
|||
// is currently in trash, but can be made to match, it will be
|
||||
// restored. Returns ErrorObjectNotFound if upload will be necessary
|
||||
// to get a matching remote file.
|
||||
func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
|
||||
func (f *Fs) createOrUpdate(ctx context.Context, file string, createTime time.Time, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: f.filePath(file),
|
||||
|
|
@ -1397,11 +1430,10 @@ func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time,
|
|||
|
||||
opts.Parameters.Set("cphash", "true")
|
||||
|
||||
fileDate := api.JottaTime(modTime).String()
|
||||
opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
|
||||
opts.ExtraHeaders["JMd5"] = md5
|
||||
opts.ExtraHeaders["JCreated"] = fileDate
|
||||
opts.ExtraHeaders["JModified"] = fileDate
|
||||
opts.ExtraHeaders["JCreated"] = api.JottaTime(createTime).String()
|
||||
opts.ExtraHeaders["JModified"] = api.JottaTime(modTime).String()
|
||||
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
|
|
@ -1464,7 +1496,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
|||
// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
|
||||
if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
|
||||
fs.Debugf(src, "Server-side copied to trashed destination, restoring")
|
||||
info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
|
||||
info, err = f.createOrUpdate(ctx, remote, srcObj.createTime, srcObj.modTime, srcObj.size, srcObj.md5)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -1727,7 +1759,9 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) {
|
|||
o.size = info.Size
|
||||
o.md5 = info.MD5
|
||||
o.mimeType = info.MimeType
|
||||
o.createTime = time.Time(info.CreatedAt)
|
||||
o.modTime = time.Time(info.ModifiedAt)
|
||||
o.updateTime = time.Time(info.UpdatedAt)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -1772,7 +1806,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
|||
// (note that if size/md5 does not match, the file content will
|
||||
// also be modified if deduplication is possible, i.e. it is
|
||||
// important to use correct/latest values)
|
||||
_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
|
||||
_, err = o.fs.createOrUpdate(ctx, o.remote, o.createTime, modTime, o.size, o.md5)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
|
||||
|
|
@ -1909,6 +1943,37 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
// Wrap the accounting back onto the stream
|
||||
in = wrap(in)
|
||||
}
|
||||
// Fetch metadata if --metadata is in use
|
||||
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||
}
|
||||
var createdTime string
|
||||
var modTime string
|
||||
if meta != nil {
|
||||
if v, ok := meta["btime"]; ok {
|
||||
t, err := time.Parse(time.RFC3339Nano, v) // metadata stores RFC3339Nano timestamps
|
||||
if err != nil {
|
||||
fs.Debugf(o, "failed to parse metadata btime: %q: %v", v, err)
|
||||
} else {
|
||||
createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
|
||||
}
|
||||
}
|
||||
if v, ok := meta["mtime"]; ok {
|
||||
t, err := time.Parse(time.RFC3339Nano, v)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "failed to parse metadata mtime: %q: %v", v, err)
|
||||
} else {
|
||||
modTime = api.Rfc3339Time(t).String()
|
||||
}
|
||||
}
|
||||
}
|
||||
if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime
|
||||
modTime = api.Rfc3339Time(src.ModTime(ctx)).String()
|
||||
}
|
||||
if createdTime == "" { // if no Created time set same as Modified
|
||||
createdTime = modTime
|
||||
}
|
||||
|
||||
// use the api to allocate the file first and get resume / deduplication info
|
||||
var resp *http.Response
|
||||
|
|
@ -1918,13 +1983,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
Options: options,
|
||||
ExtraHeaders: make(map[string]string),
|
||||
}
|
||||
fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
|
||||
|
||||
// the allocate request
|
||||
var request = api.AllocateFileRequest{
|
||||
Bytes: size,
|
||||
Created: fileDate,
|
||||
Modified: fileDate,
|
||||
Created: createdTime,
|
||||
Modified: modTime,
|
||||
Md5: md5String,
|
||||
Path: o.fs.allocatePathRaw(o.remote, true),
|
||||
}
|
||||
|
|
@ -1939,7 +2003,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
return err
|
||||
}
|
||||
|
||||
// If the file state is INCOMPLETE and CORRUPT, try to upload a then
|
||||
// If the file state is INCOMPLETE and CORRUPT, we must upload it.
|
||||
// Else, if the file state is COMPLETE, we don't need to upload it because
|
||||
// the content is already there, possibly it was created with deduplication,
|
||||
// and also any metadata changes are already performed by the allocate request.
|
||||
if response.State != "COMPLETED" {
|
||||
// how much do we still have to upload?
|
||||
remainingBytes := size - response.ResumePos
|
||||
|
|
@ -1963,22 +2030,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
}
|
||||
|
||||
// send the remaining bytes
|
||||
resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
|
||||
_, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// finally update the meta data
|
||||
o.hasMetaData = true
|
||||
o.size = result.Bytes
|
||||
o.md5 = result.Md5
|
||||
o.modTime = time.Unix(result.Modified/1000, 0)
|
||||
} else {
|
||||
// If the file state is COMPLETE we don't need to upload it because the file was already found but we still need to update our metadata
|
||||
return o.readMetaData(ctx, true)
|
||||
// Upload response contains main metadata properties (size, md5 and modTime)
|
||||
// which could be set back to the object, but it does not contain the
|
||||
// necessary information to set the createTime and updateTime properties,
|
||||
// so must therefore perform a read instead.
|
||||
}
|
||||
|
||||
return nil
|
||||
// in any case we must update the object meta data
|
||||
return o.readMetaData(ctx, true)
|
||||
}
|
||||
|
||||
func (o *Object) remove(ctx context.Context, hard bool) error {
|
||||
|
|
@ -2013,6 +2076,22 @@ func (o *Object) Remove(ctx context.Context) error {
|
|||
return o.remove(ctx, o.fs.opt.HardDelete)
|
||||
}
|
||||
|
||||
// Metadata returns metadata for an object
|
||||
//
|
||||
// It should return nil if there is no Metadata
|
||||
func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
|
||||
err = o.readMetaData(ctx, false)
|
||||
if err != nil {
|
||||
fs.Logf(o, "Failed to read metadata: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
metadata.Set("btime", o.createTime.Format(time.RFC3339Nano)) // metadata timestamps should be RFC3339Nano
|
||||
metadata.Set("mtime", o.modTime.Format(time.RFC3339Nano))
|
||||
metadata.Set("utime", o.updateTime.Format(time.RFC3339Nano))
|
||||
metadata.Set("content-type", o.mimeType)
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
|
|
@ -2027,4 +2106,5 @@ var (
|
|||
_ fs.CleanUpper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.Metadataer = (*Object)(nil)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
package jottacloud
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
|
@ -40,3 +46,48 @@ func TestReadMD5(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestMetadata(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
contents := random.String(1000)
|
||||
|
||||
item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
utime := time.Now()
|
||||
metadata := fs.Metadata{
|
||||
"btime": "2009-05-06T04:05:06.499999999Z",
|
||||
"mtime": "2010-06-07T08:09:07.599999999Z",
|
||||
//"utime" - read-only
|
||||
//"content-type" - read-only
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, "text/html", metadata)
|
||||
defer func() {
|
||||
assert.NoError(t, obj.Remove(ctx))
|
||||
}()
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
for k, v := range metadata {
|
||||
got := gotMetadata[k]
|
||||
switch k {
|
||||
case "btime":
|
||||
assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got))
|
||||
case "mtime":
|
||||
assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got))
|
||||
case "utime":
|
||||
gotUtime := fstest.Time(got)
|
||||
dt := gotUtime.Sub(utime)
|
||||
assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("utime more than 1 minute out want %v got %v delta %v", utime, gotUtime, dt))
|
||||
assert.True(t, fstest.Time(v).Equal(fstest.Time(got)))
|
||||
case "content-type":
|
||||
assert.True(t, o.MimeType(ctx) == got)
|
||||
default:
|
||||
assert.Equal(t, v, got, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Metadata", f.InternalTestMetadata)
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
|
|
|||
|
|
@ -233,7 +233,7 @@ them. Generally you should avoid these, unless you know what you are doing.
|
|||
|
||||
### --fast-list
|
||||
|
||||
This remote supports `--fast-list` which allows you to use fewer
|
||||
This backend supports `--fast-list` which allows you to use fewer
|
||||
transactions in exchange for more memory. See the [rclone
|
||||
docs](/docs/#fast-list) for more details.
|
||||
|
||||
|
|
@ -241,8 +241,9 @@ Note that the implementation in Jottacloud always uses only a single
|
|||
API request to get the entire list, so for large folders this could
|
||||
lead to long wait time before the first results are shown.
|
||||
|
||||
Note also that with rclone version 1.58 and newer information about
|
||||
[MIME types](/overview/#mime-type) are not available when using `--fast-list`.
|
||||
Note also that with rclone version 1.58 and newer, information about
|
||||
[MIME types](/overview/#mime-type) and metadata item [utime](#metadata)
|
||||
are not available when using `--fast-list`.
|
||||
|
||||
### Modified time and hashes
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ Here is an overview of the major features of each cloud storage system.
|
|||
| HiDrive | HiDrive ¹² | R/W | No | No | - | - |
|
||||
| HTTP | - | R | No | No | R | - |
|
||||
| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU |
|
||||
| Jottacloud | MD5 | R/W | Yes | No | R | - |
|
||||
| Jottacloud | MD5 | R/W | Yes | No | R | RW |
|
||||
| Koofr | MD5 | - | Yes | No | - | - |
|
||||
| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | - |
|
||||
| Mega | - | - | No | Yes | - | - |
|
||||
|
|
|
|||
Loading…
Reference in a new issue