jottacloud: add support for reading and writing metadata

Most useful is the addition of the file created timestamp, but also a timestamp for
when the file was uploaded.

Currently supporting a rather minimalistic set of metadata items, see PR #6359 for
some thoughts about possible extensions.

backend/jottacloud/api/types.go

@@ -395,7 +395,7 @@ type JottaFile struct {
 	State           string    `xml:"currentRevision>state"`
 	CreatedAt       JottaTime `xml:"currentRevision>created"`
 	ModifiedAt      JottaTime `xml:"currentRevision>modified"`
-	Updated         JottaTime `xml:"currentRevision>updated"`
+	UpdatedAt       JottaTime `xml:"currentRevision>updated"`
 	Size            int64     `xml:"currentRevision>size"`
 	MimeType        string    `xml:"currentRevision>mime"`
 	MD5             string    `xml:"currentRevision>md5"`

backend/jottacloud/jottacloud.go

@@ -92,6 +92,33 @@ func init() {
 		Description: "Jottacloud",
 		NewFs:       NewFs,
 		Config:      Config,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Jottacloud has limited support for metadata, currently an extended set of timestamps.`,
+			System: map[string]fs.MetadataHelp{
+				"btime": {
+					Help:    "Time of file birth (creation), read from rclone metadata",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z07:00",
+				},
+				"mtime": {
+					Help:    "Time of last modification, read from rclone metadata",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z07:00",
+				},
+				"utime": {
+					Help:     "Time of last upload, when current revision was created, generated by backend",
+					Type:     "RFC 3339",
+					Example:  "2006-01-02T15:04:05.999999999Z07:00",
+					ReadOnly: true,
+				},
+				"content-type": {
+					Help:     "MIME type, also known as media type",
+					Type:     "string",
+					Example:  "text/plain",
+					ReadOnly: true,
+				},
+			},
+		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
@@ -497,7 +524,9 @@ type Object struct {
 	remote      string
 	hasMetaData bool
 	size        int64
+	createTime  time.Time
 	modTime     time.Time
+	updateTime  time.Time
 	md5         string
 	mimeType    string
 }
@@ -972,6 +1001,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		CanHaveEmptyDirectories: true,
 		ReadMimeType:            true,
 		WriteMimeType:           false,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            false,
 	}).Fill(ctx, f)
 	f.jfsSrv.SetErrorHandler(errorHandler)
 	if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
@@ -1158,6 +1190,7 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback
 			remote:      filesystem.opt.Enc.ToStandardPath(path.Join(f.Path, f.Name)),
 			size:        f.Size,
 			md5:         f.Checksum,
+			createTime:  time.Time(f.Created),
 			modTime:     time.Time(f.Modified),
 		})
 	}
@@ -1387,7 +1420,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // is currently in trash, but can be made to match, it will be
 // restored. Returns ErrorObjectNotFound if upload will be necessary
 // to get a matching remote file.
-func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
+func (f *Fs) createOrUpdate(ctx context.Context, file string, createTime time.Time, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
 		Method:       "POST",
 		Path:         f.filePath(file),
@@ -1397,11 +1430,10 @@ func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time,
 
 	opts.Parameters.Set("cphash", "true")
 
-	fileDate := api.JottaTime(modTime).String()
 	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
 	opts.ExtraHeaders["JMd5"] = md5
-	opts.ExtraHeaders["JCreated"] = fileDate
+	opts.ExtraHeaders["JCreated"] = api.JottaTime(createTime).String()
-	opts.ExtraHeaders["JModified"] = fileDate
+	opts.ExtraHeaders["JModified"] = api.JottaTime(modTime).String()
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -1464,7 +1496,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
 	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
 		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
-		info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
+		info, err = f.createOrUpdate(ctx, remote, srcObj.createTime, srcObj.modTime, srcObj.size, srcObj.md5)
 	}
 
 	if err != nil {
@@ -1727,7 +1759,9 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	o.size = info.Size
 	o.md5 = info.MD5
 	o.mimeType = info.MimeType
+	o.createTime = time.Time(info.CreatedAt)
 	o.modTime = time.Time(info.ModifiedAt)
+	o.updateTime = time.Time(info.UpdatedAt)
 	return nil
 }
 
@@ -1772,7 +1806,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	// (note that if size/md5 does not match, the file content will
 	// also be modified if deduplication is possible, i.e. it is
 	// important to use correct/latest values)
-	_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
+	_, err = o.fs.createOrUpdate(ctx, o.remote, o.createTime, modTime, o.size, o.md5)
 	if err != nil {
 		if err == fs.ErrorObjectNotFound {
 			// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
@@ -1909,6 +1943,37 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// Wrap the accounting back onto the stream
 		in = wrap(in)
 	}
+	// Fetch metadata if --metadata is in use
+	meta, err := fs.GetMetadataOptions(ctx, src, options)
+	if err != nil {
+		return fmt.Errorf("failed to read metadata from source object: %w", err)
+	}
+	var createdTime string
+	var modTime string
+	if meta != nil {
+		if v, ok := meta["btime"]; ok {
+			t, err := time.Parse(time.RFC3339Nano, v) // metadata stores RFC3339Nano timestamps
+			if err != nil {
+				fs.Debugf(o, "failed to parse metadata btime: %q: %v", v, err)
+			} else {
+				createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
+			}
+		}
+		if v, ok := meta["mtime"]; ok {
+			t, err := time.Parse(time.RFC3339Nano, v)
+			if err != nil {
+				fs.Debugf(o, "failed to parse metadata mtime: %q: %v", v, err)
+			} else {
+				modTime = api.Rfc3339Time(t).String()
+			}
+		}
+	}
+	if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime
+		modTime = api.Rfc3339Time(src.ModTime(ctx)).String()
+	}
+	if createdTime == "" { // if no Created time set same as Modified
+		createdTime = modTime
+	}
 
 	// use the api to allocate the file first and get resume / deduplication info
 	var resp *http.Response
@@ -1918,13 +1983,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Options:      options,
 		ExtraHeaders: make(map[string]string),
 	}
-	fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
 
 	// the allocate request
 	var request = api.AllocateFileRequest{
 		Bytes:    size,
-		Created:  fileDate,
+		Created:  createdTime,
-		Modified: fileDate,
+		Modified: modTime,
 		Md5:      md5String,
 		Path:     o.fs.allocatePathRaw(o.remote, true),
 	}
@@ -1939,7 +2003,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	// If the file state is INCOMPLETE and CORRUPT, try to upload a then
+	// If the file state is INCOMPLETE and CORRUPT, we must upload it.
+	// Else, if the file state is COMPLETE, we don't need to upload it because
+	// the content is already there, possibly it was created with deduplication,
+	// and also any metadata changes are already performed by the allocate request.
 	if response.State != "COMPLETED" {
 		// how much do we still have to upload?
 		remainingBytes := size - response.ResumePos
@@ -1963,22 +2030,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 
 		// send the remaining bytes
-		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
+		_, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
 		if err != nil {
 			return err
 		}
 
-		// finally update the meta data
+		// Upload response contains main metadata properties (size, md5 and modTime)
-		o.hasMetaData = true
+		// which could be set back to the object, but it does not contain the
-		o.size = result.Bytes
+		// necessary information to set the createTime and updateTime properties,
-		o.md5 = result.Md5
+		// so must therefore perform a read instead.
-		o.modTime = time.Unix(result.Modified/1000, 0)
-	} else {
-		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still need to update our metadata
-		return o.readMetaData(ctx, true)
 	}
-
+	// in any case we must update the object meta data
-	return nil
+	return o.readMetaData(ctx, true)
 }
 
 func (o *Object) remove(ctx context.Context, hard bool) error {
@@ -2013,6 +2076,22 @@ func (o *Object) Remove(ctx context.Context) error {
 	return o.remove(ctx, o.fs.opt.HardDelete)
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+	err = o.readMetaData(ctx, false)
+	if err != nil {
+		fs.Logf(o, "Failed to read metadata: %v", err)
+		return nil, err
+	}
+	metadata.Set("btime", o.createTime.Format(time.RFC3339Nano)) // metadata timestamps should be RFC3339Nano
+	metadata.Set("mtime", o.modTime.Format(time.RFC3339Nano))
+	metadata.Set("utime", o.updateTime.Format(time.RFC3339Nano))
+	metadata.Set("content-type", o.mimeType)
+	return metadata, nil
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs           = (*Fs)(nil)
@@ -2027,4 +2106,5 @@ var (
 	_ fs.CleanUpper   = (*Fs)(nil)
 	_ fs.Object       = (*Object)(nil)
 	_ fs.MimeTyper    = (*Object)(nil)
+	_ fs.Metadataer   = (*Object)(nil)
 )

backend/jottacloud/jottacloud_internal_test.go

@@ -1,11 +1,17 @@
 package jottacloud
 
 import (
+	"context"
 	"crypto/md5"
 	"fmt"
 	"io"
 	"testing"
+	"time"
 
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,3 +46,48 @@ func TestReadMD5(t *testing.T) {
 		})
 	}
 }
+
+func (f *Fs) InternalTestMetadata(t *testing.T) {
+	ctx := context.Background()
+	contents := random.String(1000)
+
+	item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+	utime := time.Now()
+	metadata := fs.Metadata{
+		"btime": "2009-05-06T04:05:06.499999999Z",
+		"mtime": "2010-06-07T08:09:07.599999999Z",
+		//"utime" - read-only
+		//"content-type" - read-only
+	}
+	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, "text/html", metadata)
+	defer func() {
+		assert.NoError(t, obj.Remove(ctx))
+	}()
+	o := obj.(*Object)
+	gotMetadata, err := o.Metadata(ctx)
+	require.NoError(t, err)
+	for k, v := range metadata {
+		got := gotMetadata[k]
+		switch k {
+		case "btime":
+			assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got))
+		case "mtime":
+			assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got))
+		case "utime":
+			gotUtime := fstest.Time(got)
+			dt := gotUtime.Sub(utime)
+			assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("utime more than 1 minute out want %v got %v delta %v", utime, gotUtime, dt))
+			assert.True(t, fstest.Time(v).Equal(fstest.Time(got)))
+		case "content-type":
+			assert.True(t, o.MimeType(ctx) == got)
+		default:
+			assert.Equal(t, v, got, k)
+		}
+	}
+}
+
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("Metadata", f.InternalTestMetadata)
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)

docs/content/jottacloud.md

@@ -233,7 +233,7 @@ them. Generally you should avoid these, unless you know what you are doing.
 
 ### --fast-list
 
-This remote supports `--fast-list` which allows you to use fewer
+This backend supports `--fast-list` which allows you to use fewer
 transactions in exchange for more memory. See the [rclone
 docs](/docs/#fast-list) for more details.
 
@@ -241,8 +241,9 @@ Note that the implementation in Jottacloud always uses only a single
 API request to get the entire list, so for large folders this could
 lead to long wait time before the first results are shown.
 
-Note also that with rclone version 1.58 and newer information about
+Note also that with rclone version 1.58 and newer, information about
-[MIME types](/overview/#mime-type) are not available when using `--fast-list`.
+[MIME types](/overview/#mime-type) and metadata item [utime](#metadata)
+are not available when using `--fast-list`.
 
 ### Modified time and hashes
 

docs/content/overview.md

@@ -33,7 +33,7 @@ Here is an overview of the major features of each cloud storage system.
 | HiDrive                      | HiDrive ¹²        | R/W     | No               | No              | -         | -        |
 | HTTP                         | -                 | R       | No               | No              | R         | -        |
 | Internet Archive             | MD5, SHA1, CRC32  | R/W ¹¹  | No               | No              | -         | RWU      |
-| Jottacloud                   | MD5               | R/W     | Yes              | No              | R         | -        |
+| Jottacloud                   | MD5               | R/W     | Yes              | No              | R         | RW       |
 | Koofr                        | MD5               | -       | Yes              | No              | -         | -        |
 | Mail.ru Cloud                | Mailru ⁶          | R/W     | Yes              | No              | -         | -        |
 | Mega                         | -                 | -       | No               | Yes             | -         | -        |