forked from TrueCloudLab/rclone
jottacloud: add support for reading and writing metadata
Most useful is the addition of the file created timestamp, but also a timestamp for when the file was uploaded. Currently supporting a rather minimalistic set of metadata items, see PR #6359 for some thoughts about possible extensions.
This commit is contained in:
albertony
parent
b296f37801
commit
19ad39fa1c
5 changed files with 158 additions and 26 deletions
|
|
@ -92,6 +92,33 @@ func init() {
|
|||
Description: "Jottacloud",
|
||||
NewFs: NewFs,
|
||||
Config: Config,
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
Help: `Jottacloud has limited support for metadata, currently an extended set of timestamps.`,
|
||||
System: map[string]fs.MetadataHelp{
|
||||
"btime": {
|
||||
Help: "Time of file birth (creation), read from rclone metadata",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||
},
|
||||
"mtime": {
|
||||
Help: "Time of last modification, read from rclone metadata",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||
},
|
||||
"utime": {
|
||||
Help: "Time of last upload, when current revision was created, generated by backend",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05.999999999Z07:00",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"content-type": {
|
||||
Help: "MIME type, also known as media type",
|
||||
Type: "string",
|
||||
Example: "text/plain",
|
||||
ReadOnly: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "md5_memory_limit",
|
||||
Help: "Files bigger than this will be cached on disk to calculate the MD5 if required.",
|
||||
|
|
@ -497,7 +524,9 @@ type Object struct {
|
|||
remote string
|
||||
hasMetaData bool
|
||||
size int64
|
||||
createTime time.Time
|
||||
modTime time.Time
|
||||
updateTime time.Time
|
||||
md5 string
|
||||
mimeType string
|
||||
}
|
||||
|
|
@ -972,6 +1001,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
|||
CanHaveEmptyDirectories: true,
|
||||
ReadMimeType: true,
|
||||
WriteMimeType: false,
|
||||
ReadMetadata: true,
|
||||
WriteMetadata: true,
|
||||
UserMetadata: false,
|
||||
}).Fill(ctx, f)
|
||||
f.jfsSrv.SetErrorHandler(errorHandler)
|
||||
if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
|
||||
|
|
@ -1158,6 +1190,7 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback
|
|||
remote: filesystem.opt.Enc.ToStandardPath(path.Join(f.Path, f.Name)),
|
||||
size: f.Size,
|
||||
md5: f.Checksum,
|
||||
createTime: time.Time(f.Created),
|
||||
modTime: time.Time(f.Modified),
|
||||
})
|
||||
}
|
||||
|
|
@ -1387,7 +1420,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
|||
// is currently in trash, but can be made to match, it will be
|
||||
// restored. Returns ErrorObjectNotFound if upload will be necessary
|
||||
// to get a matching remote file.
|
||||
func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
|
||||
func (f *Fs) createOrUpdate(ctx context.Context, file string, createTime time.Time, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: f.filePath(file),
|
||||
|
|
@ -1397,11 +1430,10 @@ func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time,
|
|||
|
||||
opts.Parameters.Set("cphash", "true")
|
||||
|
||||
fileDate := api.JottaTime(modTime).String()
|
||||
opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
|
||||
opts.ExtraHeaders["JMd5"] = md5
|
||||
opts.ExtraHeaders["JCreated"] = fileDate
|
||||
opts.ExtraHeaders["JModified"] = fileDate
|
||||
opts.ExtraHeaders["JCreated"] = api.JottaTime(createTime).String()
|
||||
opts.ExtraHeaders["JModified"] = api.JottaTime(modTime).String()
|
||||
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
|
|
@ -1464,7 +1496,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
|||
// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
|
||||
if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
|
||||
fs.Debugf(src, "Server-side copied to trashed destination, restoring")
|
||||
info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
|
||||
info, err = f.createOrUpdate(ctx, remote, srcObj.createTime, srcObj.modTime, srcObj.size, srcObj.md5)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -1727,7 +1759,9 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) {
|
|||
o.size = info.Size
|
||||
o.md5 = info.MD5
|
||||
o.mimeType = info.MimeType
|
||||
o.createTime = time.Time(info.CreatedAt)
|
||||
o.modTime = time.Time(info.ModifiedAt)
|
||||
o.updateTime = time.Time(info.UpdatedAt)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -1772,7 +1806,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
|||
// (note that if size/md5 does not match, the file content will
|
||||
// also be modified if deduplication is possible, i.e. it is
|
||||
// important to use correct/latest values)
|
||||
_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
|
||||
_, err = o.fs.createOrUpdate(ctx, o.remote, o.createTime, modTime, o.size, o.md5)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
|
||||
|
|
@ -1909,6 +1943,37 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
// Wrap the accounting back onto the stream
|
||||
in = wrap(in)
|
||||
}
|
||||
// Fetch metadata if --metadata is in use
|
||||
meta, err := fs.GetMetadataOptions(ctx, src, options)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read metadata from source object: %w", err)
|
||||
}
|
||||
var createdTime string
|
||||
var modTime string
|
||||
if meta != nil {
|
||||
if v, ok := meta["btime"]; ok {
|
||||
t, err := time.Parse(time.RFC3339Nano, v) // metadata stores RFC3339Nano timestamps
|
||||
if err != nil {
|
||||
fs.Debugf(o, "failed to parse metadata btime: %q: %v", v, err)
|
||||
} else {
|
||||
createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
|
||||
}
|
||||
}
|
||||
if v, ok := meta["mtime"]; ok {
|
||||
t, err := time.Parse(time.RFC3339Nano, v)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "failed to parse metadata mtime: %q: %v", v, err)
|
||||
} else {
|
||||
modTime = api.Rfc3339Time(t).String()
|
||||
}
|
||||
}
|
||||
}
|
||||
if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime
|
||||
modTime = api.Rfc3339Time(src.ModTime(ctx)).String()
|
||||
}
|
||||
if createdTime == "" { // if no Created time set same as Modified
|
||||
createdTime = modTime
|
||||
}
|
||||
|
||||
// use the api to allocate the file first and get resume / deduplication info
|
||||
var resp *http.Response
|
||||
|
|
@ -1918,13 +1983,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
Options: options,
|
||||
ExtraHeaders: make(map[string]string),
|
||||
}
|
||||
fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
|
||||
|
||||
// the allocate request
|
||||
var request = api.AllocateFileRequest{
|
||||
Bytes: size,
|
||||
Created: fileDate,
|
||||
Modified: fileDate,
|
||||
Created: createdTime,
|
||||
Modified: modTime,
|
||||
Md5: md5String,
|
||||
Path: o.fs.allocatePathRaw(o.remote, true),
|
||||
}
|
||||
|
|
@ -1939,7 +2003,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
return err
|
||||
}
|
||||
|
||||
// If the file state is INCOMPLETE and CORRUPT, try to upload a then
|
||||
// If the file state is INCOMPLETE and CORRUPT, we must upload it.
|
||||
// Else, if the file state is COMPLETE, we don't need to upload it because
|
||||
// the content is already there, possibly it was created with deduplication,
|
||||
// and also any metadata changes are already performed by the allocate request.
|
||||
if response.State != "COMPLETED" {
|
||||
// how much do we still have to upload?
|
||||
remainingBytes := size - response.ResumePos
|
||||
|
|
@ -1963,22 +2030,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||
}
|
||||
|
||||
// send the remaining bytes
|
||||
resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
|
||||
_, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// finally update the meta data
|
||||
o.hasMetaData = true
|
||||
o.size = result.Bytes
|
||||
o.md5 = result.Md5
|
||||
o.modTime = time.Unix(result.Modified/1000, 0)
|
||||
} else {
|
||||
// If the file state is COMPLETE we don't need to upload it because the file was already found but we still need to update our metadata
|
||||
return o.readMetaData(ctx, true)
|
||||
// Upload response contains main metadata properties (size, md5 and modTime)
|
||||
// which could be set back to the object, but it does not contain the
|
||||
// necessary information to set the createTime and updateTime properties,
|
||||
// so must therefore perform a read instead.
|
||||
}
|
||||
|
||||
return nil
|
||||
// in any case we must update the object meta data
|
||||
return o.readMetaData(ctx, true)
|
||||
}
|
||||
|
||||
func (o *Object) remove(ctx context.Context, hard bool) error {
|
||||
|
|
@ -2013,6 +2076,22 @@ func (o *Object) Remove(ctx context.Context) error {
|
|||
return o.remove(ctx, o.fs.opt.HardDelete)
|
||||
}
|
||||
|
||||
// Metadata returns metadata for an object
|
||||
//
|
||||
// It should return nil if there is no Metadata
|
||||
func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
|
||||
err = o.readMetaData(ctx, false)
|
||||
if err != nil {
|
||||
fs.Logf(o, "Failed to read metadata: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
metadata.Set("btime", o.createTime.Format(time.RFC3339Nano)) // metadata timestamps should be RFC3339Nano
|
||||
metadata.Set("mtime", o.modTime.Format(time.RFC3339Nano))
|
||||
metadata.Set("utime", o.updateTime.Format(time.RFC3339Nano))
|
||||
metadata.Set("content-type", o.mimeType)
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
|
|
@ -2027,4 +2106,5 @@ var (
|
|||
_ fs.CleanUpper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.Metadataer = (*Object)(nil)
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue