potyarkin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

backend/internetarchive: ignore checksums for files using the different method

Browse source
This commit is contained in:
Lesmiscore 2022-07-16 12:11:09 +09:00 • committed by Nick Craig-Wood
parent 50f053cada
commit 049674aeab
2 changed files with 71 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
backend/internetarchive/internetarchive.go
View file

@ -42,54 +42,70 @@ func init() {
MetadataInfo: &fs.MetadataInfo{ MetadataInfo: &fs.MetadataInfo{
System: map[string]fs.MetadataHelp{ System: map[string]fs.MetadataHelp{
"name": { "name": {
Help: "Full file path, without the bucket part", Help: "Full file path, without the bucket part",
Type: "filename", Type: "filename",
Example: "backend/internetarchive/internetarchive.go", Example: "backend/internetarchive/internetarchive.go",
ReadOnly: true,
}, },
"source": { "source": {
Help: "The source of the file", Help: "The source of the file",
Type: "string", Type: "string",
Example: "original", Example: "original",
ReadOnly: true,
}, },
"mtime": { "mtime": {
Help: "Time of last modification, managed by Rclone", Help: "Time of last modification, managed by Rclone",
Type: "RFC 3339", Type: "RFC 3339",
Example: "2006-01-02T15:04:05.999999999Z", Example: "2006-01-02T15:04:05.999999999Z",
ReadOnly: true,
}, },
"size": { "size": {
Help: "File size in bytes", Help: "File size in bytes",
Type: "decimal number", Type: "decimal number",
Example: "123456", Example: "123456",
ReadOnly: true,
}, },
"md5": { "md5": {
Help: "MD5 hash calculated by Internet Archive", Help: "MD5 hash calculated by Internet Archive",
Type: "string", Type: "string",
Example: "01234567012345670123456701234567", Example: "01234567012345670123456701234567",
ReadOnly: true,
}, },
"crc32": { "crc32": {
Help: "CRC32 calculated by Internet Archive", Help: "CRC32 calculated by Internet Archive",
Type: "string", Type: "string",
Example: "01234567", Example: "01234567",
ReadOnly: true,
}, },
"sha1": { "sha1": {
Help: "SHA1 hash calculated by Internet Archive", Help: "SHA1 hash calculated by Internet Archive",
Type: "string", Type: "string",
Example: "0123456701234567012345670123456701234567", Example: "0123456701234567012345670123456701234567",
ReadOnly: true,
}, },
"format": { "format": {
Help: "Name of format identified by Internet Archive", Help: "Name of format identified by Internet Archive",
Type: "string", Type: "string",
Example: "Comma-Separated Values", Example: "Comma-Separated Values",
ReadOnly: true,
}, },
"old_version": { "old_version": {
Help: "Whether the file was replaced and moved by keep-old-version flag", Help: "Whether the file was replaced and moved by keep-old-version flag",
Type: "boolean", Type: "boolean",
Example: "true", Example: "true",
ReadOnly: true,
}, },
"viruscheck": { "viruscheck": {
Help: "The last time viruscheck process was run for the file (?)", Help: "The last time viruscheck process was run for the file (?)",
Type: "unixtime", Type: "unixtime",
Example: "1654191352", Example: "1654191352",
ReadOnly: true,
},
"summation": {
Help: "Check https://forum.rclone.org/t/31922 for how it is used",
Type: "string",
Example: "md5",
ReadOnly: true,
}, },
"rclone-ia-mtime": { "rclone-ia-mtime": {
@ -173,7 +189,7 @@ var roMetadataKey = map[string]interface{}{
// do not add mtime here, it's a documented exception // do not add mtime here, it's a documented exception
"name": nil, "source": nil, "size": nil, "md5": nil, "name": nil, "source": nil, "size": nil, "md5": nil,
"crc32": nil, "sha1": nil, "format": nil, "old_version": nil, "crc32": nil, "sha1": nil, "format": nil, "old_version": nil,
"viruscheck": nil, "viruscheck": nil, "summation": nil,
} }
// Options defines the configuration for this backend // Options defines the configuration for this backend
@ -222,6 +238,7 @@ type IAFile struct {
Md5 string `json:"md5"` Md5 string `json:"md5"`
Crc32 string `json:"crc32"` Crc32 string `json:"crc32"`
Sha1 string `json:"sha1"` Sha1 string `json:"sha1"`
Summation string `json:"summation"`
rawData json.RawMessage rawData json.RawMessage
} }
@ -1135,16 +1152,21 @@ func (f *Fs) waitDelete(ctx context.Context, bucket, bucketPath string) (err err
} }
func makeValidObject(f *Fs, remote string, file IAFile, mtime time.Time, size int64) *Object { func makeValidObject(f *Fs, remote string, file IAFile, mtime time.Time, size int64) *Object {
return &Object{ ret := &Object{
fs: f, fs: f,
remote: remote, remote: remote,
modTime: mtime, modTime: mtime,
size: size, size: size,
md5: file.Md5,
crc32: file.Crc32,
sha1: file.Sha1,
rawData: file.rawData, rawData: file.rawData,
} }
// hashes from _files.xml (where summation != "") is different from one in other files
// https://forum.rclone.org/t/internet-archive-md5-tag-in-id-files-xml-interpreted-incorrectly/31922
if file.Summation == "" {
ret.md5 = file.Md5
ret.crc32 = file.Crc32
ret.sha1 = file.Sha1
}
return ret
} }
func makeValidObject2(f *Fs, file IAFile, bucket string) *Object { func makeValidObject2(f *Fs, file IAFile, bucket string) *Object {

27
docs/content/internetarchive.md
View file

@ -12,11 +12,10 @@ Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.htm
Paths are specified as `remote:bucket` (or `remote:` for the `lsd` Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`. command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`.
Once you have made a remote (see the provider specific section above)
you can use it like this:
Unlike S3, listing up all items uploaded by you isn't supported. Unlike S3, listing up all items uploaded by you isn't supported.
Once you have made a remote, you can use it like this:
Make a new item Make a new item
rclone mkdir remote:item rclone mkdir remote:item
@ -53,6 +52,7 @@ The following are reserved by Internet Archive:
- `format` - `format`
- `old_version` - `old_version`
- `viruscheck` - `viruscheck`
- `summation`
Trying to set values to these keys is ignored with a warning. Trying to set values to these keys is ignored with a warning.
Only setting `mtime` is an exception. Doing so make it the identical behavior as setting ModTime. Only setting `mtime` is an exception. Doing so make it the identical behavior as setting ModTime.
@ -258,19 +258,20 @@ Here are the possible system metadata items for the internetarchive backend.
| Name | Help | Type | Example | Read Only | | Name | Help | Type | Example | Read Only |
|------|------|------|---------|-----------| |------|------|------|---------|-----------|
| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | N | | crc32 | CRC32 calculated by Internet Archive | string | 01234567 | **Y** |
| format | Name of format identified by Internet Archive | string | Comma-Separated Values | N | | format | Name of format identified by Internet Archive | string | Comma-Separated Values | **Y** |
| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | N | | md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | **Y** |
| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | | mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | **Y** |
| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | N | | name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | **Y** |
| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | N | | old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | **Y** |
| rclone-ia-mtime | Time of last modification, managed by Internet Archive | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | | rclone-ia-mtime | Time of last modification, managed by Internet Archive | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N |
| rclone-mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | | rclone-mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N |
| rclone-update-track | Random value used by Rclone for tracking changes inside Internet Archive | string | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | N | | rclone-update-track | Random value used by Rclone for tracking changes inside Internet Archive | string | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | N |
| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | N | | sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | **Y** |
| size | File size in bytes | decimal number | 123456 | N | | size | File size in bytes | decimal number | 123456 | **Y** |
| source | The source of the file | string | original | N | | source | The source of the file | string | original | **Y** |
| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | N | | summation | Check https://forum.rclone.org/t/31922 for how it is used | string | md5 | **Y** |
| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | **Y** |
See the [metadata](/docs/#metadata) docs for more info. See the [metadata](/docs/#metadata) docs for more info.