archive backend - WIP FIXME

- Tests all passing.
- Zip and squashfs archivers working.
- Lazily loads and caches squashfs

Squashfs
- see archive.go for more FIXMEs

.forgejo/ISSUE_TEMPLATE/bug_report.md

@@ -1,45 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: community, triage, bug
-assignees: ''
-
----
-
-<!--- Provide a general summary of the issue in the Title above -->
-
-## Expected Behavior
-<!--- If you're describing a bug, tell us what should happen -->
-<!--- If you're suggesting a change/improvement, tell us how it should work -->
-
-## Current Behavior
-<!--- If describing a bug, tell us what happens instead of the expected behavior -->
-<!--- If suggesting a change/improvement, explain the difference from current behavior -->
-
-## Possible Solution
-<!--- Not obligatory -->
-<!--- If no reason/fix/additions for the bug can be suggested, -->
-<!--- uncomment the following phrase: -->
-
-<!--- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
-
-## Steps to Reproduce (for bugs)
-<!--- Provide a link to a live example, or an unambiguous set of steps to -->
-<!--- reproduce this bug. -->
-
-1.
-
-## Context
-<!--- How has this issue affected you? What are you trying to accomplish? -->
-<!--- Providing context helps us come up with a solution that is most useful in the real world -->
-
-## Regression
-<!-- Is this issue a regression? (Yes / No) -->
-<!-- If Yes, optionally please include version or commit id or PR# that caused this regression, if you have these details. -->
-
-## Your Environment
-<!--- Include as many relevant details about the environment you experienced the bug in -->
-* Version used:
-* Server setup and configuration:
-* Operating System and version (`uname -a`):

.forgejo/ISSUE_TEMPLATE/config.yml

@@ -1 +0,0 @@
-blank_issues_enabled: false

.forgejo/workflows/builds.yaml

@@ -1,24 +0,0 @@
-on:
-  pull_request:
-  push:
-    branches:
-      - tcl/master
-
-jobs:
-  builds:
-    name: Builds
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go_versions: [ '1.22', '1.23' ]
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '${{ matrix.go_versions }}'
-
-      - name: Build binary
-        run: make

.forgejo/workflows/dco.yml

@@ -1,20 +0,0 @@
-on: [pull_request]
-
-jobs:
-  dco:
-    name: DCO
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-         fetch-depth: 0
-
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.23'
-
-      - name: Run commit format checker
-        uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3
-        with:
-          from: 'origin/${{ github.event.pull_request.base.ref }}'

.forgejo/workflows/tests.yml

@@ -1,67 +0,0 @@
-on:
-  pull_request:
-  push:
-    branches:
-      - tcl/master
-
-jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.23'
-          cache: true
-
-      - name: Install linters
-        run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
-
-      - name: Run linters
-        run: make check
-  test:
-    name: Test
-    runs-on: oci-runner
-    strategy:
-      matrix:
-        go_versions: [ '1.23' ]
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '${{ matrix.go_versions }}'
-
-      - name: Tests for the FrostFS backend
-        env:
-          RESTIC_TEST_FUSE: false
-          AIO_IMAGE: truecloudlab/frostfs-aio
-          AIO_VERSION: 1.7.0-nightly.4
-          RCLONE_CONFIG: /config/rclone.conf
-
-        # run only tests related to FrostFS backend
-        run: |-
-          podman-service.sh
-          podman info
-          
-          mkdir /config
-          printf "[TestFrostFS]\ntype = frostfs\nendpoint = localhost:8080\nwallet = /config/wallet.json\nplacement_policy = REP 1\nrequest_timeout = 20s\nconnection_timeout = 21s" > /config/rclone.conf
-          
-          echo "Run frostfs aio container"
-          docker run -d --net=host --name aio $AIO_IMAGE:$AIO_VERSION --restart always -p 8080:8080
-                            
-          echo "Wait for frostfs to start"
-          until docker exec aio curl --fail http://localhost:8083 > /dev/null 2>&1; do sleep 0.2; done;
-          
-          echo "Issue creds"
-          docker exec aio /usr/bin/issue-creds.sh native
-          echo "Copy wallet"
-          docker cp aio:/config/user-wallet.json /config/wallet.json
-
-          echo "Start tests"
-          go test -v github.com/rclone/rclone/backend/frostfs

.gitattributes

@@ -1,7 +1,3 @@
-# Go writes go.mod and go.sum with lf even on windows
-go.mod text eol=lf
-go.sum text eol=lf
-
 # Ignore generated files in GitHub language statistics and diffs
 /MANUAL.* linguist-generated=true
 /rclone.1 linguist-generated=true

.github/workflows/build.yml

@@ -27,12 +27,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.19', 'go1.20']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '1.21'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -43,14 +43,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '1.21'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
-            os: macos-latest
-            go: '>=1.23.0-rc.1'
+            os: macos-11
+            go: '1.21'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -58,15 +58,15 @@ jobs:
             deploy: true
 
           - job_name: mac_arm64
-            os: macos-latest
-            go: '>=1.23.0-rc.1'
+            os: macos-11
+            go: '1.21'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '>=1.23.0-rc.1'
+            go: '1.21'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -76,20 +76,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
+            go: '1.21'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.21
+          - job_name: go1.19
             os: ubuntu-latest
-            go: '1.21'
+            go: '1.19'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.22
+          - job_name: go1.20
             os: ubuntu-latest
-            go: '1.22'
+            go: '1.20'
             quicktest: true
             racequicktest: true
 
@@ -124,7 +124,7 @@ jobs:
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
           sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get install fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
+          sudo apt-get install fuse3 libfuse-dev rpm pkg-config
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
@@ -137,8 +137,7 @@ jobs:
           brew untap --force homebrew/cask
           brew update
           brew install --cask macfuse
-          brew install git-annex git-annex-remote-rclone
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-11'
 
       - name: Install Libraries on Windows
         shell: powershell
@@ -168,6 +167,14 @@ jobs:
           printf "\n\nSystem environment:\n\n"
           env
 
+      - name: Go module cache
+        uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
       - name: Build rclone
         shell: bash
         run: |
@@ -223,71 +230,21 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Get runner parameters
-        id: get-runner-parameters
-        shell: bash
-        run: |
-          echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT
-          echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
-
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Code quality test
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          version: latest
+
+      # Run govulncheck on the latest go version, the one we build binaries with
       - name: Install Go
-        id: setup-go
         uses: actions/setup-go@v5
         with:
-          go-version: '>=1.23.0-rc.1'
+          go-version: '1.21'
           check-latest: true
-          cache: false
-
-      - name: Cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/go/pkg/mod
-            ~/.cache/go-build
-            ~/.cache/golangci-lint
-          key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
-          restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
-
-      - name: Code quality test (Linux)
-        uses: golangci/golangci-lint-action@v6
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (Windows)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "windows"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (macOS)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "darwin"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (FreeBSD)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "freebsd"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (OpenBSD)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "openbsd"
-        with:
-          version: latest
-          skip-cache: true
 
       - name: Install govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -311,7 +268,15 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '>=1.23.0-rc.1'
+          go-version: '1.21'
+
+      - name: Go module cache
+        uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
 
       - name: Set global environment variables
         shell: bash

.github/workflows/build_publish_beta_docker_image.yml

@@ -56,7 +56,7 @@ jobs:
               run: |
                 df -h .
             - name: Build and publish image
-              uses: docker/build-push-action@v6
+              uses: docker/build-push-action@v5
               with:
                 file: Dockerfile
                 context: .

.github/workflows/build_publish_release_docker_image.yml

@@ -32,27 +32,15 @@ jobs:
             - name: Get actual major version
               id: actual_major_version
               run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKER_HUB_USER }}
-                password: ${{ secrets.DOCKER_HUB_PASSWORD }}
             - name: Build and publish image
-              uses: docker/build-push-action@v6
+              uses: ilteoood/docker_buildx@1.1.0
               with:
-                file: Dockerfile
-                context: .
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                push: true
-                tags: |
-                  rclone/rclone:latest
-                  rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }}
-                  rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }}
-                  rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
+                tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
     build_docker_volume_plugin:
         if: github.repository == 'rclone/rclone'

.github/workflows/notify.yml

@@ -1,15 +0,0 @@
-name: Notify users based on issue labels
-
-on:
-  issues:
-      types: [labeled]
-
-jobs:
-  notify:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: jenschelkopf/issue-label-notification-action@1.3
-          with:
-             token: ${{ secrets.NOTIFY_ACTION_TOKEN }}
-             recipients: |
-                  Support Contract=@rclone/support

.github/workflows/winget.yml

@@ -1,14 +1,14 @@
-name: Publish to Winget
-on:
-  release:
-    types: [released]
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: vedantmgoyal2009/winget-releaser@v2
-        with:
-          identifier: Rclone.Rclone
-          installers-regex: '-windows-\w+\.zip$'
-          token: ${{ secrets.WINGET_TOKEN }}
+name: Publish to Winget
+on:
+  release:
+    types: [released]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: vedantmgoyal2009/winget-releaser@v2
+        with:
+          identifier: Rclone.Rclone
+          installers-regex: '-windows-\w+\.zip$'
+          token: ${{ secrets.WINGET_TOKEN }}

.gitignore

@@ -3,13 +3,10 @@ _junk/
 rclone
 rclone.exe
 build
-/docs/public/
-/docs/.hugo_build.lock
-/docs/static/img/logos/
+docs/public
 rclone.iml
 .idea
 .history
-.vscode
 *.test
 *.iml
 fuzz-build.zip
@@ -18,5 +15,6 @@ fuzz-build.zip
 Thumbs.db
 __pycache__
 .DS_Store
+/docs/static/img/logos/
 resource_windows_*.syso
 .devcontainer

.golangci.yml

@@ -13,7 +13,6 @@ linters:
     - stylecheck
     - unused
     - misspell
-    - gocritic
     #- prealloc
     #- maligned
   disable-all: true
@@ -99,46 +98,3 @@ linters-settings:
     # Only enable the checks performed by the staticcheck stand-alone tool,
     # as documented here: https://staticcheck.io/docs/configuration/options/#checks
     checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]
-  gocritic:
-    # Enable all default checks with some exceptions and some additions (commented).
-    # Cannot use both enabled-checks and disabled-checks, so must specify all to be used.
-    disable-all: true
-    enabled-checks:
-      #- appendAssign # Enabled by default
-      - argOrder
-      - assignOp
-      - badCall
-      - badCond
-      #- captLocal # Enabled by default
-      - caseOrder
-      - codegenComment
-      #- commentFormatting # Enabled by default
-      - defaultCaseOrder
-      - deprecatedComment
-      - dupArg
-      - dupBranchBody
-      - dupCase
-      - dupSubExpr
-      - elseif
-      #- exitAfterDefer # Enabled by default
-      - flagDeref
-      - flagName
-      #- ifElseChain # Enabled by default
-      - mapKey
-      - newDeref
-      - offBy1
-      - regexpMust
-      - ruleguard # Not enabled by default
-      #- singleCaseSwitch # Enabled by default
-      - sloppyLen
-      - sloppyTypeAssert
-      - switchTrue
-      - typeSwitchVar
-      - underef
-      - unlambda
-      - unslice
-      - valSwap
-      - wrapperFunc
-    settings:
-      ruleguard:
-        rules: "${configDir}/bin/rules.go"

CONTRIBUTING.md

@@ -209,7 +209,7 @@ altogether with an HTML report and test retries then from the
 project root:
 
     go install github.com/rclone/rclone/fstest/test_all
-    test_all -backends drive
+    test_all -backend drive
 
 ### Full integration testing
 
@@ -508,7 +508,7 @@ You'll need to modify the following files
 - `backend/s3/s3.go`
     - Add the provider to `providerOption` at the top of the file
     - Add endpoints and other config for your provider gated on the provider in `fs.RegInfo`.
-    - Exclude your provider from generic config questions (eg `region` and `endpoint).
+    - Exclude your provider from genric config questions (eg `region` and `endpoint).
     - Add the provider to the `setQuirks` function - see the documentation there.
 - `docs/content/s3.md`
     - Add the provider at the top of the page.

MAINTAINERS.md

@@ -21,8 +21,6 @@ Current active maintainers of rclone are:
 | Chun-Hung Tseng  | @henrybear327     | Proton Drive Backend         |
 | Hideo Aoyama     | @boukendesho      | snap packaging               |
 | nielash          | @nielash          | bisync                       |
-| Dan McArdle      | @dmcardle         | gitannex                     |
-| Sam Harrison     | @childish-sambino | filescom                     |
 
 **This is a work in progress Draft**
 

Makefile

@@ -36,14 +36,13 @@ ifdef BETA_SUBDIR
 endif
 BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
-BETA_UPLOAD_ROOT := beta.rclone.org:
+BETA_UPLOAD_ROOT := memstore:beta-rclone-org
 BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
 LINTTAGS=--build-tags "$(GOTAGS)"
 endif
-LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
 
 .PHONY: rclone test_all vars version
 
@@ -51,7 +50,7 @@ rclone:
 ifeq ($(GO_OS),windows)
 	go run bin/resource_windows.go -version $(TAG) -syso resource_windows_`go env GOARCH`.syso
 endif
-	go build -v $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS)
+	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS)
 ifeq ($(GO_OS),windows)
 	rm resource_windows_`go env GOARCH`.syso
 endif
@@ -60,7 +59,7 @@ endif
 	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
 
 test_all:
-	go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
+	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
 
 vars:
 	@echo SHELL="'$(SHELL)'"
@@ -88,13 +87,13 @@ test:	rclone test_all
 
 # Quick test
 quicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) ./...
 
 racequicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
 
 compiletest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./...
 
 # Do source code quality checks
 check:	rclone
@@ -104,7 +103,7 @@ check:	rclone
 
 # Get the build dependencies
 build_dep:
-	go run bin/get-github-release.go -use-api -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
 
 # Get the release dependencies we only install on linux
 release_dep_linux:
@@ -168,7 +167,7 @@ website:
 	@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
 
 upload_website:	website
-	rclone -v sync docs/public www.rclone.org:
+	rclone -v sync docs/public memstore:www-rclone-org
 
 upload_test_website:	website
 	rclone -P sync docs/public test-rclone-org:
@@ -195,8 +194,8 @@ check_sign:
 	cd build && gpg --verify SHA256SUMS && gpg --decrypt SHA256SUMS | sha256sum -c
 
 upload:
-	rclone -P copy build/ downloads.rclone.org:/$(TAG)
-	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
+	rclone -P copy build/ memstore:downloads-rclone-org/$(TAG)
+	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "memstore:downloads-rclone-org/$(TAG)/$$i" "memstore:downloads-rclone-org/$$j"'
 
 upload_github:
 	./bin/upload-github $(TAG)
@@ -206,7 +205,7 @@ cross:	doc
 
 beta:
 	go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-	rclone -v copy build/ pub.rclone.org:/$(TAG)
+	rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
 	@echo Beta release ready at https://pub.rclone.org/$(TAG)/
 
 log_since_last_release:
@@ -219,18 +218,18 @@ ci_upload:
 	sudo chown -R $$USER build
 	find build -type l -delete
 	gzip -r9v build
-	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
+	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
+	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
 endif
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
 ci_beta:
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
@@ -239,7 +238,7 @@ fetch_binaries:
 	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
-	cd docs && hugo server --logLevel info -w --disableFastRender
+	cd docs && hugo server -v -w --disableFastRender
 
 tag:	retag doc
 	bin/make_changelog.py $(LAST_TAG) $(VERSION) > docs/content/changelog.md.new

README.md

@@ -1,21 +1,3 @@
-<div align="center">
-<sup>Special thanks to our sponsor:</sup>
-<br>
-<br>
-<a href="https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103">
-  <div>
-    <img src="https://rclone.org/img/logos/warp-github.svg" width="300" alt="Warp">
-  </div>
-  <b>Warp is a modern, Rust-based terminal with AI built in so you and your team can build great software, faster.</b>
-  <div>
-    <sup>Visit warp.dev to learn more.</sup>
-  </div>
-</a>
-<br>
-<hr>
-</div>
-<br>
-
 [<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-light-mode-only)
 [<img src="https://rclone.org/img/logo_on_dark__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-dark-mode-only)
 
@@ -41,6 +23,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
   * Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
+  * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
   * ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
@@ -55,18 +38,14 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
   * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
-  * Files.com [:page_facing_up:](https://rclone.org/filescom/)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
-  * GoFile [:page_facing_up:](https://rclone.org/gofile/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
   * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
-  * Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
   * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
   * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
-  * ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
   * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
@@ -76,7 +55,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
   * Linkbox [:page_facing_up:](https://rclone.org/linkbox)
   * Linode Object Storage [:page_facing_up:](https://rclone.org/s3/#linode)
-  * Magalu Object Storage [:page_facing_up:](https://rclone.org/s3/#magalu)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
@@ -96,7 +74,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * Petabox [:page_facing_up:](https://rclone.org/s3/#petabox)
   * PikPak [:page_facing_up:](https://rclone.org/pikpak/)
-  * Pixeldrain [:page_facing_up:](https://rclone.org/pixeldrain/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
   * Proton Drive [:page_facing_up:](https://rclone.org/protondrive/)
@@ -105,7 +82,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
-  * rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
   * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
@@ -116,7 +92,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
   * Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
   * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
-  * Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
@@ -145,7 +120,6 @@ These backends adapt or modify other storage providers
   * Partial syncs supported on a whole file basis
   * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
-  * [Bisync](https://rclone.org/bisync/) (two way) to keep two directories in sync bidirectionally
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
   * Can sync to and from network, e.g. two different cloud accounts
   * Optional large file chunking ([Chunker](https://rclone.org/chunker/))

RELEASE.md

@@ -37,44 +37,18 @@ This file describes how to make the various kinds of releases
 
 ## Update dependencies
 
-Early in the next release cycle update the dependencies.
+Early in the next release cycle update the dependencies
 
   * Review any pinned packages in go.mod and remove if possible
-  * `make updatedirect`
-  * `make GOTAGS=cmount`
-  * `make compiletest`
-  * Fix anything which doesn't compile at this point and commit changes here
-  * `git commit -a -v -m "build: update all dependencies"`
-
-If the `make updatedirect` upgrades the version of go in the `go.mod`
-then go to manual mode. `go1.20` here is the lowest supported version
-in the `go.mod`.
-
-```
-go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
-go get -d $(cat /tmp/potential-upgrades)
-go mod tidy -go=1.20 -compat=1.20
-```
-
-If the `go mod tidy` fails use the output from it to remove the
-package which can't be upgraded from `/tmp/potential-upgrades` when
-done
-
-```
-git co go.mod go.sum
-```
-
-And try again.
-
-Optionally upgrade the direct and indirect dependencies. This is very
-likely to fail if the manual method was used abve - in that case
-ignore it as it is too time consuming to fix.
-
-  * `make update`
-  * `make GOTAGS=cmount`
-  * `make compiletest`
+  * make updatedirect
+  * make GOTAGS=cmount
+  * make compiletest
+  * git commit -a -v
+  * make update
+  * make GOTAGS=cmount
+  * make compiletest
   * roll back any updates which didn't compile
-  * `git commit -a -v --amend`
+  * git commit -a -v --amend
   * **NB** watch out for this changing the default go version in `go.mod`
 
 Note that `make update` updates all direct and indirect dependencies
@@ -83,9 +57,6 @@ doing that so it may be necessary to roll back dependencies to the
 version specified by `make updatedirect` in order to get rclone to
 build.
 
-Once it compiles locally, push it on a test branch and commit fixes
-until the tests pass.
-
 ## Tidy beta
 
 At some point after the release run
@@ -168,8 +139,6 @@ docker buildx build -t rclone/rclone:testing --progress=plain --platform linux/a
 
 To make a full build then set the tags correctly and add `--push`
 
-Note that you can't only build one architecture - you need to build them all.
-
 ```
-docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
+docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
 ```

VERSION

@@ -1 +1 @@
-v1.68.2
+v1.66.0

backend/alias/alias_internal_test.go

@@ -23,8 +23,8 @@ func prepare(t *testing.T, root string) {
 	configfile.Install()
 
 	// Configure the remote
-	config.FileSetValue(remoteName, "type", "alias")
-	config.FileSetValue(remoteName, "remote", root)
+	config.FileSet(remoteName, "type", "alias")
+	config.FileSet(remoteName, "remote", root)
 }
 
 func TestNewFS(t *testing.T) {
@@ -81,12 +81,10 @@ func TestNewFS(t *testing.T) {
 		for i, gotEntry := range gotEntries {
 			what := fmt.Sprintf("%s, entry=%d", what, i)
 			wantEntry := test.entries[i]
-			_, isDir := gotEntry.(fs.Directory)
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			if !isDir {
-				require.Equal(t, wantEntry.size, gotEntry.Size(), what)
-			}
+			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
+			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}
 	}

backend/all/all.go

@@ -4,6 +4,7 @@ package all
 import (
 	// Active file systems
 	_ "github.com/rclone/rclone/backend/alias"
+	_ "github.com/rclone/rclone/backend/amazonclouddrive"
 	_ "github.com/rclone/rclone/backend/azureblob"
 	_ "github.com/rclone/rclone/backend/azurefiles"
 	_ "github.com/rclone/rclone/backend/b2"
@@ -17,10 +18,7 @@ import (
 	_ "github.com/rclone/rclone/backend/dropbox"
 	_ "github.com/rclone/rclone/backend/fichier"
 	_ "github.com/rclone/rclone/backend/filefabric"
-	_ "github.com/rclone/rclone/backend/filescom"
-	_ "github.com/rclone/rclone/backend/frostfs"
 	_ "github.com/rclone/rclone/backend/ftp"
-	_ "github.com/rclone/rclone/backend/gofile"
 	_ "github.com/rclone/rclone/backend/googlecloudstorage"
 	_ "github.com/rclone/rclone/backend/googlephotos"
 	_ "github.com/rclone/rclone/backend/hasher"
@@ -42,7 +40,6 @@ import (
 	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
 	_ "github.com/rclone/rclone/backend/pikpak"
-	_ "github.com/rclone/rclone/backend/pixeldrain"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
 	_ "github.com/rclone/rclone/backend/protondrive"
 	_ "github.com/rclone/rclone/backend/putio"
@@ -57,7 +54,6 @@ import (
 	_ "github.com/rclone/rclone/backend/storj"
 	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"
-	_ "github.com/rclone/rclone/backend/ulozto"
 	_ "github.com/rclone/rclone/backend/union"
 	_ "github.com/rclone/rclone/backend/uptobox"
 	_ "github.com/rclone/rclone/backend/webdav"

backend/amazonclouddrive/amazonclouddrive_test.go

@@ -0,0 +1,21 @@
+// Test AmazonCloudDrive filesystem interface
+
+//go:build acd
+// +build acd
+
+package amazonclouddrive_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/amazonclouddrive"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.NilObject = fs.Object((*amazonclouddrive.Object)(nil))
+	fstests.RemoteName = "TestAmazonCloudDrive:"
+	fstests.Run(t)
+}

backend/archive/archive.go

@@ -0,0 +1,677 @@
+// Package archive implements a backend to archive multiple remotes in a directory tree
+package archive
+
+// FIXME factor common code between backends out - eg VFS initialization
+
+// FIXME can we generalize the VFS handle caching and use it in zip backend
+
+// Factor more stuff out if possible
+
+// Odd stats which are probably coming from the VFS
+// *  tensorflow.sqfs:  0% /3.074Gi, 204.426Ki/s, 4h22m46s
+
+// FIXME this will perform poorly for unpacking as the VFS Reader is bad
+// at multiple streams - need cache mode setting?
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	// Import all the required archivers here
+	_ "github.com/rclone/rclone/backend/archive/squashfs"
+	_ "github.com/rclone/rclone/backend/archive/zip"
+
+	"github.com/rclone/rclone/backend/archive/archiver"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/fspath"
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// Register with Fs
+func init() {
+	fsi := &fs.RegInfo{
+		Name:        "archive",
+		Description: "Read archives",
+		NewFs:       NewFs,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Any metadata supported by the underlying remote is read and written.`,
+		},
+		Options: []fs.Option{{
+			Name: "remote",
+			Help: `Remote to wrap to read archives from.
+
+Normally should contain a ':' and a path, e.g. "myremote:path/to/dir",
+"myremote:bucket" or "myremote:".
+
+If this is left empty, then the archive backend will use the root as
+the remote.
+
+This means that you can use :archive:remote:path and it will be
+equivalent to setting remote="remote:path".
+`,
+			Required: false,
+		}},
+	}
+	fs.Register(fsi)
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	Remote string `config:"remote"`
+}
+
+// Fs represents a archive of upstreams
+type Fs struct {
+	name     string       // name of this remote
+	features *fs.Features // optional features
+	opt      Options      // options for this Fs
+	root     string       // the path we are working on
+	f        fs.Fs        // remote we are wrapping
+	wrapper  fs.Fs        // fs that wraps us
+
+	mu       sync.Mutex          // protects the below
+	archives map[string]*archive // the archives we have, by path
+}
+
+// A single open archive
+type archive struct {
+	archiver archiver.Archiver // archiver responsible
+	remote   string            // path to the archive
+	prefix   string            // prefix to add on to listings
+	root     string            // root of the archive to remove from listings
+	mu       sync.Mutex        // protects the following variables
+	f        fs.Fs             // the archive Fs, may be nil
+}
+
+// If remote is an archive then return it otherwise return nil
+func findArchive(remote string) *archive {
+	// FIXME use something faster than linear search?
+	for _, archiver := range archiver.Archivers {
+		if strings.HasSuffix(remote, archiver.Extension) {
+			return &archive{
+				archiver: archiver,
+				remote:   remote,
+				prefix:   remote,
+				root:     "",
+			}
+		}
+	}
+	return nil
+}
+
+// Find an archive buried in remote
+func subArchive(remote string) *archive {
+	archive := findArchive(remote)
+	if archive != nil {
+		return archive
+	}
+	parent := path.Dir(remote)
+	if parent == "/" || parent == "." {
+		return nil
+	}
+	return subArchive(parent)
+}
+
+// If remote is an archive then return it otherwise return nil
+func (f *Fs) findArchive(remote string) (archive *archive) {
+	archive = findArchive(remote)
+	if archive != nil {
+		f.mu.Lock()
+		f.archives[remote] = archive
+		f.mu.Unlock()
+	}
+	return archive
+}
+
+// Instantiate archive if it hasn't been instantiated yet
+//
+// This is done lazily so that we can list a directory full of
+// archives without opening them all.
+func (a *archive) init(ctx context.Context, f fs.Fs) (fs.Fs, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if a.f != nil {
+		return a.f, nil
+	}
+	newFs, err := a.archiver.New(ctx, f, a.remote, a.prefix, a.root)
+	if err != nil && err != fs.ErrorIsFile {
+		return nil, fmt.Errorf("failed to create archive %q: %w", a.remote, err)
+	}
+	a.f = newFs
+	return a.f, nil
+}
+
+// NewFs constructs an Fs from the path.
+//
+// The returned Fs is the actual Fs, referenced by remote in the config
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs.Fs, err error) {
+	// defer log.Trace(nil, "name=%q, root=%q, m=%v", name, root, m)("f=%+v, err=%v", &outFs, &err)
+	// Parse config into Options struct
+	opt := new(Options)
+	err = configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	remote := opt.Remote
+	origRoot := root
+
+	// If remote is empty, use the root instead
+	if remote == "" {
+		remote = root
+		root = ""
+	}
+	isDirectory := strings.HasSuffix(remote, "/")
+	remote = strings.TrimRight(remote, "/")
+	if remote == "" {
+		remote = "/"
+	}
+	if strings.HasPrefix(remote, name+":") {
+		return nil, errors.New("can't point archive remote at itself - check the value of the upstreams setting")
+	}
+
+	_ = isDirectory
+
+	foundArchive := subArchive(remote)
+	if foundArchive != nil {
+		fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
+		// Archive path
+		foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
+		// Path to the archive
+		archiveRemote := remote[:len(foundArchive.remote)]
+		// Remote is archive leaf name
+		foundArchive.remote = path.Base(archiveRemote)
+		foundArchive.prefix = ""
+		// Point remote to archive file
+		remote = archiveRemote
+	}
+
+	// Make sure to remove trailing . referring to the current dir
+	if path.Base(root) == "." {
+		root = strings.TrimSuffix(root, ".")
+	}
+	remotePath := fspath.JoinRootPath(remote, root)
+	wrappedFs, err := cache.Get(ctx, remotePath)
+	if err != fs.ErrorIsFile && err != nil {
+		return nil, fmt.Errorf("failed to make remote %q to wrap: %w", remote, err)
+	}
+
+	f := &Fs{
+		name: name,
+		//root:     path.Join(remotePath, root),
+		root:     origRoot,
+		opt:      *opt,
+		f:        wrappedFs,
+		archives: make(map[string]*archive),
+	}
+	cache.PinUntilFinalized(f.f, f)
+	// the features here are ones we could support, and they are
+	// ANDed with the ones from wrappedFs
+	f.features = (&fs.Features{
+		CaseInsensitive:         true,
+		DuplicateFiles:          false,
+		ReadMimeType:            true,
+		WriteMimeType:           true,
+		CanHaveEmptyDirectories: true,
+		BucketBased:             true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
+	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
+
+	if foundArchive != nil {
+		fs.Debugf(f, "Root is an archive")
+		if err != fs.ErrorIsFile {
+			return nil, fmt.Errorf("expecting to find a file at %q", remote)
+		}
+		return foundArchive.init(ctx, f.f)
+	}
+	// Correct root if definitely pointing to a file
+	if err == fs.ErrorIsFile {
+		f.root = path.Dir(f.root)
+		if f.root == "." || f.root == "/" {
+			f.root = ""
+		}
+	}
+	return f, err
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("archive root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Rmdir removes the root directory of the Fs object
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	return f.f.Rmdir(ctx, dir)
+}
+
+// Hashes returns hash.HashNone to indicate remote hashing is unavailable
+func (f *Fs) Hashes() hash.Set {
+	return f.f.Hashes()
+}
+
+// Mkdir makes the root directory of the Fs object
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	return f.f.Mkdir(ctx, dir)
+}
+
+// Purge all files in the directory
+//
+// Implement this if you have a way of deleting all the files
+// quicker than just running Remove() on the result of List()
+//
+// Return an error if it doesn't exist
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	do := f.f.Features().Purge
+	if do == nil {
+		return fs.ErrorCantPurge
+	}
+	return do(ctx, dir)
+}
+
+// Copy src to this remote using server-side copy operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	do := f.f.Features().Copy
+	if do == nil {
+		return nil, fs.ErrorCantCopy
+	}
+	// FIXME
+	// o, ok := src.(*Object)
+	// if !ok {
+	// 	return nil, fs.ErrorCantCopy
+	// }
+	return do(ctx, src, remote)
+}
+
+// Move src to this remote using server-side move operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	do := f.f.Features().Move
+	if do == nil {
+		return nil, fs.ErrorCantMove
+	}
+	// FIXME
+	// o, ok := src.(*Object)
+	// if !ok {
+	// 	return nil, fs.ErrorCantMove
+	// }
+	return do(ctx, src, remote)
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove
+//
+// If destination exists then return fs.ErrorDirExists
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
+	do := f.f.Features().DirMove
+	if do == nil {
+		return fs.ErrorCantDirMove
+	}
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(srcFs, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+	return do(ctx, srcFs.f, srcRemote, dstRemote)
+}
+
+// ChangeNotify calls the passed function with a path
+// that has had changes. If the implementation
+// uses polling, it should adhere to the given interval.
+// At least one value will be written to the channel,
+// specifying the initial value and updated values might
+// follow. A 0 Duration should pause the polling.
+// The ChangeNotify implementation must empty the channel
+// regularly. When the channel gets closed, the implementation
+// should stop polling and release resources.
+func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), ch <-chan time.Duration) {
+	do := f.f.Features().ChangeNotify
+	if do == nil {
+		return
+	}
+	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
+		// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
+		notifyFunc(path, entryType)
+	}
+	do(ctx, wrappedNotifyFunc, ch)
+}
+
+// DirCacheFlush resets the directory cache - used in testing
+// as an optional interface
+func (f *Fs) DirCacheFlush() {
+	do := f.f.Features().DirCacheFlush
+	if do != nil {
+		do()
+	}
+}
+
+func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
+	var o fs.Object
+	var err error
+	if stream {
+		o, err = f.f.Features().PutStream(ctx, in, src, options...)
+	} else {
+		o, err = f.f.Put(ctx, in, src, options...)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	o, err := f.NewObject(ctx, src.Remote())
+	switch err {
+	case nil:
+		return o, o.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		return f.put(ctx, in, src, false, options...)
+	default:
+		return nil, err
+	}
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	o, err := f.NewObject(ctx, src.Remote())
+	switch err {
+	case nil:
+		return o, o.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		return f.put(ctx, in, src, true, options...)
+	default:
+		return nil, err
+	}
+}
+
+// About gets quota information from the Fs
+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
+	do := f.f.Features().About
+	if do == nil {
+		return nil, errors.New("not supported by underlying remote")
+	}
+	return do(ctx)
+}
+
+// Find the Fs for the directory
+func (f *Fs) findFs(ctx context.Context, dir string) (subFs fs.Fs, err error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	subFs = f.f
+
+	// FIXME should do this with a better datastructure like a prefix tree
+	// FIXME want to find the longest first otherwise nesting won't work
+	dirSlash := dir + "/"
+	for archiverRemote, archive := range f.archives {
+		subRemote := archiverRemote + "/"
+		if strings.HasPrefix(dirSlash, subRemote) {
+			subFs, err = archive.init(ctx, f.f)
+			if err != nil {
+				return nil, err
+			}
+			break
+		}
+	}
+
+	return subFs, nil
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)
+
+	subFs, err := f.findFs(ctx, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	entries, err = subFs.List(ctx, dir)
+	if err != nil {
+		return nil, err
+	}
+	for i, entry := range entries {
+		// Can only unarchive files
+		if o, ok := entry.(fs.Object); ok {
+			remote := o.Remote()
+			archive := f.findArchive(remote)
+			if archive != nil {
+				// Overwrite entry with directory
+				entries[i] = fs.NewDir(remote, o.ModTime(ctx))
+			}
+		}
+	}
+	return entries, nil
+}
+
+// NewObject creates a new remote archive file object
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+
+	dir := path.Dir(remote)
+	if dir == "/" || dir == "." {
+		dir = ""
+	}
+
+	subFs, err := f.findFs(ctx, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	o, err := subFs.NewObject(ctx, remote)
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// Precision is the greatest precision of all the archivers
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Shutdown the backend, closing any background tasks and any
+// cached connections.
+func (f *Fs) Shutdown(ctx context.Context) error {
+	if do := f.f.Features().Shutdown; do != nil {
+		return do(ctx)
+	}
+	return nil
+}
+
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	do := f.f.Features().PublicLink
+	if do == nil {
+		return "", errors.New("PublicLink not supported")
+	}
+	return do(ctx, remote, expire, unlink)
+}
+
+// PutUnchecked in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+//
+// May create duplicates or return errors if src already
+// exists.
+func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	do := f.f.Features().PutUnchecked
+	if do == nil {
+		return nil, errors.New("can't PutUnchecked")
+	}
+	o, err := do(ctx, in, src, options...)
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// MergeDirs merges the contents of all the directories passed
+// in into the first one and rmdirs the other directories.
+func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
+	if len(dirs) == 0 {
+		return nil
+	}
+	do := f.f.Features().MergeDirs
+	if do == nil {
+		return errors.New("MergeDirs not supported")
+	}
+	return do(ctx, dirs)
+}
+
+// CleanUp the trash in the Fs
+//
+// Implement this if you have a way of emptying the trash or
+// otherwise cleaning up old versions of files.
+func (f *Fs) CleanUp(ctx context.Context) error {
+	do := f.f.Features().CleanUp
+	if do == nil {
+		return errors.New("not supported by underlying remote")
+	}
+	return do(ctx)
+}
+
+// OpenWriterAt opens with a handle for random access writes
+//
+// Pass in the remote desired and the size if known.
+//
+// It truncates any existing object
+func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
+	do := f.f.Features().OpenWriterAt
+	if do == nil {
+		return nil, fs.ErrorNotImplemented
+	}
+	return do(ctx, remote, size)
+}
+
+// UnWrap returns the Fs that this Fs is wrapping
+func (f *Fs) UnWrap() fs.Fs {
+	return f.f
+}
+
+// WrapFs returns the Fs that is wrapping this Fs
+func (f *Fs) WrapFs() fs.Fs {
+	return f.wrapper
+}
+
+// SetWrapper sets the Fs that is wrapping this Fs
+func (f *Fs) SetWrapper(wrapper fs.Fs) {
+	f.wrapper = wrapper
+}
+
+// OpenChunkWriter returns the chunk size and a ChunkWriter
+//
+// Pass in the remote and the src object
+// You can also use options to hint at the desired chunk size
+func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
+	do := f.f.Features().OpenChunkWriter
+	if do == nil {
+		return info, nil, fs.ErrorNotImplemented
+	}
+	return do(ctx, remote, src, options...)
+}
+
+// UserInfo returns info about the connected user
+func (f *Fs) UserInfo(ctx context.Context) (map[string]string, error) {
+	do := f.f.Features().UserInfo
+	if do == nil {
+		return nil, fs.ErrorNotImplemented
+	}
+	return do(ctx)
+}
+
+// Disconnect the current user
+func (f *Fs) Disconnect(ctx context.Context) error {
+	do := f.f.Features().Disconnect
+	if do == nil {
+		return fs.ErrorNotImplemented
+	}
+	return do(ctx)
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs              = (*Fs)(nil)
+	_ fs.Purger          = (*Fs)(nil)
+	_ fs.PutStreamer     = (*Fs)(nil)
+	_ fs.Copier          = (*Fs)(nil)
+	_ fs.Mover           = (*Fs)(nil)
+	_ fs.DirMover        = (*Fs)(nil)
+	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.ChangeNotifier  = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
+	_ fs.Shutdowner      = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
+	_ fs.PutUncheckeder  = (*Fs)(nil)
+	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.CleanUpper      = (*Fs)(nil)
+	_ fs.OpenWriterAter  = (*Fs)(nil)
+	_ fs.OpenChunkWriter = (*Fs)(nil)
+	_ fs.UserInfoer      = (*Fs)(nil)
+	_ fs.Disconnecter    = (*Fs)(nil)
+	// FIXME _ fs.FullObject      = (*Object)(nil)
+)

backend/archive/archive_internal_test.go

@@ -0,0 +1,220 @@
+package archive
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/filter"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// FIXME need to test Open with seek
+// FIXME need to test NewObject
+
+// run - run a shell command
+func run(t *testing.T, args ...string) {
+	cmd := exec.Command(args[0], args[1:]...)
+	fs.Debugf(nil, "run args = %v", args)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf(`
+----------------------------
+Failed to run %v: %v
+Command output was:
+%s
+----------------------------
+`, args, err, out)
+	}
+}
+
+// check the dst and src are identical
+func checkTree(ctx context.Context, name string, t *testing.T, dstArchive, src string, expectedCount int) {
+	t.Run(name, func(t *testing.T) {
+		fs.Debugf(nil, "check %q vs %q", dstArchive, src)
+		Farchive, err := cache.Get(ctx, dstArchive)
+		if err != fs.ErrorIsFile {
+			require.NoError(t, err)
+		}
+		Fsrc, err := cache.Get(ctx, src)
+		if err != fs.ErrorIsFile {
+			require.NoError(t, err)
+		}
+
+		var matches bytes.Buffer
+		opt := operations.CheckOpt{
+			Fdst:  Farchive,
+			Fsrc:  Fsrc,
+			Match: &matches,
+		}
+
+		for _, action := range []string{"Check", "Download"} {
+			t.Run(action, func(t *testing.T) {
+				matches.Reset()
+				if action == "Download" {
+					assert.NoError(t, operations.CheckDownload(ctx, &opt))
+				} else {
+					assert.NoError(t, operations.Check(ctx, &opt))
+				}
+				if expectedCount > 0 {
+					assert.Equal(t, expectedCount, strings.Count(matches.String(), "\n"))
+				}
+			})
+		}
+
+		t.Run("NewObject", func(t *testing.T) {
+			// Check we can run NewObject on all files and read them
+			assert.NoError(t, operations.ListFn(ctx, Fsrc, func(srcObj fs.Object) {
+				if t.Failed() {
+					return
+				}
+				remote := srcObj.Remote()
+				archiveObj, err := Farchive.NewObject(ctx, remote)
+				require.NoError(t, err, remote)
+				assert.Equal(t, remote, archiveObj.Remote(), remote)
+
+				// Test that the contents are the same
+				archiveBuf := fstests.ReadObject(ctx, t, archiveObj, -1)
+				srcBuf := fstests.ReadObject(ctx, t, srcObj, -1)
+				assert.Equal(t, srcBuf, archiveBuf)
+
+				if len(srcBuf) < 81 {
+					return
+				}
+
+				// Tests that Open works with SeekOption
+				assert.Equal(t, srcBuf[50:], fstests.ReadObject(ctx, t, archiveObj, -1, &fs.SeekOption{Offset: 50}), "contents differ after seek")
+
+				// Tests that Open works with RangeOption
+				for _, test := range []struct {
+					ro                 fs.RangeOption
+					wantStart, wantEnd int
+				}{
+					{fs.RangeOption{Start: 5, End: 15}, 5, 16},
+					{fs.RangeOption{Start: 80, End: -1}, 80, len(srcBuf)},
+					{fs.RangeOption{Start: 81, End: 100000}, 81, len(srcBuf)},
+					{fs.RangeOption{Start: -1, End: 20}, len(srcBuf) - 20, len(srcBuf)}, // if start is omitted this means get the final bytes
+					// {fs.RangeOption{Start: -1, End: -1}, 0, len(srcBuf)}, - this seems to work but the RFC doesn't define it
+				} {
+					got := fstests.ReadObject(ctx, t, archiveObj, -1, &test.ro)
+					foundAt := strings.Index(srcBuf, got)
+					help := fmt.Sprintf("%#v failed want [%d:%d] got [%d:%d]", test.ro, test.wantStart, test.wantEnd, foundAt, foundAt+len(got))
+					assert.Equal(t, srcBuf[test.wantStart:test.wantEnd], got, help)
+				}
+
+				// Test that the modtimes are correct
+				fstest.AssertTimeEqualWithPrecision(t, remote, srcObj.ModTime(ctx), archiveObj.ModTime(ctx), Farchive.Precision())
+
+				// Test that the sizes are correct
+				assert.Equal(t, srcObj.Size(), archiveObj.Size())
+
+				// Test that Strings are OK
+				assert.Equal(t, srcObj.String(), archiveObj.String())
+			}))
+		})
+
+		// t.Logf("Fdst ------------- %v", Fdst)
+		// operations.List(ctx, Fdst, os.Stdout)
+		// t.Logf("Fsrc ------------- %v", Fsrc)
+		// operations.List(ctx, Fsrc, os.Stdout)
+	})
+
+}
+
+// test creating and reading back some archives
+//
+// Note that this uses rclone and zip as external binaries.
+func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T, output, input string)) {
+	ctx := context.Background()
+	checkFiles := 1000
+
+	// create random test input files
+	inputRoot := t.TempDir()
+	input := filepath.Join(inputRoot, archiveName)
+	require.NoError(t, os.Mkdir(input, 0777))
+	run(t, "rclone", "test", "makefiles", "--files", strconv.Itoa(checkFiles), "--ascii", input)
+
+	// Create the archive
+	output := t.TempDir()
+	zipFile := path.Join(output, archiveName)
+	archiveFn(t, zipFile, input)
+
+	// Check the archive itself
+	checkTree(ctx, "Archive", t, ":archive:"+zipFile, input, checkFiles)
+
+	// Now check a subdirectory
+	fis, err := os.ReadDir(input)
+	require.NoError(t, err)
+	subDir := "NOT FOUND"
+	aFile := "NOT FOUND"
+	for _, fi := range fis {
+		if fi.IsDir() {
+			subDir = fi.Name()
+		} else {
+			aFile = fi.Name()
+		}
+	}
+	checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
+
+	// Now check a single file
+	fiCtx, fi := filter.AddConfig(ctx)
+	require.NoError(t, fi.AddRule("+ "+aFile))
+	require.NoError(t, fi.AddRule("- *"))
+	checkTree(fiCtx, "SingleFile", t, ":archive:"+zipFile+"/"+aFile, filepath.Join(input, aFile), 0)
+
+	// Now check the level above
+	checkTree(ctx, "Root", t, ":archive:"+output, inputRoot, checkFiles)
+	// run(t, "cp", "-a", inputRoot, output, "/tmp/test-"+archiveName)
+}
+
+// Make sure we have the executable named
+func skipIfNoExe(t *testing.T, exeName string) {
+	_, err := exec.LookPath(exeName)
+	if err != nil {
+		t.Skipf("%s executable not installed", exeName)
+	}
+}
+
+// Test creating and reading back some archives
+//
+// Note that this uses rclone and zip as external binaries.
+func TestArchiveZip(t *testing.T) {
+	fstest.Initialise()
+	skipIfNoExe(t, "zip")
+	skipIfNoExe(t, "rclone")
+	testArchive(t, "test.zip", func(t *testing.T, output, input string) {
+		oldcwd, err := os.Getwd()
+		require.NoError(t, err)
+		require.NoError(t, os.Chdir(input))
+		defer func() {
+			require.NoError(t, os.Chdir(oldcwd))
+		}()
+		run(t, "zip", "-9r", output, ".")
+	})
+}
+
+// Test creating and reading back some archives
+//
+// Note that this uses rclone and squashfs as external binaries.
+func TestArchiveSquashfs(t *testing.T) {
+	fstest.Initialise()
+	skipIfNoExe(t, "mksquashfs")
+	skipIfNoExe(t, "rclone")
+	testArchive(t, "test.sqfs", func(t *testing.T, output, input string) {
+		run(t, "mksquashfs", input, output)
+	})
+}

backend/archive/archive_test.go

@@ -0,0 +1,65 @@
+// Test Archive filesystem interface
+package archive_test
+
+import (
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	_ "github.com/rclone/rclone/backend/memory"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+var (
+	unimplementableFsMethods = []string{"ListR"}
+	// In these tests we receive objects from the underlying remote which don't implement these methods
+	unimplementableObjectMethods = []string{"GetTier", "ID", "Metadata", "MimeType", "SetTier", "UnWrap"}
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	if *fstest.RemoteName == "" {
+		t.Skip("Skipping as -remote not set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:                   *fstest.RemoteName,
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
+	})
+}
+
+func TestLocal(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	remote := t.TempDir()
+	name := "TestArchiveLocal"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "archive"},
+			{Name: name, Key: "remote", Value: remote},
+		},
+		QuickTestOK:                  true,
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
+	})
+}
+
+func TestMemory(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	remote := ":memory:"
+	name := "TestArchiveMemory"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "archive"},
+			{Name: name, Key: "remote", Value: remote},
+		},
+		QuickTestOK:                  true,
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
+	})
+}

backend/archive/archiver/archiver.go

@@ -0,0 +1,24 @@
+// Package archiver registers all the archivers
+package archiver
+
+import (
+	"context"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Archiver describes an archive package
+type Archiver struct {
+	// New constructs an Fs from the (wrappedFs, remote) with the objects
+	// prefix with prefix and rooted at root
+	New       func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error)
+	Extension string
+}
+
+// Archivers is a slice of all registered archivers
+var Archivers []Archiver
+
+// Register adds the archivers provided to the list of known archivers
+func Register(as ...Archiver) {
+	Archivers = append(Archivers, as...)
+}

backend/archive/base/base.go

@@ -0,0 +1,233 @@
+// Package base is a base archive Fs
+package base
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/vfs"
+)
+
+// Fs represents a wrapped fs.Fs
+type Fs struct {
+	f           fs.Fs
+	wrapper     fs.Fs
+	name        string
+	features    *fs.Features // optional features
+	vfs         *vfs.VFS
+	node        vfs.Node // archive object
+	remote      string   // remote of the archive object
+	prefix      string   // position for objects
+	prefixSlash string   // position for objects with a slash on
+	root        string   // position to read from within the archive
+}
+
+var errNotImplemented = errors.New("internal error: method not implemented in archiver")
+
+// New constructs an Fs from the (wrappedFs, remote) with the objects
+// prefix with prefix and rooted at root
+func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs, error) {
+	// FIXME vfs cache?
+	// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
+	fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
+	VFS := vfs.New(wrappedFs, nil)
+	node, err := VFS.Stat(remote)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
+	}
+
+	f := &Fs{
+		f:           wrappedFs,
+		name:        path.Join(fs.ConfigString(wrappedFs), remote),
+		vfs:         VFS,
+		node:        node,
+		remote:      remote,
+		root:        root,
+		prefix:      prefix,
+		prefixSlash: prefix + "/",
+	}
+
+	// FIXME
+	// the features here are ones we could support, and they are
+	// ANDed with the ones from wrappedFs
+	//
+	// FIXME some of these need to be forced on - CanHaveEmptyDirectories
+	f.features = (&fs.Features{
+		CaseInsensitive:         false,
+		DuplicateFiles:          false,
+		ReadMimeType:            false, // MimeTypes not supported with gzip
+		WriteMimeType:           false,
+		BucketBased:             false,
+		CanHaveEmptyDirectories: true,
+	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
+
+	return f, nil
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return f.name
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	return nil, errNotImplemented
+}
+
+// NewObject finds the Object at remote.
+func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
+	return nil, errNotImplemented
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Mkdir makes the directory (container, bucket)
+//
+// Shouldn't return an error if it already exists
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	return vfs.EROFS
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	return vfs.EROFS
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
+	return nil, vfs.EROFS
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.None)
+}
+
+// UnWrap returns the Fs that this Fs is wrapping
+func (f *Fs) UnWrap() fs.Fs {
+	return f.f
+}
+
+// WrapFs returns the Fs that is wrapping this Fs
+func (f *Fs) WrapFs() fs.Fs {
+	return f.wrapper
+}
+
+// SetWrapper sets the Fs that is wrapping this Fs
+func (f *Fs) SetWrapper(wrapper fs.Fs) {
+	f.wrapper = wrapper
+}
+
+// Object describes an object to be read from the raw zip file
+type Object struct {
+	f      *Fs
+	remote string
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.f
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.Remote()
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+	return -1
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return time.Now()
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	return vfs.EROFS
+}
+
+// Storable raturns a boolean indicating if this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Hash returns the selected checksum of the file
+// If no checksum is available it returns ""
+func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
+	return "", hash.ErrUnsupported
+}
+
+// Open opens the file for read.  Call Close() on the returned io.ReadCloser
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
+	return nil, errNotImplemented
+}
+
+// Update in to the object with the modTime given of the given size
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	return vfs.EROFS
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	return vfs.EROFS
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs        = (*Fs)(nil)
+	_ fs.UnWrapper = (*Fs)(nil)
+	_ fs.Wrapper   = (*Fs)(nil)
+	_ fs.Object    = (*Object)(nil)
+)

backend/archive/squashfs/cache.go

@@ -0,0 +1,124 @@
+package squashfs
+
+// Could just be using bare object Open with RangeRequest which
+// would transfer the minimum amount of data but may be slower.
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/rclone/rclone/vfs"
+)
+
+// Cache file handles for accessing the file
+type cache struct {
+	node  vfs.Node
+	fhsMu sync.Mutex
+	fhs   []cacheHandle
+}
+
+// A cached file handle
+type cacheHandle struct {
+	offset int64
+	fh     vfs.Handle
+}
+
+// Make a new cache
+func newCache(node vfs.Node) *cache {
+	return &cache{
+		node: node,
+	}
+}
+
+// Get a vfs.Handle from the pool or open one
+//
+// This tries to find an open file handle which doesn't require seeking.
+func (c *cache) open(off int64) (fh vfs.Handle, err error) {
+	c.fhsMu.Lock()
+	defer c.fhsMu.Unlock()
+
+	if len(c.fhs) > 0 {
+		// Look for exact match first
+		for i, cfh := range c.fhs {
+			if cfh.offset == off {
+				// fs.Debugf(nil, "CACHE MATCH")
+				c.fhs = append(c.fhs[:i], c.fhs[i+1:]...)
+				return cfh.fh, nil
+
+			}
+		}
+		// fs.Debugf(nil, "CACHE MISS")
+		// Just take the first one if not found
+		cfh := c.fhs[0]
+		c.fhs = c.fhs[1:]
+		return cfh.fh, nil
+	}
+
+	fh, err = c.node.Open(os.O_RDONLY)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open squashfs archive: %w", err)
+	}
+
+	return fh, nil
+}
+
+// Close a vfs.Handle or return it to the pool
+//
+// off should be the offset the file handle would read from without seeking
+func (c *cache) close(fh vfs.Handle, off int64) {
+	c.fhsMu.Lock()
+	defer c.fhsMu.Unlock()
+
+	c.fhs = append(c.fhs, cacheHandle{
+		offset: off,
+		fh:     fh,
+	})
+}
+
+// ReadAt reads len(p) bytes into p starting at offset off in the underlying
+// input source. It returns the number of bytes read (0 <= n <= len(p)) and any
+// error encountered.
+//
+// When ReadAt returns n < len(p), it returns a non-nil error explaining why
+// more bytes were not returned. In this respect, ReadAt is stricter than Read.
+//
+// Even if ReadAt returns n < len(p), it may use all of p as scratch
+// space during the call. If some data is available but not len(p) bytes,
+// ReadAt blocks until either all the data is available or an error occurs.
+// In this respect ReadAt is different from Read.
+//
+// If the n = len(p) bytes returned by ReadAt are at the end of the input
+// source, ReadAt may return either err == EOF or err == nil.
+//
+// If ReadAt is reading from an input source with a seek offset, ReadAt should
+// not affect nor be affected by the underlying seek offset.
+//
+// Clients of ReadAt can execute parallel ReadAt calls on the same input
+// source.
+//
+// Implementations must not retain p.
+func (c *cache) ReadAt(p []byte, off int64) (n int, err error) {
+	fh, err := c.open(off)
+	if err != nil {
+		return n, err
+	}
+	defer func() {
+		c.close(fh, off+int64(len(p)))
+	}()
+	// fs.Debugf(nil, "ReadAt(p[%d], off=%d, fh=%p)", len(p), off, fh)
+	return fh.ReadAt(p, off)
+}
+
+var errCacheNotImplemented = errors.New("internal error: squashfs cache doesn't implement method")
+
+// WriteAt method dummy stub to satisfy interface
+func (c *cache) WriteAt(p []byte, off int64) (n int, err error) {
+	return 0, errCacheNotImplemented
+}
+
+// Seek method dummy stub to satisfy interface
+func (c *cache) Seek(offset int64, whence int) (int64, error) {
+	return 0, errCacheNotImplemented
+}

backend/archive/squashfs/squashfs.go

@@ -0,0 +1,446 @@
+// Package squashfs implements a squashfs archiver for the archive backend
+package squashfs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/ncw/go-diskfs/filesystem/squashfs"
+	"github.com/rclone/rclone/backend/archive/archiver"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/log"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/vfs"
+	"github.com/rclone/rclone/vfs/vfscommon"
+)
+
+func init() {
+	archiver.Register(archiver.Archiver{
+		New:       New,
+		Extension: ".sqfs",
+	})
+}
+
+// Fs represents a wrapped fs.Fs
+type Fs struct {
+	f           fs.Fs
+	wrapper     fs.Fs
+	name        string
+	features    *fs.Features // optional features
+	vfs         *vfs.VFS
+	sqfs        *squashfs.FileSystem // interface to the squashfs
+	c           *cache
+	node        vfs.Node // squashfs file object - set if reading
+	remote      string   // remote of the squashfs file object
+	prefix      string   // position for objects
+	prefixSlash string   // position for objects with a slash on
+	root        string   // position to read from within the archive
+}
+
+// New constructs an Fs from the (wrappedFs, remote) with the objects
+// prefix with prefix and rooted at root
+func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
+	// FIXME vfs cache?
+	// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
+	fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
+	vfsOpt := vfscommon.DefaultOpt
+	vfsOpt.ReadWait = 0
+	VFS := vfs.New(wrappedFs, &vfsOpt)
+	node, err := VFS.Stat(remote)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
+	}
+
+	c := newCache(node)
+
+	// FIXME blocksize
+	sqfs, err := squashfs.Read(c, node.Size(), 0, 1024*1024)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read squashfs: %w", err)
+	}
+
+	f := &Fs{
+		f:           wrappedFs,
+		name:        path.Join(fs.ConfigString(wrappedFs), remote),
+		vfs:         VFS,
+		node:        node,
+		sqfs:        sqfs,
+		c:           c,
+		remote:      remote,
+		root:        strings.Trim(root, "/"),
+		prefix:      prefix,
+		prefixSlash: prefix + "/",
+	}
+	if prefix == "" {
+		f.prefixSlash = ""
+	}
+
+	singleObject := false
+
+	// Find the directory the root points to
+	if f.root != "" && !strings.HasSuffix(root, "/") {
+		native, err := f.toNative("")
+		if err == nil {
+			native = strings.TrimRight(native, "/")
+			_, err := f.newObjectNative(native)
+			if err == nil {
+				// If it pointed to a file, find the directory above
+				f.root = path.Dir(f.root)
+				if f.root == "." || f.root == "/" {
+					f.root = ""
+				}
+			}
+		}
+	}
+
+	// FIXME
+	// the features here are ones we could support, and they are
+	// ANDed with the ones from wrappedFs
+	//
+	// FIXME some of these need to be forced on - CanHaveEmptyDirectories
+	f.features = (&fs.Features{
+		CaseInsensitive:         false,
+		DuplicateFiles:          false,
+		ReadMimeType:            false, // MimeTypes not supported with gsquashfs
+		WriteMimeType:           false,
+		BucketBased:             false,
+		CanHaveEmptyDirectories: true,
+	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
+
+	if singleObject {
+		return f, fs.ErrorIsFile
+	}
+	return f, nil
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return fmt.Sprintf("Squashfs %q", f.name)
+}
+
+// This turns a remote into a native path in the squashfs starting with a /
+func (f *Fs) toNative(remote string) (string, error) {
+	native := strings.Trim(remote, "/")
+	if f.prefix == "" {
+		native = "/" + native
+	} else if native == f.prefix {
+		native = "/"
+	} else if !strings.HasPrefix(native, f.prefixSlash) {
+		return "", fmt.Errorf("internal error: %q doesn't start with prefix %q", native, f.prefixSlash)
+	} else {
+		native = native[len(f.prefix):]
+	}
+	if f.root != "" {
+		native = "/" + f.root + native
+	}
+	return native, nil
+}
+
+// Turn a (nativeDir, leaf) into a remote
+func (f *Fs) fromNative(nativeDir string, leaf string) string {
+	// fs.Debugf(nil, "nativeDir = %q, leaf = %q, root=%q", nativeDir, leaf, f.root)
+	dir := nativeDir
+	if f.root != "" {
+		dir = strings.TrimPrefix(dir, "/"+f.root)
+	}
+	remote := f.prefixSlash + strings.Trim(path.Join(dir, leaf), "/")
+	// fs.Debugf(nil, "dir = %q, remote=%q", dir, remote)
+	return remote
+}
+
+// Convert a FileInfo into an Object from native dir
+func (f *Fs) objectFromFileInfo(nativeDir string, item squashfs.FileStat) *Object {
+	return &Object{
+		fs:      f,
+		remote:  f.fromNative(nativeDir, item.Name()),
+		size:    item.Size(),
+		modTime: item.ModTime(),
+		item:    item,
+	}
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
+
+	nativeDir, err := f.toNative(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	items, err := f.sqfs.ReadDir(nativeDir)
+	if err != nil {
+		return nil, fmt.Errorf("read squashfs: couldn't read directory: %w", err)
+	}
+
+	entries = make(fs.DirEntries, 0, len(items))
+	for _, fi := range items {
+		item, ok := fi.(squashfs.FileStat)
+		if !ok {
+			return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
+		}
+		// fs.Debugf(item.Name(), "entry = %#v", item)
+		var entry fs.DirEntry
+		if err != nil {
+			return nil, fmt.Errorf("error reading item %q: %q", item.Name(), err)
+		}
+		if item.IsDir() {
+			var remote = f.fromNative(nativeDir, item.Name())
+			entry = fs.NewDir(remote, item.ModTime())
+		} else {
+			if item.Mode().IsRegular() {
+				entry = f.objectFromFileInfo(nativeDir, item)
+			} else {
+				fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
+				continue
+			}
+		}
+		entries = append(entries, entry)
+	}
+
+	// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
+	return entries, nil
+}
+
+// newObjectNative finds the object at the native path passed in
+func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
+	// get the path and filename
+	dir, leaf := path.Split(nativePath)
+	dir = strings.TrimRight(dir, "/")
+	leaf = strings.Trim(leaf, "/")
+
+	// FIXME need to detect directory not found
+	fis, err := f.sqfs.ReadDir(dir)
+	if err != nil {
+
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	for _, fi := range fis {
+		if fi.Name() == leaf {
+			if fi.IsDir() {
+				return nil, fs.ErrorNotAFile
+			}
+			item, ok := fi.(squashfs.FileStat)
+			if !ok {
+				return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
+			}
+			o = f.objectFromFileInfo(dir, item)
+			break
+		}
+	}
+	if o == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return o, nil
+}
+
+// NewObject finds the Object at remote.
+func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
+	defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
+
+	nativePath, err := f.toNative(remote)
+	if err != nil {
+		return nil, err
+	}
+	return f.newObjectNative(nativePath)
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Mkdir makes the directory (container, bucket)
+//
+// Shouldn't return an error if it already exists
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	return vfs.EROFS
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	return vfs.EROFS
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
+	return nil, vfs.EROFS
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.None)
+}
+
+// UnWrap returns the Fs that this Fs is wrapping
+func (f *Fs) UnWrap() fs.Fs {
+	return f.f
+}
+
+// WrapFs returns the Fs that is wrapping this Fs
+func (f *Fs) WrapFs() fs.Fs {
+	return f.wrapper
+}
+
+// SetWrapper sets the Fs that is wrapping this Fs
+func (f *Fs) SetWrapper(wrapper fs.Fs) {
+	f.wrapper = wrapper
+}
+
+// Object describes an object to be read from the raw squashfs file
+type Object struct {
+	fs      *Fs
+	remote  string
+	size    int64
+	modTime time.Time
+	item    squashfs.FileStat
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.Remote()
+}
+
+// Turn a squashfs path into a full path for the parent Fs
+// func (o *Object) path(remote string) string {
+// 	return path.Join(o.fs.prefix, remote)
+// }
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+	return o.size
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modTime
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	return vfs.EROFS
+}
+
+// Storable raturns a boolean indicating if this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Hash returns the selected checksum of the file
+// If no checksum is available it returns ""
+func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
+	return "", hash.ErrUnsupported
+}
+
+// Open opens the file for read.  Call Close() on the returned io.ReadCloser
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
+	var offset, limit int64 = 0, -1
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			offset = x.Offset
+		case *fs.RangeOption:
+			offset, limit = x.Decode(o.Size())
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+
+	remote, err := o.fs.toNative(o.remote)
+	if err != nil {
+		return nil, err
+	}
+
+	fs.Debugf(o, "Opening %q", remote)
+	//fh, err := o.fs.sqfs.OpenFile(remote, os.O_RDONLY)
+	fh, err := o.item.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	// discard data from start as necessary
+	if offset > 0 {
+		_, err = fh.Seek(offset, io.SeekStart)
+		if err != nil {
+			return nil, err
+		}
+	}
+	// If limited then don't return everything
+	if limit >= 0 {
+		fs.Debugf(nil, "limit=%d, offset=%d, options=%v", limit, offset, options)
+		return readers.NewLimitedReadCloser(fh, limit), nil
+	}
+
+	return fh, nil
+}
+
+// Update in to the object with the modTime given of the given size
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	return vfs.EROFS
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	return vfs.EROFS
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs        = (*Fs)(nil)
+	_ fs.UnWrapper = (*Fs)(nil)
+	_ fs.Wrapper   = (*Fs)(nil)
+	_ fs.Object    = (*Object)(nil)
+)

backend/archive/zip/zip.go

@@ -0,0 +1,385 @@
+// Package zip implements a zip archiver for the archive backend
+package zip
+
+import (
+	"archive/zip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/archive/archiver"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/dirtree"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/log"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/vfs"
+	"github.com/rclone/rclone/vfs/vfscommon"
+)
+
+func init() {
+	archiver.Register(archiver.Archiver{
+		New:       New,
+		Extension: ".zip",
+	})
+}
+
+// Fs represents a wrapped fs.Fs
+type Fs struct {
+	f           fs.Fs
+	wrapper     fs.Fs
+	name        string
+	features    *fs.Features // optional features
+	vfs         *vfs.VFS
+	node        vfs.Node        // zip file object - set if reading
+	remote      string          // remote of the zip file object
+	prefix      string          // position for objects
+	prefixSlash string          // position for objects with a slash on
+	root        string          // position to read from within the archive
+	dt          dirtree.DirTree // read from zipfile
+}
+
+// New constructs an Fs from the (wrappedFs, remote) with the objects
+// prefix with prefix and rooted at root
+func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
+	// FIXME vfs cache?
+	// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
+	fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
+	vfsOpt := vfscommon.DefaultOpt
+	vfsOpt.ReadWait = 0
+	VFS := vfs.New(wrappedFs, &vfsOpt)
+	node, err := VFS.Stat(remote)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
+	}
+
+	f := &Fs{
+		f:           wrappedFs,
+		name:        path.Join(fs.ConfigString(wrappedFs), remote),
+		vfs:         VFS,
+		node:        node,
+		remote:      remote,
+		root:        root,
+		prefix:      prefix,
+		prefixSlash: prefix + "/",
+	}
+
+	// Read the contents of the zip file
+	singleObject, err := f.readZip()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open zip file: %w", err)
+	}
+
+	// FIXME
+	// the features here are ones we could support, and they are
+	// ANDed with the ones from wrappedFs
+	//
+	// FIXME some of these need to be forced on - CanHaveEmptyDirectories
+	f.features = (&fs.Features{
+		CaseInsensitive:         false,
+		DuplicateFiles:          false,
+		ReadMimeType:            false, // MimeTypes not supported with gzip
+		WriteMimeType:           false,
+		BucketBased:             false,
+		CanHaveEmptyDirectories: true,
+	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
+
+	if singleObject {
+		return f, fs.ErrorIsFile
+	}
+	return f, nil
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return fmt.Sprintf("Zip %q", f.name)
+}
+
+// readZip the zip file into f
+//
+// Returns singleObject=true if f.root points to a file
+func (f *Fs) readZip() (singleObject bool, err error) {
+	if f.node == nil {
+		return singleObject, fs.ErrorDirNotFound
+	}
+	size := f.node.Size()
+	if size < 0 {
+		return singleObject, errors.New("can't read from zip file with unknown size")
+	}
+	r, err := f.node.Open(os.O_RDONLY)
+	if err != nil {
+		return singleObject, fmt.Errorf("failed to open zip file: %w", err)
+	}
+	zr, err := zip.NewReader(r, size)
+	if err != nil {
+		return singleObject, fmt.Errorf("failed to read zip file: %w", err)
+	}
+	dt := dirtree.New()
+	for _, file := range zr.File {
+		remote := strings.Trim(path.Clean(file.Name), "/")
+		if remote == "." {
+			remote = ""
+		}
+		remote = path.Join(f.prefix, remote)
+		if f.root != "" {
+			// Ignore all files outside the root
+			if !strings.HasPrefix(remote, f.root) {
+				continue
+			}
+			if remote == f.root {
+				remote = ""
+			} else {
+				remote = strings.TrimPrefix(remote, f.root+"/")
+			}
+		}
+		if strings.HasSuffix(file.Name, "/") {
+			dir := fs.NewDir(remote, file.Modified)
+			dt.AddDir(dir)
+		} else {
+			if remote == "" {
+				remote = path.Base(f.root)
+				singleObject = true
+				dt = dirtree.New()
+			}
+			o := &Object{
+				f:      f,
+				remote: remote,
+				fh:     &file.FileHeader,
+				file:   file,
+			}
+			dt.Add(o)
+			if singleObject {
+				break
+			}
+		}
+	}
+	dt.CheckParents("")
+	dt.Sort()
+	f.dt = dt
+	//fs.Debugf(nil, "dt = %v", dt)
+	return singleObject, nil
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
+	// _, err = f.strip(dir)
+	// if err != nil {
+	// 	return nil, err
+	// }
+	entries, ok := f.dt[dir]
+	if !ok {
+		return nil, fs.ErrorDirNotFound
+	}
+	fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
+	return entries, nil
+}
+
+// NewObject finds the Object at remote.
+func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
+	defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
+	if f.dt == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	_, entry := f.dt.Find(remote)
+	if entry == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	o, ok := entry.(*Object)
+	if !ok {
+		return nil, fs.ErrorNotAFile
+	}
+	return o, nil
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Mkdir makes the directory (container, bucket)
+//
+// Shouldn't return an error if it already exists
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	return vfs.EROFS
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	return vfs.EROFS
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
+	return nil, vfs.EROFS
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.CRC32)
+}
+
+// UnWrap returns the Fs that this Fs is wrapping
+func (f *Fs) UnWrap() fs.Fs {
+	return f.f
+}
+
+// WrapFs returns the Fs that is wrapping this Fs
+func (f *Fs) WrapFs() fs.Fs {
+	return f.wrapper
+}
+
+// SetWrapper sets the Fs that is wrapping this Fs
+func (f *Fs) SetWrapper(wrapper fs.Fs) {
+	f.wrapper = wrapper
+}
+
+// Object describes an object to be read from the raw zip file
+type Object struct {
+	f      *Fs
+	remote string
+	fh     *zip.FileHeader
+	file   *zip.File
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.f
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.Remote()
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+	return int64(o.fh.UncompressedSize64)
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.fh.Modified
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	return vfs.EROFS
+}
+
+// Storable raturns a boolean indicating if this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Hash returns the selected checksum of the file
+// If no checksum is available it returns ""
+func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
+	if ht == hash.CRC32 {
+		// FIXME return empty CRC if writing
+		if o.f.dt == nil {
+			return "", nil
+		}
+		return fmt.Sprintf("%08x", o.fh.CRC32), nil
+	}
+	return "", hash.ErrUnsupported
+}
+
+// Open opens the file for read.  Call Close() on the returned io.ReadCloser
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
+	var offset, limit int64 = 0, -1
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			offset = x.Offset
+		case *fs.RangeOption:
+			offset, limit = x.Decode(o.Size())
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+
+	rc, err = o.file.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	// discard data from start as necessary
+	if offset > 0 {
+		_, err = io.CopyN(io.Discard, rc, offset)
+		if err != nil {
+			return nil, err
+		}
+	}
+	// If limited then don't return everything
+	if limit >= 0 {
+		return readers.NewLimitedReadCloser(rc, limit), nil
+	}
+
+	return rc, nil
+}
+
+// Update in to the object with the modTime given of the given size
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	return vfs.EROFS
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	return vfs.EROFS
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs        = (*Fs)(nil)
+	_ fs.UnWrapper = (*Fs)(nil)
+	_ fs.Wrapper   = (*Fs)(nil)
+	_ fs.Object    = (*Object)(nil)
+)

backend/azureblob/azureblob.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
@@ -7,7 +8,6 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/base64"
-	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -401,24 +401,6 @@ rclone does if you know the container exists already.
 			Help:     `If set, do not do HEAD before GET when getting objects.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "delete_snapshots",
-			Help: `Set to specify how to deal with snapshots on blob deletion.`,
-			Examples: []fs.OptionExample{
-				{
-					Value: "",
-					Help:  "By default, the delete operation fails if a blob has snapshots",
-				}, {
-					Value: string(blob.DeleteSnapshotsOptionTypeInclude),
-					Help:  "Specify 'include' to remove the root blob and all its snapshots",
-				}, {
-					Value: string(blob.DeleteSnapshotsOptionTypeOnly),
-					Help:  "Specify 'only' to remove only the snapshots but keep the root blob.",
-				},
-			},
-			Default:   "",
-			Exclusive: true,
-			Advanced:  true,
 		}},
 	})
 }
@@ -455,7 +437,6 @@ type Options struct {
 	DirectoryMarkers           bool                 `config:"directory_markers"`
 	NoCheckContainer           bool                 `config:"no_check_container"`
 	NoHeadObject               bool                 `config:"no_head_object"`
-	DeleteSnapshots            string               `config:"delete_snapshots"`
 }
 
 // Fs represents a remote azure server
@@ -711,11 +692,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ClientOptions: policyClientOptions,
 	}
 
-	// Here we auth by setting one of cred, sharedKeyCred, f.svc or anonymous
+	// Here we auth by setting one of cred, sharedKeyCred or f.svc
 	var (
 		cred          azcore.TokenCredential
 		sharedKeyCred *service.SharedKeyCredential
-		anonymous     = false
 	)
 	switch {
 	case opt.EnvAuth:
@@ -875,9 +855,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
 		}
-	case opt.Account != "":
-		// Anonymous access
-		anonymous = true
 	default:
 		return nil, errors.New("no authentication method configured")
 	}
@@ -907,12 +884,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			if err != nil {
 				return nil, fmt.Errorf("create client failed: %w", err)
 			}
-		} else if anonymous {
-			// Anonymous public access
-			f.svc, err = service.NewClientWithNoCredential(opt.Endpoint, &clientOpt)
-			if err != nil {
-				return nil, fmt.Errorf("create public client failed: %w", err)
-			}
 		}
 	}
 	if f.svc == nil {
@@ -1098,7 +1069,7 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 			isDirectory := isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote)
 			if isDirectory {
 				// Don't insert the root directory
-				if remote == f.opt.Enc.ToStandardPath(directory) {
+				if remote == directory {
 					continue
 				}
 				// process directory markers as directories
@@ -1995,21 +1966,34 @@ func (rs *readSeekCloser) Close() error {
 	return nil
 }
 
+// increment the array as LSB binary
+func increment(xs *[8]byte) {
+	for i, digit := range xs {
+		newDigit := digit + 1
+		xs[i] = newDigit
+		if newDigit >= digit {
+			// exit if no carry
+			break
+		}
+	}
+}
+
 // record chunk number and id for Close
 type azBlock struct {
-	chunkNumber uint64
+	chunkNumber int
 	id          string
 }
 
 // Implements the fs.ChunkWriter interface
 type azChunkWriter struct {
-	chunkSize int64
-	size      int64
-	f         *Fs
-	ui        uploadInfo
-	blocksMu  sync.Mutex // protects the below
-	blocks    []azBlock  // list of blocks for finalize
-	o         *Object
+	chunkSize     int64
+	size          int64
+	f             *Fs
+	ui            uploadInfo
+	blocksMu      sync.Mutex // protects the below
+	blocks        []azBlock  // list of blocks for finalize
+	binaryBlockID [8]byte    // block counter as LSB first 8 bytes
+	o             *Object
 }
 
 // OpenChunkWriter returns the chunk size and a ChunkWriter
@@ -2094,16 +2078,16 @@ func (w *azChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 		return 0, nil
 	}
 	md5sum := m.Sum(nil)
+	transactionalMD5 := md5sum[:]
 
 	// increment the blockID and save the blocks for finalize
-	var binaryBlockID [8]byte // block counter as LSB first 8 bytes
-	binary.LittleEndian.PutUint64(binaryBlockID[:], uint64(chunkNumber))
-	blockID := base64.StdEncoding.EncodeToString(binaryBlockID[:])
+	increment(&w.binaryBlockID)
+	blockID := base64.StdEncoding.EncodeToString(w.binaryBlockID[:])
 
 	// Save the blockID for the commit
 	w.blocksMu.Lock()
 	w.blocks = append(w.blocks, azBlock{
-		chunkNumber: uint64(chunkNumber),
+		chunkNumber: chunkNumber,
 		id:          blockID,
 	})
 	w.blocksMu.Unlock()
@@ -2116,7 +2100,7 @@ func (w *azChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 		}
 		options := blockblob.StageBlockOptions{
 			// Specify the transactional md5 for the body, to be validated by the service.
-			TransactionalValidation: blob.TransferValidationTypeMD5(md5sum),
+			TransactionalValidation: blob.TransferValidationTypeMD5(transactionalMD5),
 		}
 		_, err = w.ui.blb.StageBlock(ctx, blockID, &readSeekCloser{Reader: reader, Seeker: reader}, &options)
 		if err != nil {
@@ -2168,20 +2152,9 @@ func (w *azChunkWriter) Close(ctx context.Context) (err error) {
 		return w.blocks[i].chunkNumber < w.blocks[j].chunkNumber
 	})
 
-	// Create and check a list of block IDs
+	// Create a list of block IDs
 	blockIDs := make([]string, len(w.blocks))
 	for i := range w.blocks {
-		if w.blocks[i].chunkNumber != uint64(i) {
-			return fmt.Errorf("internal error: expecting chunkNumber %d but got %d", i, w.blocks[i].chunkNumber)
-		}
-		chunkBytes, err := base64.StdEncoding.DecodeString(w.blocks[i].id)
-		if err != nil {
-			return fmt.Errorf("internal error: bad block ID: %w", err)
-		}
-		chunkNumber := binary.LittleEndian.Uint64(chunkBytes)
-		if w.blocks[i].chunkNumber != chunkNumber {
-			return fmt.Errorf("internal error: expecting decoded chunkNumber %d but got %d", w.blocks[i].chunkNumber, chunkNumber)
-		}
 		blockIDs[i] = w.blocks[i].id
 	}
 
@@ -2383,10 +2356,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
 	blb := o.getBlobSVC()
-	opt := blob.DeleteOptions{}
-	if o.fs.opt.DeleteSnapshots != "" {
-		action := blob.DeleteSnapshotsOptionType(o.fs.opt.DeleteSnapshots)
-		opt.DeleteSnapshots = &action
+	//only := blob.DeleteSnapshotsOptionTypeOnly
+	opt := blob.DeleteOptions{
+		//DeleteSnapshots: &only,
 	}
 	return o.fs.pacer.Call(func() (bool, error) {
 		_, err := blb.Delete(ctx, &opt)

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 
@@ -16,3 +17,21 @@ func (f *Fs) InternalTest(t *testing.T) {
 	enabled = f.Features().GetTier
 	assert.True(t, enabled)
 }
+
+func TestIncrement(t *testing.T) {
+	for _, test := range []struct {
+		in   [8]byte
+		want [8]byte
+	}{
+		{[8]byte{0, 0, 0, 0}, [8]byte{1, 0, 0, 0}},
+		{[8]byte{0xFE, 0, 0, 0}, [8]byte{0xFF, 0, 0, 0}},
+		{[8]byte{0xFF, 0, 0, 0}, [8]byte{0, 1, 0, 0}},
+		{[8]byte{0, 1, 0, 0}, [8]byte{1, 1, 0, 0}},
+		{[8]byte{0xFF, 0xFF, 0xFF, 0xFE}, [8]byte{0, 0, 0, 0xFF}},
+		{[8]byte{0xFF, 0xFF, 0xFF, 0xFF}, [8]byte{0, 0, 0, 0, 1}},
+		{[8]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, [8]byte{0, 0, 0, 0, 0, 0, 0}},
+	} {
+		increment(&test.in)
+		assert.Equal(t, test.want, test.in)
+	}
+}

backend/azureblob/azureblob_test.go

@@ -1,6 +1,7 @@
 // Test AzureBlob filesystem interface
 
 //go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9 || solaris || js
+// +build plan9 solaris js
 
-// Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob

backend/azurefiles/azurefiles.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 // Package azurefiles provides an interface to Microsoft Azure Files
 package azurefiles
@@ -1035,10 +1036,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if _, createErr := fc.Create(ctx, size, nil); createErr != nil {
 			return fmt.Errorf("update: unable to create file: %w", createErr)
 		}
-	} else if size != o.Size() {
+	} else {
 		// Resize the file if needed
-		if _, resizeErr := fc.Resize(ctx, size, nil); resizeErr != nil {
-			return fmt.Errorf("update: unable to resize while trying to update: %w ", resizeErr)
+		if size != o.Size() {
+			if _, resizeErr := fc.Resize(ctx, size, nil); resizeErr != nil {
+				return fmt.Errorf("update: unable to resize while trying to update: %w ", resizeErr)
+			}
 		}
 	}
 

backend/azurefiles/azurefiles_internal_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package azurefiles
 

backend/azurefiles/azurefiles_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package azurefiles
 

backend/azurefiles/azurefiles_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9 || js
+// +build plan9 js
 
-// Package azurefiles provides an interface to Microsoft Azure Files
 package azurefiles

backend/b2/api/types_test.go

@@ -42,11 +42,11 @@ func TestTimestampIsZero(t *testing.T) {
 }
 
 func TestTimestampEqual(t *testing.T) {
-	assert.False(t, emptyT.Equal(emptyT)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
+	assert.False(t, emptyT.Equal(emptyT))
 	assert.False(t, t0.Equal(emptyT))
 	assert.False(t, emptyT.Equal(t0))
 	assert.False(t, t0.Equal(t1))
 	assert.False(t, t1.Equal(t0))
-	assert.True(t, t0.Equal(t0)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
-	assert.True(t, t1.Equal(t1)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
+	assert.True(t, t0.Equal(t0))
+	assert.True(t, t1.Equal(t1))
 }

backend/b2/b2.go

@@ -60,7 +60,6 @@ const (
 	defaultChunkSize    = 96 * fs.Mebi
 	defaultUploadCutoff = 200 * fs.Mebi
 	largeFileCopyCutoff = 4 * fs.Gibi // 5E9 is the max
-	defaultMaxAge       = 24 * time.Hour
 )
 
 // Globals
@@ -102,7 +101,7 @@ below will cause b2 to return specific errors:
   * "force_cap_exceeded"
 
 These will be set in the "X-Bz-Test-Mode" header which is documented
-in the [b2 integrations checklist](https://www.backblaze.com/docs/cloud-storage-integration-checklist).`,
+in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration_checklist.html).`,
 			Default:  "",
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
@@ -194,12 +193,9 @@ Example:
 			Advanced: true,
 		}, {
 			Name: "download_auth_duration",
-			Help: `Time before the public link authorization token will expire in s or suffix ms|s|m|h|d.
-
-This is used in combination with "rclone link" for making files
-accessible to the public and sets the duration before the download
-authorization token will expire.
+			Help: `Time before the authorization token will expire in s or suffix ms|s|m|h|d.
 
+The duration before the download authorization token will expire.
 The minimum value is 1 second. The maximum value is one week.`,
 			Default:  fs.Duration(7 * 24 * time.Hour),
 			Advanced: true,
@@ -244,7 +240,7 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			// See: https://www.backblaze.com/docs/cloud-storage-files
+			// See: https://www.backblaze.com/b2/docs/files.html
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			// FIXME: allow /, but not leading, trailing or double
 			Default: (encoder.Display |
@@ -299,14 +295,13 @@ type Fs struct {
 
 // Object describes a b2 object
 type Object struct {
-	fs       *Fs               // what this object is part of
-	remote   string            // The remote path
-	id       string            // b2 id of the file
-	modTime  time.Time         // The modified time of the object if known
-	sha1     string            // SHA-1 hash if known
-	size     int64             // Size of the object
-	mimeType string            // Content-Type of the object
-	meta     map[string]string // The object metadata if known - may be nil - with lower case keys
+	fs       *Fs       // what this object is part of
+	remote   string    // The remote path
+	id       string    // b2 id of the file
+	modTime  time.Time // The modified time of the object if known
+	sha1     string    // SHA-1 hash if known
+	size     int64     // Size of the object
+	mimeType string    // Content-Type of the object
 }
 
 // ------------------------------------------------------------
@@ -364,7 +359,7 @@ var retryErrorCodes = []int{
 	504, // Gateway Time-out
 }
 
-// shouldRetryNoReauth returns a boolean as to whether this resp and err
+// shouldRetryNoAuth returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
@@ -1250,7 +1245,7 @@ func (f *Fs) deleteByID(ctx context.Context, ID, Name string) error {
 // if oldOnly is true then it deletes only non current files.
 //
 // Implemented here so we can make sure we delete old versions.
-func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden bool, deleteUnfinished bool, maxAge time.Duration) error {
+func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 	bucket, directory := f.split(dir)
 	if bucket == "" {
 		return errors.New("can't purge from root")
@@ -1268,7 +1263,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 		}
 	}
 	var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool {
-		return time.Since(time.Time(timestamp)) > maxAge
+		return time.Since(time.Time(timestamp)).Hours() > 24
 	}
 
 	// Delete Config.Transfers in parallel
@@ -1291,21 +1286,6 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 			}
 		}()
 	}
-	if oldOnly {
-		if deleteHidden && deleteUnfinished {
-			fs.Infof(f, "cleaning bucket %q of all hidden files, and pending multipart uploads older than %v", bucket, maxAge)
-		} else if deleteHidden {
-			fs.Infof(f, "cleaning bucket %q of all hidden files", bucket)
-		} else if deleteUnfinished {
-			fs.Infof(f, "cleaning bucket %q of pending multipart uploads older than %v", bucket, maxAge)
-		} else {
-			fs.Errorf(f, "cleaning bucket %q of nothing. This should never happen!", bucket)
-			return nil
-		}
-	} else {
-		fs.Infof(f, "cleaning bucket %q of all files", bucket)
-	}
-
 	last := ""
 	checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
 		if !isDirectory {
@@ -1316,14 +1296,14 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 			tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
 			if oldOnly && last != remote {
 				// Check current version of the file
-				if deleteHidden && object.Action == "hide" {
+				if object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
 					toBeDeleted <- object
-				} else if deleteUnfinished && object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
+				} else if object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
 					toBeDeleted <- object
 				} else {
-					fs.Debugf(remote, "Not deleting current version (id %q) %q dated %v (%v ago)", object.ID, object.Action, time.Time(object.UploadTimestamp).Local(), time.Since(time.Time(object.UploadTimestamp)))
+					fs.Debugf(remote, "Not deleting current version (id %q) %q", object.ID, object.Action)
 				}
 			} else {
 				fs.Debugf(remote, "Deleting (id %q)", object.ID)
@@ -1345,17 +1325,12 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 
 // Purge deletes all the files and directories including the old versions.
 func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purge(ctx, dir, false, false, false, defaultMaxAge)
+	return f.purge(ctx, dir, false)
 }
 
-// CleanUp deletes all hidden files and pending multipart uploads older than 24 hours.
+// CleanUp deletes all the hidden files.
 func (f *Fs) CleanUp(ctx context.Context) error {
-	return f.purge(ctx, "", true, true, true, defaultMaxAge)
-}
-
-// cleanUp deletes all hidden files and/or pending multipart uploads older than the specified age.
-func (f *Fs) cleanUp(ctx context.Context, deleteHidden bool, deleteUnfinished bool, maxAge time.Duration) (err error) {
-	return f.purge(ctx, "", true, deleteHidden, deleteUnfinished, maxAge)
+	return f.purge(ctx, "", true)
 }
 
 // copy does a server-side copy from dstObj <- srcObj
@@ -1567,7 +1542,7 @@ func (o *Object) Size() int64 {
 //
 // Make sure it is lower case.
 //
-// Remove unverified prefix - see https://www.backblaze.com/docs/cloud-storage-upload-files-with-the-native-api
+// Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
 // Some tools (e.g. Cyberduck) use this
 func cleanSHA1(sha1 string) string {
 	const unverified = "unverified:"
@@ -1594,14 +1569,7 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	o.size = Size
 	// Use the UploadTimestamp if can't get file info
 	o.modTime = time.Time(UploadTimestamp)
-	err = o.parseTimeString(Info[timeKey])
-	if err != nil {
-		return err
-	}
-	// For now, just set "mtime" in metadata
-	o.meta = make(map[string]string, 1)
-	o.meta["mtime"] = o.modTime.Format(time.RFC3339Nano)
-	return nil
+	return o.parseTimeString(Info[timeKey])
 }
 
 // decodeMetaData sets the metadata in the object from an api.File
@@ -1703,16 +1671,6 @@ func timeString(modTime time.Time) string {
 	return strconv.FormatInt(modTime.UnixNano()/1e6, 10)
 }
 
-// parseTimeStringHelper converts a decimal string number of milliseconds
-// elapsed since January 1, 1970 UTC into a time.Time
-func parseTimeStringHelper(timeString string) (time.Time, error) {
-	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
-	if err != nil {
-		return time.Time{}, err
-	}
-	return time.Unix(unixMilliseconds/1e3, (unixMilliseconds%1e3)*1e6).UTC(), nil
-}
-
 // parseTimeString converts a decimal string number of milliseconds
 // elapsed since January 1, 1970 UTC into a time.Time and stores it in
 // the modTime variable.
@@ -1720,12 +1678,12 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 	if timeString == "" {
 		return nil
 	}
-	modTime, err := parseTimeStringHelper(timeString)
+	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
 	if err != nil {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
 		return nil
 	}
-	o.modTime = modTime
+	o.modTime = time.Unix(unixMilliseconds/1e3, (unixMilliseconds%1e3)*1e6).UTC()
 	return nil
 }
 
@@ -1802,14 +1760,14 @@ func (file *openFile) Close() (err error) {
 
 	// Check to see we read the correct number of bytes
 	if file.o.Size() != file.bytes {
-		return fmt.Errorf("corrupted on transfer: lengths differ want %d vs got %d", file.o.Size(), file.bytes)
+		return fmt.Errorf("object corrupted on transfer - length mismatch (want %d got %d)", file.o.Size(), file.bytes)
 	}
 
 	// Check the SHA1
 	receivedSHA1 := file.o.sha1
 	calculatedSHA1 := fmt.Sprintf("%x", file.hash.Sum(nil))
 	if receivedSHA1 != "" && receivedSHA1 != calculatedSHA1 {
-		return fmt.Errorf("corrupted on transfer: SHA1 hashes differ want %q vs got %q", receivedSHA1, calculatedSHA1)
+		return fmt.Errorf("object corrupted on transfer - SHA1 mismatch (want %q got %q)", receivedSHA1, calculatedSHA1)
 	}
 
 	return nil
@@ -1879,14 +1837,6 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		ContentType:     resp.Header.Get("Content-Type"),
 		Info:            Info,
 	}
-
-	// Embryonic metadata support - just mtime
-	o.meta = make(map[string]string, 1)
-	modTime, err := parseTimeStringHelper(info.Info[timeKey])
-	if err == nil {
-		o.meta["mtime"] = modTime.Format(time.RFC3339Nano)
-	}
-
 	// When reading files from B2 via cloudflare using
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
@@ -1984,7 +1934,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		if err == nil {
 			fs.Debugf(o, "File is big enough for chunked streaming")
-			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil, options...)
+			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 			if err != nil {
 				o.fs.putRW(rw)
 				return err
@@ -2016,10 +1966,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return o.decodeMetaDataFileInfo(up.info)
 	}
 
-	modTime, err := o.getModTime(ctx, src, options)
-	if err != nil {
-		return err
-	}
+	modTime := src.ModTime(ctx)
 
 	calculatedSha1, _ := src.Hash(ctx, hash.SHA1)
 	if calculatedSha1 == "" {
@@ -2124,36 +2071,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return o.decodeMetaDataFileInfo(&response)
 }
 
-// Get modTime from the source; if --metadata is set, fetch the src metadata and get it from there.
-// When metadata support is added to b2, this method will need a more generic name
-func (o *Object) getModTime(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption) (time.Time, error) {
-	modTime := src.ModTime(ctx)
-
-	// Fetch metadata if --metadata is in use
-	meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
-	if err != nil {
-		return time.Time{}, fmt.Errorf("failed to read metadata from source object: %w", err)
-	}
-	// merge metadata into request and user metadata
-	for k, v := range meta {
-		k = strings.ToLower(k)
-		// For now, the only metadata we're concerned with is "mtime"
-		switch k {
-		case "mtime":
-			// mtime in meta overrides source ModTime
-			metaModTime, err := time.Parse(time.RFC3339Nano, v)
-			if err != nil {
-				fs.Debugf(o, "failed to parse metadata %s: %q: %v", k, v, err)
-			} else {
-				modTime = metaModTime
-			}
-		default:
-			// Do nothing for now
-		}
-	}
-	return modTime, nil
-}
-
 // OpenChunkWriter returns the chunk size and a ChunkWriter
 //
 // Pass in the remote and the src object
@@ -2185,7 +2102,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 		Concurrency: o.fs.opt.UploadConcurrency,
 		//LeavePartsOnError: o.fs.opt.LeavePartsOnError,
 	}
-	up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
+	up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil)
 	return info, up, err
 }
 
@@ -2323,56 +2240,8 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 	return bucket.LifecycleRules, nil
 }
 
-var cleanupHelp = fs.CommandHelp{
-	Name:  "cleanup",
-	Short: "Remove unfinished large file uploads.",
-	Long: `This command removes unfinished large file uploads of age greater than
-max-age, which defaults to 24 hours.
-
-Note that you can use --interactive/-i or --dry-run with this command to see what
-it would do.
-
-    rclone backend cleanup b2:bucket/path/to/object
-    rclone backend cleanup -o max-age=7w b2:bucket/path/to/object
-
-Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
-`,
-	Opts: map[string]string{
-		"max-age": "Max age of upload to delete",
-	},
-}
-
-func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	maxAge := defaultMaxAge
-	if opt["max-age"] != "" {
-		maxAge, err = fs.ParseDuration(opt["max-age"])
-		if err != nil {
-			return nil, fmt.Errorf("bad max-age: %w", err)
-		}
-	}
-	return nil, f.cleanUp(ctx, false, true, maxAge)
-}
-
-var cleanupHiddenHelp = fs.CommandHelp{
-	Name:  "cleanup-hidden",
-	Short: "Remove old versions of files.",
-	Long: `This command removes any old hidden versions of files.
-
-Note that you can use --interactive/-i or --dry-run with this command to see what
-it would do.
-
-    rclone backend cleanup-hidden b2:bucket/path/to/dir
-`,
-}
-
-func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	return nil, f.cleanUp(ctx, true, false, 0)
-}
-
 var commandHelp = []fs.CommandHelp{
 	lifecycleHelp,
-	cleanupHelp,
-	cleanupHiddenHelp,
 }
 
 // Command the backend to run a named command
@@ -2388,10 +2257,6 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 	switch name {
 	case "lifecycle":
 		return f.lifecycleCommand(ctx, name, arg, opt)
-	case "cleanup":
-		return f.cleanupCommand(ctx, name, arg, opt)
-	case "cleanup-hidden":
-		return f.cleanupHiddenCommand(ctx, name, arg, opt)
 	default:
 		return nil, fs.ErrorCommandNotFound
 	}

backend/b2/b2_internal_test.go

@@ -1,29 +1,15 @@
 package b2
 
 import (
-	"context"
-	"crypto/sha1"
-	"fmt"
-	"path"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/backend/b2/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/cache"
-	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/bucket"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/rclone/rclone/lib/version"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 // Test b2 string encoding
-// https://www.backblaze.com/docs/cloud-storage-native-api-string-encoding
+// https://www.backblaze.com/b2/docs/string_encoding.html
 
 var encodeTest = []struct {
 	fullyEncoded     string
@@ -184,303 +170,9 @@ func TestParseTimeString(t *testing.T) {
 
 }
 
-// Return a map of the headers in the options with keys stripped of the "x-bz-info-" prefix
-func OpenOptionToMetaData(options []fs.OpenOption) map[string]string {
-	var headers = make(map[string]string)
-	for _, option := range options {
-		k, v := option.Header()
-		k = strings.ToLower(k)
-		if strings.HasPrefix(k, headerPrefix) {
-			headers[k[len(headerPrefix):]] = v
-		}
-	}
-
-	return headers
-}
-
-func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string, chunkSize string) {
-	what := fmt.Sprintf("Size%s/UploadCutoff%s/ChunkSize%s", size, uploadCutoff, chunkSize)
-	t.Run(what, func(t *testing.T) {
-		ctx := context.Background()
-
-		ss := fs.SizeSuffix(0)
-		err := ss.Set(size)
-		require.NoError(t, err)
-		original := random.String(int(ss))
-
-		contents := fstest.Gz(t, original)
-		mimeType := "text/html"
-
-		if chunkSize != "" {
-			ss := fs.SizeSuffix(0)
-			err := ss.Set(chunkSize)
-			require.NoError(t, err)
-			_, err = f.SetUploadChunkSize(ss)
-			require.NoError(t, err)
-		}
-
-		if uploadCutoff != "" {
-			ss := fs.SizeSuffix(0)
-			err := ss.Set(uploadCutoff)
-			require.NoError(t, err)
-			_, err = f.SetUploadCutoff(ss)
-			require.NoError(t, err)
-		}
-
-		item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z"))
-		btime := time.Now()
-		metadata := fs.Metadata{
-			// Just mtime for now - limit to milliseconds since x-bz-info-src_last_modified_millis can't support any
-
-			"mtime": "2009-05-06T04:05:06.499Z",
-		}
-
-		// Need to specify HTTP options with the header prefix since they are passed as-is
-		options := []fs.OpenOption{
-			&fs.HTTPOption{Key: "X-Bz-Info-a", Value: "1"},
-			&fs.HTTPOption{Key: "X-Bz-Info-b", Value: "2"},
-		}
-
-		obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, mimeType, metadata, options...)
-		defer func() {
-			assert.NoError(t, obj.Remove(ctx))
-		}()
-		o := obj.(*Object)
-		gotMetadata, err := o.getMetaData(ctx)
-		require.NoError(t, err)
-
-		// X-Bz-Info-a & X-Bz-Info-b
-		optMetadata := OpenOptionToMetaData(options)
-		for k, v := range optMetadata {
-			got := gotMetadata.Info[k]
-			assert.Equal(t, v, got, k)
-		}
-
-		// mtime
-		for k, v := range metadata {
-			got := o.meta[k]
-			assert.Equal(t, v, got, k)
-		}
-
-		assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
-
-		// Modification time from the x-bz-info-src_last_modified_millis header
-		var mtime api.Timestamp
-		err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey]))
-		if err != nil {
-			fs.Debugf(o, "Bad "+timeHeader+" header: %v", err)
-		}
-		assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time")
-
-		// Upload time
-		gotBtime := time.Time(gotMetadata.UploadTimestamp)
-		dt := gotBtime.Sub(btime)
-		assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt))
-
-		t.Run("GzipEncoding", func(t *testing.T) {
-			// Test that the gzipped file we uploaded can be
-			// downloaded
-			checkDownload := func(wantContents string, wantSize int64, wantHash string) {
-				gotContents := fstests.ReadObject(ctx, t, o, -1)
-				assert.Equal(t, wantContents, gotContents)
-				assert.Equal(t, wantSize, o.Size())
-				gotHash, err := o.Hash(ctx, hash.SHA1)
-				require.NoError(t, err)
-				assert.Equal(t, wantHash, gotHash)
-			}
-
-			t.Run("NoDecompress", func(t *testing.T) {
-				checkDownload(contents, int64(len(contents)), sha1Sum(t, contents))
-			})
-		})
-	})
-}
-
-func (f *Fs) InternalTestMetadata(t *testing.T) {
-	// 1 kB regular file
-	f.internalTestMetadata(t, "1kiB", "", "")
-
-	// 10 MiB large file
-	f.internalTestMetadata(t, "10MiB", "6MiB", "6MiB")
-}
-
-func sha1Sum(t *testing.T, s string) string {
-	hash := sha1.Sum([]byte(s))
-	return fmt.Sprintf("%x", hash)
-}
-
-// This is adapted from the s3 equivalent.
-func (f *Fs) InternalTestVersions(t *testing.T) {
-	ctx := context.Background()
-
-	// Small pause to make the LastModified different since AWS
-	// only seems to track them to 1 second granularity
-	time.Sleep(2 * time.Second)
-
-	// Create an object
-	const dirName = "versions"
-	const fileName = dirName + "/" + "test-versions.txt"
-	contents := random.String(100)
-	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
-	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
-	defer func() {
-		assert.NoError(t, obj.Remove(ctx))
-	}()
-	objMetadata, err := obj.(*Object).getMetaData(ctx)
-	require.NoError(t, err)
-
-	// Small pause
-	time.Sleep(2 * time.Second)
-
-	// Remove it
-	assert.NoError(t, obj.Remove(ctx))
-
-	// Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity
-	time.Sleep(2 * time.Second)
-
-	// And create it with different size and contents
-	newContents := random.String(101)
-	newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z"))
-	newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true)
-	newObjMetadata, err := newObj.(*Object).getMetaData(ctx)
-	require.NoError(t, err)
-
-	t.Run("Versions", func(t *testing.T) {
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-
-		// Read the contents
-		entries, err := f.List(ctx, dirName)
-		require.NoError(t, err)
-		tests := 0
-		var fileNameVersion string
-		for _, entry := range entries {
-			t.Log(entry)
-			remote := entry.Remote()
-			if remote == fileName {
-				t.Run("ReadCurrent", func(t *testing.T) {
-					assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
-				})
-				tests++
-			} else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName {
-				t.Run("ReadVersion", func(t *testing.T) {
-					assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
-				})
-				assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time")
-				fileNameVersion = remote
-				tests++
-			}
-		}
-		assert.Equal(t, 2, tests, "object missing from listing")
-
-		// Check we can read the object with a version suffix
-		t.Run("NewObject", func(t *testing.T) {
-			o, err := f.NewObject(ctx, fileNameVersion)
-			require.NoError(t, err)
-			require.NotNil(t, o)
-			assert.Equal(t, int64(100), o.Size(), o.Remote())
-		})
-
-		// Check we can make a NewFs from that object with a version suffix
-		t.Run("NewFs", func(t *testing.T) {
-			newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion)
-			// Make sure --b2-versions is set in the config of the new remote
-			fs.Debugf(nil, "oldPath = %q", newPath)
-			lastColon := strings.LastIndex(newPath, ":")
-			require.True(t, lastColon >= 0)
-			newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:]
-			fs.Debugf(nil, "newPath = %q", newPath)
-			fNew, err := cache.Get(ctx, newPath)
-			// This should return pointing to a file
-			require.Equal(t, fs.ErrorIsFile, err)
-			require.NotNil(t, fNew)
-			// With the directory above
-			assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew)))
-		})
-	})
-
-	t.Run("VersionAt", func(t *testing.T) {
-		// We set --b2-version-at for this test so make sure we reset it at the end
-		defer func() {
-			f.opt.VersionAt = fs.Time{}
-		}()
-
-		var (
-			firstObjectTime  = time.Time(objMetadata.UploadTimestamp)
-			secondObjectTime = time.Time(newObjMetadata.UploadTimestamp)
-		)
-
-		for _, test := range []struct {
-			what     string
-			at       time.Time
-			want     []fstest.Item
-			wantErr  error
-			wantSize int64
-		}{
-			{
-				what:    "Before",
-				at:      firstObjectTime.Add(-time.Second),
-				want:    fstests.InternalTestFiles,
-				wantErr: fs.ErrorObjectNotFound,
-			},
-			{
-				what:     "AfterOne",
-				at:       firstObjectTime.Add(time.Second),
-				want:     append([]fstest.Item{item}, fstests.InternalTestFiles...),
-				wantSize: 100,
-			},
-			{
-				what:    "AfterDelete",
-				at:      secondObjectTime.Add(-time.Second),
-				want:    fstests.InternalTestFiles,
-				wantErr: fs.ErrorObjectNotFound,
-			},
-			{
-				what:     "AfterTwo",
-				at:       secondObjectTime.Add(time.Second),
-				want:     append([]fstest.Item{newItem}, fstests.InternalTestFiles...),
-				wantSize: 101,
-			},
-		} {
-			t.Run(test.what, func(t *testing.T) {
-				f.opt.VersionAt = fs.Time(test.at)
-				t.Run("List", func(t *testing.T) {
-					fstest.CheckListing(t, f, test.want)
-				})
-				// b2 NewObject doesn't work with VersionAt
-				//t.Run("NewObject", func(t *testing.T) {
-				//	gotObj, gotErr := f.NewObject(ctx, fileName)
-				//	assert.Equal(t, test.wantErr, gotErr)
-				//	if gotErr == nil {
-				//		assert.Equal(t, test.wantSize, gotObj.Size())
-				//	}
-				//})
-			})
-		}
-	})
-
-	t.Run("Cleanup", func(t *testing.T) {
-		require.NoError(t, f.cleanUp(ctx, true, false, 0))
-		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
-		fstest.CheckListing(t, f, items)
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-		fstest.CheckListing(t, f, items)
-	})
-
-	// Purge gets tested later
-}
-
 // -run TestIntegration/FsMkdir/FsPutFiles/Internal
 func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("Metadata", f.InternalTestMetadata)
-	t.Run("Versions", f.InternalTestVersions)
+	// Internal tests go here
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/b2/upload.go

@@ -1,6 +1,6 @@
 // Upload large files for b2
 //
-// Docs - https://www.backblaze.com/docs/cloud-storage-large-files
+// Docs - https://www.backblaze.com/b2/docs/large_files.html
 
 package b2
 
@@ -91,7 +91,7 @@ type largeUpload struct {
 // newLargeUpload starts an upload of object o from in with metadata in src
 //
 // If newInfo is set then metadata from that will be used instead of reading it from src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File, options ...fs.OpenOption) (up *largeUpload, err error) {
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	size := src.Size()
 	parts := 0
 	chunkSize := defaultChunkSize
@@ -104,6 +104,11 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 			parts++
 		}
 	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_start_large_file",
+	}
 	bucket, bucketPath := o.split()
 	bucketID, err := f.getBucketID(ctx, bucket)
 	if err != nil {
@@ -113,27 +118,12 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		BucketID: bucketID,
 		Name:     f.opt.Enc.FromStandardPath(bucketPath),
 	}
-	optionsToSend := make([]fs.OpenOption, 0, len(options))
 	if newInfo == nil {
-		modTime, err := o.getModTime(ctx, src, options)
-		if err != nil {
-			return nil, err
-		}
-
+		modTime := src.ModTime(ctx)
 		request.ContentType = fs.MimeType(ctx, src)
 		request.Info = map[string]string{
 			timeKey: timeString(modTime),
 		}
-		// Custom upload headers - remove header prefix since they are sent in the body
-		for _, option := range options {
-			k, v := option.Header()
-			k = strings.ToLower(k)
-			if strings.HasPrefix(k, headerPrefix) {
-				request.Info[k[len(headerPrefix):]] = v
-			} else {
-				optionsToSend = append(optionsToSend, option)
-			}
-		}
 		// Set the SHA1 if known
 		if !o.fs.opt.DisableCheckSum || doCopy {
 			if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
@@ -144,11 +134,6 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		request.ContentType = newInfo.ContentType
 		request.Info = newInfo.Info
 	}
-	opts := rest.Opts{
-		Method:  "POST",
-		Path:    "/b2_start_large_file",
-		Options: optionsToSend,
-	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)

backend/cache/cache.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 // Package cache implements a virtual provider to cache existing remotes.
 package cache
@@ -409,16 +410,18 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
 			if err != nil {
 				return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
 			}
-		} else if opt.PlexPassword != "" && opt.PlexUsername != "" {
-			decPass, err := obscure.Reveal(opt.PlexPassword)
-			if err != nil {
-				decPass = opt.PlexPassword
-			}
-			f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, opt.PlexInsecure, func(token string) {
-				m.Set("plex_token", token)
-			})
-			if err != nil {
-				return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
+		} else {
+			if opt.PlexPassword != "" && opt.PlexUsername != "" {
+				decPass, err := obscure.Reveal(opt.PlexPassword)
+				if err != nil {
+					decPass = opt.PlexPassword
+				}
+				f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, opt.PlexInsecure, func(token string) {
+					m.Set("plex_token", token)
+				})
+				if err != nil {
+					return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
+				}
 			}
 		}
 	}

backend/cache/cache_internal_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js && !race
+// +build !plan9,!js,!race
 
 package cache_test
 
@@ -10,6 +11,7 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
+	"log"
 	"math/rand"
 	"os"
 	"path"
@@ -28,11 +30,10 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/lib/random"
-	"github.com/rclone/rclone/vfs/vfscommon"
+	"github.com/rclone/rclone/vfs/vfsflags"
 	"github.com/stretchr/testify/require"
 )
 
@@ -92,7 +93,7 @@ func TestMain(m *testing.M) {
 	goflag.Parse()
 	var rc int
 
-	fs.Logf(nil, "Running with the following params: \n remote: %v", remoteName)
+	log.Printf("Running with the following params: \n remote: %v", remoteName)
 	runInstance = newRun()
 	rc = m.Run()
 	os.Exit(rc)
@@ -122,10 +123,10 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
 
 /* TODO: is this testing something?
 func TestInternalVfsCache(t *testing.T) {
-	vfscommon.Opt.DirCacheTime = time.Second * 30
+	vfsflags.Opt.DirCacheTime = time.Second * 30
 	testSize := int64(524288000)
 
-	vfscommon.Opt.CacheMode = vfs.CacheModeWrites
+	vfsflags.Opt.CacheMode = vfs.CacheModeWrites
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -337,7 +338,7 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 
 func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
-	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second)
+	vfsflags.Opt.DirCacheTime = time.Second
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
@@ -367,7 +368,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 func TestInternalLargeWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
-	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second)
+	vfsflags.Opt.DirCacheTime = time.Second
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
@@ -407,7 +408,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	// update in the wrapped fs
 	originalSize, err := runInstance.size(t, rootFs, "data.bin")
 	require.NoError(t, err)
-	fs.Logf(nil, "original size: %v", originalSize)
+	log.Printf("original size: %v", originalSize)
 
 	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
@@ -416,7 +417,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	if runInstance.rootIsCrypt {
 		data2, err = base64.StdEncoding.DecodeString(cryptedText3Base64)
 		require.NoError(t, err)
-		expectedSize++ // FIXME newline gets in, likely test data issue
+		expectedSize = expectedSize + 1 // FIXME newline gets in, likely test data issue
 	} else {
 		data2 = []byte("test content")
 	}
@@ -424,7 +425,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	err = o.Update(context.Background(), bytes.NewReader(data2), objInfo)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(data2)), o.Size())
-	fs.Logf(nil, "updated size: %v", len(data2))
+	log.Printf("updated size: %v", len(data2))
 
 	// get a new instance from the cache
 	if runInstance.wrappedIsExternal {
@@ -484,49 +485,49 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	err = runInstance.retryBlock(func() error {
 		li, err := runInstance.list(t, rootFs, "test")
 		if err != nil {
-			fs.Logf(nil, "err: %v", err)
+			log.Printf("err: %v", err)
 			return err
 		}
 		if len(li) != 2 {
-			fs.Logf(nil, "not expected listing /test: %v", li)
+			log.Printf("not expected listing /test: %v", li)
 			return fmt.Errorf("not expected listing /test: %v", li)
 		}
 
 		li, err = runInstance.list(t, rootFs, "test/one")
 		if err != nil {
-			fs.Logf(nil, "err: %v", err)
+			log.Printf("err: %v", err)
 			return err
 		}
 		if len(li) != 0 {
-			fs.Logf(nil, "not expected listing /test/one: %v", li)
+			log.Printf("not expected listing /test/one: %v", li)
 			return fmt.Errorf("not expected listing /test/one: %v", li)
 		}
 
 		li, err = runInstance.list(t, rootFs, "test/second")
 		if err != nil {
-			fs.Logf(nil, "err: %v", err)
+			log.Printf("err: %v", err)
 			return err
 		}
 		if len(li) != 1 {
-			fs.Logf(nil, "not expected listing /test/second: %v", li)
+			log.Printf("not expected listing /test/second: %v", li)
 			return fmt.Errorf("not expected listing /test/second: %v", li)
 		}
 		if fi, ok := li[0].(os.FileInfo); ok {
 			if fi.Name() != "data.bin" {
-				fs.Logf(nil, "not expected name: %v", fi.Name())
+				log.Printf("not expected name: %v", fi.Name())
 				return fmt.Errorf("not expected name: %v", fi.Name())
 			}
 		} else if di, ok := li[0].(fs.DirEntry); ok {
 			if di.Remote() != "test/second/data.bin" {
-				fs.Logf(nil, "not expected remote: %v", di.Remote())
+				log.Printf("not expected remote: %v", di.Remote())
 				return fmt.Errorf("not expected remote: %v", di.Remote())
 			}
 		} else {
-			fs.Logf(nil, "unexpected listing: %v", li)
+			log.Printf("unexpected listing: %v", li)
 			return fmt.Errorf("unexpected listing: %v", li)
 		}
 
-		fs.Logf(nil, "complete listing: %v", li)
+		log.Printf("complete listing: %v", li)
 		return nil
 	}, 12, time.Second*10)
 	require.NoError(t, err)
@@ -576,43 +577,43 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	err = runInstance.retryBlock(func() error {
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
 		if !found {
-			fs.Logf(nil, "not found /test")
+			log.Printf("not found /test")
 			return fmt.Errorf("not found /test")
 		}
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one")))
 		if !found {
-			fs.Logf(nil, "not found /test/one")
+			log.Printf("not found /test/one")
 			return fmt.Errorf("not found /test/one")
 		}
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one"), runInstance.encryptRemoteIfNeeded(t, "test2")))
 		if !found {
-			fs.Logf(nil, "not found /test/one/test2")
+			log.Printf("not found /test/one/test2")
 			return fmt.Errorf("not found /test/one/test2")
 		}
 		li, err := runInstance.list(t, rootFs, "test/one")
 		if err != nil {
-			fs.Logf(nil, "err: %v", err)
+			log.Printf("err: %v", err)
 			return err
 		}
 		if len(li) != 1 {
-			fs.Logf(nil, "not expected listing /test/one: %v", li)
+			log.Printf("not expected listing /test/one: %v", li)
 			return fmt.Errorf("not expected listing /test/one: %v", li)
 		}
 		if fi, ok := li[0].(os.FileInfo); ok {
 			if fi.Name() != "test2" {
-				fs.Logf(nil, "not expected name: %v", fi.Name())
+				log.Printf("not expected name: %v", fi.Name())
 				return fmt.Errorf("not expected name: %v", fi.Name())
 			}
 		} else if di, ok := li[0].(fs.DirEntry); ok {
 			if di.Remote() != "test/one/test2" {
-				fs.Logf(nil, "not expected remote: %v", di.Remote())
+				log.Printf("not expected remote: %v", di.Remote())
 				return fmt.Errorf("not expected remote: %v", di.Remote())
 			}
 		} else {
-			fs.Logf(nil, "unexpected listing: %v", li)
+			log.Printf("unexpected listing: %v", li)
 			return fmt.Errorf("unexpected listing: %v", li)
 		}
-		fs.Logf(nil, "complete listing /test/one/test2")
+		log.Printf("complete listing /test/one/test2")
 		return nil
 	}, 12, time.Second*10)
 	require.NoError(t, err)
@@ -707,7 +708,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 
 func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	id := fmt.Sprintf("tieer%v", time.Now().Unix())
-	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second * 4) // needs to be lower than the defined
+	vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -742,7 +743,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
 }
 
 func TestInternalBug2117(t *testing.T) {
-	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second * 10)
+	vfsflags.Opt.DirCacheTime = time.Second * 10
 
 	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
@@ -770,24 +771,24 @@ func TestInternalBug2117(t *testing.T) {
 
 	di, err := runInstance.list(t, rootFs, "test/dir1/dir2")
 	require.NoError(t, err)
-	fs.Logf(nil, "len: %v", len(di))
+	log.Printf("len: %v", len(di))
 	require.Len(t, di, 1)
 
 	time.Sleep(time.Second * 30)
 
 	di, err = runInstance.list(t, rootFs, "test/dir1/dir2")
 	require.NoError(t, err)
-	fs.Logf(nil, "len: %v", len(di))
+	log.Printf("len: %v", len(di))
 	require.Len(t, di, 1)
 
 	di, err = runInstance.list(t, rootFs, "test/dir1")
 	require.NoError(t, err)
-	fs.Logf(nil, "len: %v", len(di))
+	log.Printf("len: %v", len(di))
 	require.Len(t, di, 4)
 
 	di, err = runInstance.list(t, rootFs, "test")
 	require.NoError(t, err)
-	fs.Logf(nil, "len: %v", len(di))
+	log.Printf("len: %v", len(di))
 	require.Len(t, di, 4)
 }
 
@@ -828,7 +829,7 @@ func newRun() *run {
 	} else {
 		r.tmpUploadDir = uploadDir
 	}
-	fs.Logf(nil, "Temp Upload Dir: %v", r.tmpUploadDir)
+	log.Printf("Temp Upload Dir: %v", r.tmpUploadDir)
 
 	return r
 }
@@ -849,8 +850,8 @@ func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string {
 func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, flags map[string]string) (fs.Fs, *cache.Persistent) {
 	fstest.Initialise()
 	remoteExists := false
-	for _, s := range config.GetRemotes() {
-		if s.Name == remote {
+	for _, s := range config.FileSections() {
+		if s == remote {
 			remoteExists = true
 		}
 	}
@@ -874,12 +875,12 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	cacheRemote := remote
 	if !remoteExists {
 		localRemote := remote + "-local"
-		config.FileSetValue(localRemote, "type", "local")
-		config.FileSetValue(localRemote, "nounc", "true")
+		config.FileSet(localRemote, "type", "local")
+		config.FileSet(localRemote, "nounc", "true")
 		m.Set("type", "cache")
 		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
-		remoteType := config.GetValue(remote, "type")
+		remoteType := config.FileGet(remote, "type")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -890,14 +891,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 				m.Set("password", cryptPassword1)
 				m.Set("password2", cryptPassword2)
 			}
-			remoteRemote := config.GetValue(remote, "remote")
+			remoteRemote := config.FileGet(remote, "remote")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := config.GetValue(remoteWrapping, "type")
+			remoteType := config.FileGet(remoteWrapping, "type")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil
@@ -934,7 +935,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 
 	if purge {
-		_ = operations.Purge(context.Background(), f, "")
+		_ = f.Features().Purge(context.Background(), "")
+		require.NoError(t, err)
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
@@ -947,7 +949,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 }
 
 func (r *run) cleanupFs(t *testing.T, f fs.Fs) {
-	err := operations.Purge(context.Background(), f, "")
+	err := f.Features().Purge(context.Background(), "")
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
 	require.NoError(t, err)
@@ -1191,7 +1193,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 func (r *run) cleanSize(t *testing.T, size int64) int64 {
 	if r.rootIsCrypt {
 		denominator := int64(65536 + 16)
-		size -= 32
+		size = size - 32
 		quotient := size / denominator
 		remainder := size % denominator
 		return (quotient*65536 + remainder - 16)

backend/cache/cache_test.go

@@ -1,6 +1,7 @@
 // Test Cache filesystem interface
 
 //go:build !plan9 && !js && !race
+// +build !plan9,!js,!race
 
 package cache_test
 
@@ -15,11 +16,10 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                      "TestCache:",
-		NilObject:                       (*cache.Object)(nil),
-		UnimplementableFsMethods:        []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter", "DirSetModTime", "MkdirMetadata"},
-		UnimplementableObjectMethods:    []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata", "SetMetadata"},
-		UnimplementableDirectoryMethods: []string{"Metadata", "SetMetadata", "SetModTime"},
-		SkipInvalidUTF8:                 true, // invalid UTF-8 confuses the cache
+		RemoteName:                   "TestCache:",
+		NilObject:                    (*cache.Object)(nil),
+		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter"},
+		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"},
+		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})
 }

backend/cache/cache_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9 || js
+// +build plan9 js
 
-// Package cache implements a virtual provider to cache existing remotes.
 package cache

backend/cache/cache_upload_test.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js && !race
+// +build !plan9,!js,!race
 
 package cache_test
 

backend/cache/directory.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/handle.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 
@@ -118,7 +119,7 @@ func (r *Handle) startReadWorkers() {
 	r.scaleWorkers(totalWorkers)
 }
 
-// scaleWorkers will increase the worker pool count by the provided amount
+// scaleOutWorkers will increase the worker pool count by the provided amount
 func (r *Handle) scaleWorkers(desired int) {
 	current := r.workers
 	if current == desired {
@@ -208,7 +209,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 	offset := chunkStart % int64(r.cacheFs().opt.ChunkSize)
 
 	// we align the start offset of the first chunk to a likely chunk in the storage
-	chunkStart -= offset
+	chunkStart = chunkStart - offset
 	r.queueOffset(chunkStart)
 	found := false
 
@@ -327,7 +328,7 @@ func (r *Handle) Seek(offset int64, whence int) (int64, error) {
 
 	chunkStart := r.offset - (r.offset % int64(r.cacheFs().opt.ChunkSize))
 	if chunkStart >= int64(r.cacheFs().opt.ChunkSize) {
-		chunkStart -= int64(r.cacheFs().opt.ChunkSize)
+		chunkStart = chunkStart - int64(r.cacheFs().opt.ChunkSize)
 	}
 	r.queueOffset(chunkStart)
 
@@ -415,8 +416,10 @@ func (w *worker) run() {
 					continue
 				}
 			}
-		} else if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
-			continue
+		} else {
+			if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
+				continue
+			}
 		}
 
 		chunkEnd := chunkStart + int64(w.r.cacheFs().opt.ChunkSize)

backend/cache/object.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/plex.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/storage_memory.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/storage_persistent.go

@@ -1,4 +1,5 @@
 //go:build !plan9 && !js
+// +build !plan9,!js
 
 package cache
 

backend/cache/utils_test.go

@@ -1,6 +1,3 @@
-//go:build !plan9 && !js
-// +build !plan9,!js
-
 package cache
 
 import bolt "go.etcd.io/bbolt"

backend/chunker/chunker.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/lib/encoder"
 )
 
 // Chunker's composite files have one or more chunks
@@ -102,10 +101,8 @@ var (
 //
 // And still chunker's primary function is to chunk large files
 // rather than serve as a generic metadata container.
-const (
-	maxMetadataSize        = 1023
-	maxMetadataSizeWritten = 255
-)
+const maxMetadataSize = 1023
+const maxMetadataSizeWritten = 255
 
 // Current/highest supported metadata format.
 const metadataVersion = 2
@@ -308,6 +305,7 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		root: rpath,
 		opt:  *opt,
 	}
+	cache.PinUntilFinalized(f.base, f)
 	f.dirSort = true // processEntries requires that meta Objects prerun data chunks atm.
 
 	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType, opt.Transactions); err != nil {
@@ -319,15 +317,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// i.e. `rpath` does not exist in the wrapped remote, but chunker
 	// detects a composite file because it finds the first chunk!
 	// (yet can't satisfy fstest.CheckListing, will ignore)
-	if err == nil && !f.useMeta {
+	if err == nil && !f.useMeta && strings.Contains(rpath, "/") {
 		firstChunkPath := f.makeChunkName(remotePath, 0, "", "")
-		newBase, testErr := cache.Get(ctx, baseName+firstChunkPath)
+		_, testErr := cache.Get(ctx, baseName+firstChunkPath)
 		if testErr == fs.ErrorIsFile {
-			f.base = newBase
 			err = testErr
 		}
 	}
-	cache.PinUntilFinalized(f.base, f)
 
 	// Correct root if definitely pointing to a file
 	if err == fs.ErrorIsFile {
@@ -342,18 +338,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// Note 2: features.Fill() points features.PutStream to our PutStream,
 	// but features.Mask() will nullify it if wrappedFs does not have it.
 	f.features = (&fs.Features{
-		CaseInsensitive:          true,
-		DuplicateFiles:           true,
-		ReadMimeType:             false, // Object.MimeType not supported
-		WriteMimeType:            true,
-		BucketBased:              true,
-		CanHaveEmptyDirectories:  true,
-		ServerSideAcrossConfigs:  true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
+		CaseInsensitive:         true,
+		DuplicateFiles:          true,
+		ReadMimeType:            false, // Object.MimeType not supported
+		WriteMimeType:           true,
+		BucketBased:             true,
+		CanHaveEmptyDirectories: true,
+		ServerSideAcrossConfigs: true,
 	}).Fill(ctx, f).Mask(ctx, baseFs).WrapsFs(f, baseFs)
 
 	f.features.Disable("ListR") // Recursive listing may cause chunker skip files
@@ -830,7 +821,8 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 			}
 		case fs.Directory:
 			isSubdir[entry.Remote()] = true
-			wrapDir := fs.NewDirWrapper(entry.Remote(), entry)
+			wrapDir := fs.NewDirCopy(ctx, entry)
+			wrapDir.SetRemote(entry.Remote())
 			tempEntries = append(tempEntries, wrapDir)
 		default:
 			if f.opt.FailHard {
@@ -963,11 +955,6 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 		}
 		if caseInsensitive {
 			sameMain = strings.EqualFold(mainRemote, remote)
-			if sameMain && f.base.Features().IsLocal {
-				// on local, make sure the EqualFold still holds true when accounting for encoding.
-				// sometimes paths with special characters will only normalize the same way in Standard Encoding.
-				sameMain = strings.EqualFold(encoder.OS.FromStandardPath(mainRemote), encoder.OS.FromStandardPath(remote))
-			}
 		} else {
 			sameMain = mainRemote == remote
 		}
@@ -981,13 +968,13 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 			}
 			continue
 		}
-		// fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
+		//fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
 		if err := o.addChunk(entry, chunkNo); err != nil {
 			return nil, err
 		}
 	}
 
-	if o.main == nil && len(o.chunks) == 0 {
+	if o.main == nil && (o.chunks == nil || len(o.chunks) == 0) {
 		// Scanning hasn't found data chunks with conforming names.
 		if f.useMeta || quickScan {
 			// Metadata is required but absent and there are no chunks.
@@ -1143,8 +1130,8 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 // put implements Put, PutStream, PutUnchecked, Update
 func (f *Fs) put(
 	ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption,
-	basePut putFn, action string, target fs.Object,
-) (obj fs.Object, err error) {
+	basePut putFn, action string, target fs.Object) (obj fs.Object, err error) {
+
 	// Perform consistency checks
 	if err := f.forbidChunk(src, remote); err != nil {
 		return nil, fmt.Errorf("%s refused: %w", action, err)
@@ -1584,14 +1571,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.base.Mkdir(ctx, dir)
 }
 
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	if do := f.base.Features().MkdirMetadata; do != nil {
-		return do(ctx, dir, metadata)
-	}
-	return nil, fs.ErrorNotImplemented
-}
-
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -1909,14 +1888,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return do(ctx, srcFs.base, srcRemote, dstRemote)
 }
 
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	if do := f.base.Features().DirSetModTime; do != nil {
-		return do(ctx, dir, modTime)
-	}
-	return fs.ErrorNotImplemented
-}
-
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -1965,7 +1936,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		return
 	}
 	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-		// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
+		//fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
 		if entryType == fs.EntryObject {
 			mainPath, _, _, xactID := f.parseChunkName(path)
 			metaXactID := ""
@@ -2577,8 +2548,6 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)

backend/chunker/chunker_test.go

@@ -36,7 +36,6 @@ func TestIntegration(t *testing.T) {
 			"GetTier",
 			"SetTier",
 			"Metadata",
-			"SetMetadata",
 		},
 		UnimplementableFsMethods: []string{
 			"PublicLink",

backend/combine/combine.go

@@ -222,23 +222,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 	}
 	// check features
 	var features = (&fs.Features{
-		CaseInsensitive:          true,
-		DuplicateFiles:           false,
-		ReadMimeType:             true,
-		WriteMimeType:            true,
-		CanHaveEmptyDirectories:  true,
-		BucketBased:              true,
-		SetTier:                  true,
-		GetTier:                  true,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
-		PartialUploads:           true,
+		CaseInsensitive:         true,
+		DuplicateFiles:          false,
+		ReadMimeType:            true,
+		WriteMimeType:           true,
+		CanHaveEmptyDirectories: true,
+		BucketBased:             true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}).Fill(ctx, f)
 	canMove := true
 	for _, u := range f.upstreams {
@@ -445,32 +440,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return u.f.Mkdir(ctx, uRemote)
 }
 
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	u, uRemote, err := f.findUpstream(dir)
-	if err != nil {
-		return nil, err
-	}
-	do := u.f.Features().MkdirMetadata
-	if do == nil {
-		return nil, fs.ErrorNotImplemented
-	}
-	newDir, err := do(ctx, uRemote, metadata)
-	if err != nil {
-		return nil, err
-	}
-	entries := fs.DirEntries{newDir}
-	entries, err = u.wrapEntries(ctx, entries)
-	if err != nil {
-		return nil, err
-	}
-	newDir, ok := entries[0].(fs.Directory)
-	if !ok {
-		return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0])
-	}
-	return newDir, nil
-}
-
 // purge the upstream or fallback to a slow way
 func (u *upstream) purge(ctx context.Context, dir string) (err error) {
 	if do := u.f.Features().Purge; do != nil {
@@ -786,11 +755,12 @@ func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.D
 		case fs.Object:
 			entries[i] = u.newObject(x)
 		case fs.Directory:
-			newPath, err := u.pathAdjustment.do(x.Remote())
+			newDir := fs.NewDirCopy(ctx, x)
+			newPath, err := u.pathAdjustment.do(newDir.Remote())
 			if err != nil {
 				return nil, err
 			}
-			newDir := fs.NewDirWrapper(newPath, x)
+			newDir.SetRemote(newPath)
 			entries[i] = newDir
 		default:
 			return nil, fmt.Errorf("unknown entry type %T", entry)
@@ -813,7 +783,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if f.root == "" && dir == "" {
 		entries = make(fs.DirEntries, 0, len(f.upstreams))
 		for combineDir := range f.upstreams {
-			d := fs.NewLimitedDirWrapper(combineDir, fs.NewDir(combineDir, f.when))
+			d := fs.NewDir(combineDir, f.when)
 			entries = append(entries, d)
 		}
 		return entries, nil
@@ -995,22 +965,6 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	return do(ctx, uDirs)
 }
 
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	u, uDir, err := f.findUpstream(dir)
-	if err != nil {
-		return err
-	}
-	if uDir == "" {
-		fs.Debugf(dir, "Can't set modtime on upstream root. skipping.")
-		return nil
-	}
-	if do := u.f.Features().DirSetModTime; do != nil {
-		return do(ctx, uDir, modTime)
-	}
-	return fs.ErrorNotImplemented
-}
-
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -1119,17 +1073,6 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
-// SetMetadata sets metadata for an Object
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	do, ok := o.Object.(fs.SetMetadataer)
-	if !ok {
-		return fs.ErrorNotImplemented
-	}
-	return do.SetMetadata(ctx, metadata)
-}
-
 // SetTier performs changing storage tier of the Object if
 // multiple storage classes supported
 func (o *Object) SetTier(tier string) error {
@@ -1156,8 +1099,6 @@ var (
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.OpenWriterAter  = (*Fs)(nil)
 	_ fs.FullObject      = (*Object)(nil)

backend/compress/compress.go

@@ -38,7 +38,6 @@ import (
 const (
 	initialChunkSize = 262144  // Initial and max sizes of chunks when reading parts of the file. Currently
 	maxChunkSize     = 8388608 // at 256 KiB and 8 MiB.
-	chunkStreams     = 0       // Streams to use for reading
 
 	bufferSize          = 8388608
 	heuristicBytes      = 1048576
@@ -184,23 +183,18 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:          true,
-		DuplicateFiles:           false,
-		ReadMimeType:             false,
-		WriteMimeType:            false,
-		GetTier:                  true,
-		SetTier:                  true,
-		BucketBased:              true,
-		CanHaveEmptyDirectories:  true,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
-		PartialUploads:           true,
+		CaseInsensitive:         true,
+		DuplicateFiles:          false,
+		ReadMimeType:            false,
+		WriteMimeType:           false,
+		GetTier:                 true,
+		SetTier:                 true,
+		BucketBased:             true,
+		CanHaveEmptyDirectories: true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 	// We support reading MIME types no matter the wrapped fs
 	f.features.ReadMimeType = true
@@ -456,7 +450,7 @@ func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.Mul
 		if err != nil {
 			fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 		}
-		return fmt.Errorf("corrupted on transfer: %v compressed hashes differ src(%s) %q vs dst(%s) %q", ht, f.Fs, srcHash, o.Fs(), dstHash)
+		return fmt.Errorf("corrupted on transfer: %v compressed hashes differ %q vs %q", ht, srcHash, dstHash)
 	}
 	return nil
 }
@@ -790,14 +784,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.Fs.Mkdir(ctx, dir)
 }
 
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	if do := f.Fs.Features().MkdirMetadata; do != nil {
-		return do(ctx, dir, metadata)
-	}
-	return nil, fs.ErrorNotImplemented
-}
-
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -941,14 +927,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return do(ctx, srcFs.Fs, srcRemote, dstRemote)
 }
 
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	if do := f.Fs.Features().DirSetModTime; do != nil {
-		return do(ctx, dir, modTime)
-	}
-	return fs.ErrorNotImplemented
-}
-
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -1287,17 +1265,6 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
-// SetMetadata sets metadata for an Object
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	do, ok := o.Object.(fs.SetMetadataer)
-	if !ok {
-		return fs.ErrorNotImplemented
-	}
-	return do.SetMetadata(ctx, metadata)
-}
-
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
@@ -1363,7 +1330,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 		}
 	}
 	// Get a chunkedreader for the wrapped object
-	chunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize, chunkStreams)
+	chunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize)
 	// Get file handle
 	var file io.Reader
 	if offset != 0 {
@@ -1530,8 +1497,6 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.UnWrapper       = (*Fs)(nil)

backend/crypt/cipher.go

@@ -329,7 +329,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
 	for _, runeValue := range plaintext {
 		dir += int(runeValue)
 	}
-	dir %= 256
+	dir = dir % 256
 
 	// We'll use this number to store in the result filename...
 	var result bytes.Buffer
@@ -450,7 +450,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if pos >= 26 {
 				pos -= 6
 			}
-			pos -= thisdir
+			pos = pos - thisdir
 			if pos < 0 {
 				pos += 52
 			}
@@ -888,7 +888,7 @@ func (fh *decrypter) fillBuffer() (err error) {
 		fs.Errorf(nil, "crypt: ignoring: %v", ErrorEncryptedBadBlock)
 		// Zero out the bad block and continue
 		for i := range (*fh.buf)[:n] {
-			fh.buf[i] = 0
+			(*fh.buf)[i] = 0
 		}
 	}
 	fh.bufIndex = 0

backend/crypt/crypt.go

@@ -130,16 +130,6 @@ trying to recover an encrypted file with errors and it is desired to
 recover as much of the file as possible.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "strict_names",
-			Help: `If set, this will raise an error when crypt comes across a filename that can't be decrypted.
-
-(By default, rclone will just log a NOTICE and continue as normal.)
-This can happen if encrypted and unencrypted files are stored in the same
-directory (which is not recommended.) It may also indicate a more serious
-problem that should be investigated.`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name: "filename_encoding",
 			Help: `How to encode the encrypted filename to text string.
@@ -273,24 +263,19 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:          !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
-		DuplicateFiles:           true,
-		ReadMimeType:             false, // MimeTypes not supported with crypt
-		WriteMimeType:            false,
-		BucketBased:              true,
-		CanHaveEmptyDirectories:  true,
-		SetTier:                  true,
-		GetTier:                  true,
-		ServerSideAcrossConfigs:  opt.ServerSideAcrossConfigs,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
-		PartialUploads:           true,
+		CaseInsensitive:         !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
+		DuplicateFiles:          true,
+		ReadMimeType:            false, // MimeTypes not supported with crypt
+		WriteMimeType:           false,
+		BucketBased:             true,
+		CanHaveEmptyDirectories: true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -309,7 +294,6 @@ type Options struct {
 	PassBadBlocks           bool   `config:"pass_bad_blocks"`
 	FilenameEncoding        string `config:"filename_encoding"`
 	Suffix                  string `config:"suffix"`
-	StrictNames             bool   `config:"strict_names"`
 }
 
 // Fs represents a wrapped fs.Fs
@@ -344,64 +328,45 @@ func (f *Fs) String() string {
 }
 
 // Encrypt an object file name to entries.
-func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) error {
+func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
 	remote := obj.Remote()
 	decryptedRemote, err := f.cipher.DecryptFileName(remote)
 	if err != nil {
-		if f.opt.StrictNames {
-			return fmt.Errorf("%s: undecryptable file name detected: %v", remote, err)
-		}
-		fs.Logf(remote, "Skipping undecryptable file name: %v", err)
-		return nil
+		fs.Debugf(remote, "Skipping undecryptable file name: %v", err)
+		return
 	}
 	if f.opt.ShowMapping {
 		fs.Logf(decryptedRemote, "Encrypts to %q", remote)
 	}
 	*entries = append(*entries, f.newObject(obj))
-	return nil
 }
 
 // Encrypt a directory file name to entries.
-func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) error {
+func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) {
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
-		if f.opt.StrictNames {
-			return fmt.Errorf("%s: undecryptable dir name detected: %v", remote, err)
-		}
-		fs.Logf(remote, "Skipping undecryptable dir name: %v", err)
-		return nil
+		fs.Debugf(remote, "Skipping undecryptable dir name: %v", err)
+		return
 	}
 	if f.opt.ShowMapping {
 		fs.Logf(decryptedRemote, "Encrypts to %q", remote)
 	}
 	*entries = append(*entries, f.newDir(ctx, dir))
-	return nil
 }
 
 // Encrypt some directory entries.  This alters entries returning it as newEntries.
 func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntries fs.DirEntries, err error) {
 	newEntries = entries[:0] // in place filter
-	errors := 0
-	var firsterr error
 	for _, entry := range entries {
 		switch x := entry.(type) {
 		case fs.Object:
-			err = f.add(&newEntries, x)
+			f.add(&newEntries, x)
 		case fs.Directory:
-			err = f.addDir(ctx, &newEntries, x)
+			f.addDir(ctx, &newEntries, x)
 		default:
 			return nil, fmt.Errorf("unknown object type %T", entry)
 		}
-		if err != nil {
-			errors++
-			if firsterr == nil {
-				firsterr = err
-			}
-		}
-	}
-	if firsterr != nil {
-		return nil, fmt.Errorf("there were %v undecryptable name errors. first error: %v", errors, firsterr)
 	}
 	return newEntries, nil
 }
@@ -520,7 +485,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 				if err != nil {
 					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 				}
-				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hashes differ src(%s) %q vs dst(%s) %q", ht, f.Fs, srcHash, o.Fs(), dstHash)
+				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
 			}
 			fs.Debugf(src, "%v = %s OK", ht, srcHash)
 		}
@@ -555,37 +520,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.Fs.Mkdir(ctx, f.cipher.EncryptDirName(dir))
 }
 
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	do := f.Fs.Features().MkdirMetadata
-	if do == nil {
-		return nil, fs.ErrorNotImplemented
-	}
-	newDir, err := do(ctx, f.cipher.EncryptDirName(dir), metadata)
-	if err != nil {
-		return nil, err
-	}
-	var entries = make(fs.DirEntries, 0, 1)
-	err = f.addDir(ctx, &entries, newDir)
-	if err != nil {
-		return nil, err
-	}
-	newDir, ok := entries[0].(fs.Directory)
-	if !ok {
-		return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0])
-	}
-	return newDir, nil
-}
-
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	do := f.Fs.Features().DirSetModTime
-	if do == nil {
-		return fs.ErrorNotImplemented
-	}
-	return do(ctx, f.cipher.EncryptDirName(dir), modTime)
-}
-
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -827,7 +761,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	}
 	out := make([]fs.Directory, len(dirs))
 	for i, dir := range dirs {
-		out[i] = fs.NewDirWrapper(f.cipher.EncryptDirName(dir.Remote()), dir)
+		out[i] = fs.NewDirCopy(ctx, dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
 	}
 	return do(ctx, out)
 }
@@ -1063,14 +997,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // newDir returns a dir with the Name decrypted
 func (f *Fs) newDir(ctx context.Context, dir fs.Directory) fs.Directory {
+	newDir := fs.NewDirCopy(ctx, dir)
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
 		fs.Debugf(remote, "Undecryptable dir name: %v", err)
 	} else {
-		remote = decryptedRemote
+		newDir.SetRemote(decryptedRemote)
 	}
-	newDir := fs.NewDirWrapper(remote, dir)
 	return newDir
 }
 
@@ -1248,17 +1182,6 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
-// SetMetadata sets metadata for an Object
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	do, ok := o.Object.(fs.SetMetadataer)
-	if !ok {
-		return fs.ErrorNotImplemented
-	}
-	return do.SetMetadata(ctx, metadata)
-}
-
 // MimeType returns the content type of the Object if
 // known, or "" if not
 //
@@ -1284,8 +1207,6 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Wrapper         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)

backend/drive/drive.go

@@ -151,7 +151,6 @@ func (rwChoices) Choices() []fs.BitsChoicesInfo {
 		{Bit: uint64(rwOff), Name: "off"},
 		{Bit: uint64(rwRead), Name: "read"},
 		{Bit: uint64(rwWrite), Name: "write"},
-		{Bit: uint64(rwFailOK), Name: "failok"},
 	}
 }
 
@@ -161,7 +160,6 @@ type rwChoice = fs.Bits[rwChoices]
 const (
 	rwRead rwChoice = 1 << iota
 	rwWrite
-	rwFailOK
 	rwOff rwChoice = 0
 )
 
@@ -175,9 +173,6 @@ var rwExamples = fs.OptionExamples{{
 }, {
 	Value: rwWrite.String(),
 	Help:  "Write the value only",
-}, {
-	Value: rwFailOK.String(),
-	Help:  "If writing fails log errors only, don't fail the transfer",
 }, {
 	Value: (rwRead | rwWrite).String(),
 	Help:  "Read and Write the value.",
@@ -292,10 +287,7 @@ func init() {
 		},
 		MetadataInfo: &fs.MetadataInfo{
 			System: systemMetadataInfo,
-			Help: `User metadata is stored in the properties field of the drive object.
-
-Metadata is supported on files and directories.
-`,
+			Help:   `User metadata is stored in the properties field of the drive object.`,
 		},
 		Options: append(driveOAuthOptions(), []fs.Option{{
 			Name: "scope",
@@ -878,11 +870,6 @@ type Object struct {
 	v2Download bool   // generate v2 download link ondemand
 }
 
-// Directory describes a drive directory
-type Directory struct {
-	baseObject
-}
-
 // ------------------------------------------------------------
 
 // Name of the remote (as passed into NewFs)
@@ -1387,20 +1374,15 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.features = (&fs.Features{
-		DuplicateFiles:           true,
-		ReadMimeType:             true,
-		WriteMimeType:            true,
-		CanHaveEmptyDirectories:  true,
-		ServerSideAcrossConfigs:  opt.ServerSideAcrossConfigs,
-		FilterAware:              true,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: false, // FIXME need to check!
+		DuplicateFiles:          true,
+		ReadMimeType:            true,
+		WriteMimeType:           true,
+		CanHaveEmptyDirectories: true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		FilterAware:             true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.
@@ -1747,72 +1729,26 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	return pathIDOut, found, err
 }
 
-// createDir makes a directory with pathID as parent and name leaf with optional metadata
-func (f *Fs) createDir(ctx context.Context, pathID, leaf string, metadata fs.Metadata) (info *drive.File, err error) {
-	leaf = f.opt.Enc.FromStandardName(leaf)
-	pathID = actualID(pathID)
-	createInfo := &drive.File{
-		Name:     leaf,
-		MimeType: driveFolderType,
-		Parents:  []string{pathID},
-	}
-	var updateMetadata updateMetadataFn
-	if len(metadata) > 0 {
-		updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true)
-		if err != nil {
-			return nil, fmt.Errorf("create dir: failed to update metadata: %w", err)
-		}
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		info, err = f.svc.Files.Create(createInfo).
-			Fields(f.getFileFields(ctx)).
-			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	if updateMetadata != nil {
-		err = updateMetadata(ctx, info)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return info, nil
-}
-
-// updateDir updates an existing a directory with the metadata passed in
-func (f *Fs) updateDir(ctx context.Context, dirID string, metadata fs.Metadata) (info *drive.File, err error) {
-	if len(metadata) == 0 {
-		return f.getFile(ctx, dirID, f.getFileFields(ctx))
-	}
-	dirID = actualID(dirID)
-	updateInfo := &drive.File{}
-	updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true)
-	if err != nil {
-		return nil, fmt.Errorf("update dir: failed to update metadata from source object: %w", err)
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		info, err = f.svc.Files.Update(dirID, updateInfo).
-			Fields(f.getFileFields(ctx)).
-			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	err = updateMetadata(ctx, info)
-	if err != nil {
-		return nil, err
-	}
-	return info, nil
-}
-
 // CreateDir makes a directory with pathID as parent and name leaf
 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
-	info, err := f.createDir(ctx, pathID, leaf, nil)
+	leaf = f.opt.Enc.FromStandardName(leaf)
+	// fmt.Println("Making", path)
+	// Define the metadata for the directory we are going to create.
+	pathID = actualID(pathID)
+	createInfo := &drive.File{
+		Name:        leaf,
+		Description: leaf,
+		MimeType:    driveFolderType,
+		Parents:     []string{pathID},
+	}
+	var info *drive.File
+	err = f.pacer.Call(func() (bool, error) {
+		info, err = f.svc.Files.Create(createInfo).
+			Fields("id").
+			SupportsAllDrives(true).
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
+	})
 	if err != nil {
 		return "", err
 	}
@@ -1923,7 +1859,7 @@ func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string
 	return "", "", isDocument
 }
 
-// findExportFormat works out the optimum export settings
+// findExportFormatByMimeType works out the optimum export settings
 // for the given drive.File.
 //
 // Look through the exportExtensions and find the first format that can be
@@ -2219,13 +2155,13 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		case in <- job:
 		default:
 			overflow = append(overflow, job)
-			wg.Done()
+			wg.Add(-1)
 		}
 	}
 
 	// Send the entry to the caller, queueing any directories as new jobs
 	cb := func(entry fs.DirEntry) error {
-		if d, isDir := entry.(fs.Directory); isDir {
+		if d, isDir := entry.(*fs.Dir); isDir {
 			job := listREntry{actualID(d.ID()), d.Remote()}
 			sendJob(job)
 		}
@@ -2402,11 +2338,11 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File
 		if item.ResourceKey != "" {
 			f.dirResourceKeys.Store(item.Id, item.ResourceKey)
 		}
-		baseObject, err := f.newBaseObject(ctx, remote, item)
-		if err != nil {
-			return nil, err
+		when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
+		d := fs.NewDir(remote, when).SetID(item.Id)
+		if len(item.Parents) > 0 {
+			d.SetParentID(item.Parents[0])
 		}
-		d := &Directory{baseObject: baseObject}
 		return d, nil
 	case f.opt.AuthOwnerOnly && !isAuthOwned(item):
 		// ignore object
@@ -2434,6 +2370,7 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim
 	// Define the metadata for the file we are going to create.
 	createInfo := &drive.File{
 		Name:         leaf,
+		Description:  leaf,
 		Parents:      []string{directoryID},
 		ModifiedTime: modTime.Format(timeFormatOut),
 	}
@@ -2598,59 +2535,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return err
 }
 
-// MkdirMetadata makes the directory passed in as dir.
-//
-// It shouldn't return an error if it already exists.
-//
-// If the metadata is not nil it is set.
-//
-// It returns the directory that was created.
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	var info *drive.File
-	dirID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err == fs.ErrorDirNotFound {
-		// Directory does not exist so create it
-		var leaf, parentID string
-		leaf, parentID, err = f.dirCache.FindPath(ctx, dir, true)
-		if err != nil {
-			return nil, err
-		}
-		info, err = f.createDir(ctx, parentID, leaf, metadata)
-	} else if err == nil {
-		// Directory exists and needs updating
-		info, err = f.updateDir(ctx, dirID, metadata)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// Convert the info into a directory entry
-	entry, err := f.itemToDirEntry(ctx, dir, info)
-	if err != nil {
-		return nil, err
-	}
-	dirEntry, ok := entry.(fs.Directory)
-	if !ok {
-		return nil, fmt.Errorf("internal error: expecting %T to be an fs.Directory", entry)
-	}
-
-	return dirEntry, nil
-}
-
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	dirID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return err
-	}
-	o := baseObject{
-		fs:     f,
-		remote: dir,
-		id:     dirID,
-	}
-	return o.SetModTime(ctx, modTime)
-}
-
 // delete a file or directory unconditionally by ID
 func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
 	return f.pacer.Call(func() (bool, error) {
@@ -2794,12 +2678,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		createInfo.Description = ""
 	}
 
-	// Adjust metadata if required
-	updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, fs.MetadataAsOpenOptions(ctx), createInfo, false)
-	if err != nil {
-		return nil, err
-	}
-
 	// get the ID of the thing to copy
 	// copy the contents if CopyShortcutContent
 	// else copy the shortcut only
@@ -2813,7 +2691,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		copy := f.svc.Files.Copy(id, createInfo).
-			Fields(f.getFileFields(ctx)).
+			Fields(partialFields).
 			SupportsAllDrives(true).
 			KeepRevisionForever(f.opt.KeepRevisionForever)
 		srcObj.addResourceKey(copy.Header())
@@ -2833,7 +2711,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// FIXME remove this when google fixes the problem!
 	if isDoc {
 		// A short sleep is needed here in order to make the
-		// change effective, without it is ignored. This is
+		// change effective, without it is is ignored. This is
 		// probably some eventual consistency nastiness.
 		sleepTime := 2 * time.Second
 		fs.Debugf(f, "Sleeping for %v before setting the modtime to work around drive bug - see #4517", sleepTime)
@@ -2849,11 +2727,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			fs.Errorf(existingObject, "Failed to remove existing object after copy: %v", err)
 		}
 	}
-	// Finalise metadata
-	err = updateMetadata(ctx, info)
-	if err != nil {
-		return nil, err
-	}
 	return newObject, nil
 }
 
@@ -3027,19 +2900,13 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	dstParents := strings.Join(dstInfo.Parents, ",")
 	dstInfo.Parents = nil
 
-	// Adjust metadata if required
-	updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, fs.MetadataAsOpenOptions(ctx), dstInfo, true)
-	if err != nil {
-		return nil, err
-	}
-
 	// Do the move
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Update(shortcutID(srcObj.id), dstInfo).
 			RemoveParents(srcParentID).
 			AddParents(dstParents).
-			Fields(f.getFileFields(ctx)).
+			Fields(partialFields).
 			SupportsAllDrives(true).
 			Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
@@ -3048,11 +2915,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
-	// Finalise metadata
-	err = updateMetadata(ctx, info)
-	if err != nil {
-		return nil, err
-	}
 	return f.newObjectWithInfo(ctx, remote, info)
 }
 
@@ -3558,50 +3420,6 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 	return nil
 }
 
-func (f *Fs) query(ctx context.Context, query string) (entries []*drive.File, err error) {
-	list := f.svc.Files.List()
-	if query != "" {
-		list.Q(query)
-	}
-
-	if f.opt.ListChunk > 0 {
-		list.PageSize(f.opt.ListChunk)
-	}
-	list.SupportsAllDrives(true)
-	list.IncludeItemsFromAllDrives(true)
-	if f.isTeamDrive && !f.opt.SharedWithMe {
-		list.DriveId(f.opt.TeamDriveID)
-		list.Corpora("drive")
-	}
-	// If using appDataFolder then need to add Spaces
-	if f.rootFolderID == "appDataFolder" {
-		list.Spaces("appDataFolder")
-	}
-
-	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.getFileFields(ctx))
-
-	var results []*drive.File
-	for {
-		var files *drive.FileList
-		err = f.pacer.Call(func() (bool, error) {
-			files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
-			return f.shouldRetry(ctx, err)
-		})
-		if err != nil {
-			return nil, fmt.Errorf("failed to execute query: %w", err)
-		}
-		if files.IncompleteSearch {
-			fs.Errorf(f, "search result INCOMPLETE")
-		}
-		results = append(results, files.Files...)
-		if files.NextPageToken == "" {
-			break
-		}
-		list.PageToken(files.NextPageToken)
-	}
-	return results, nil
-}
-
 var commandHelp = []fs.CommandHelp{{
 	Name:  "get",
 	Short: "Get command for fetching the drive config parameters",
@@ -3752,47 +3570,6 @@ Use the --interactive/-i or --dry-run flag to see what would be copied before co
 }, {
 	Name:  "importformats",
 	Short: "Dump the import formats for debug purposes",
-}, {
-	Name:  "query",
-	Short: "List files using Google Drive query language",
-	Long: `This command lists files based on a query
-
-Usage:
-
-    rclone backend query drive: query
-    
-The query syntax is documented at [Google Drive Search query terms and 
-operators](https://developers.google.com/drive/api/guides/ref-search-terms).
-
-For example:
-
-	rclone backend query drive: "'0ABc9DEFGHIJKLMNop0QRatUVW3X' in parents and name contains 'foo'"
-
-If the query contains literal ' or \ characters, these need to be escaped with
-\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a 
-file named "foo ' \.txt":
-
-	rclone backend query drive: "name = 'foo \' \\\.txt'"
-
-The result is a JSON array of matches, for example:
-
-    [
-	{
-		"createdTime": "2017-06-29T19:58:28.537Z",
-		"id": "0AxBe_CDEF4zkGHI4d0FjYko2QkD",
-		"md5Checksum": "68518d16be0c6fbfab918be61d658032",
-		"mimeType": "text/plain",
-		"modifiedTime": "2024-02-02T10:40:02.874Z",
-		"name": "foo ' \\.txt",
-		"parents": [
-			"0BxAe_BCDE4zkFGZpcWJGek0xbzC"
-		],
-		"resourceKey": "0-ABCDEFGHIXJQpIGqBJq3MC",
-		"sha1Checksum": "8f284fa768bfb4e45d076a579ab3905ab6bfa893",
-		"size": "311",
-		"webViewLink": "https://drive.google.com/file/d/0AxBe_CDEF4zkGHI4d0FjYko2QkD/view?usp=drivesdk\u0026resourcekey=0-ABCDEFGHIXJQpIGqBJq3MC"
-	}
-    ]`,
 }}
 
 // Command the backend to run a named command
@@ -3910,17 +3687,6 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		return f.exportFormats(ctx), nil
 	case "importformats":
 		return f.importFormats(ctx), nil
-	case "query":
-		if len(arg) == 1 {
-			query := arg[0]
-			var results, err = f.query(ctx, query)
-			if err != nil {
-				return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err)
-			}
-			return results, nil
-		} else {
-			return nil, errors.New("need a query argument")
-		}
 	default:
 		return nil, fs.ErrorCommandNotFound
 	}
@@ -3965,7 +3731,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	return "", hash.ErrUnsupported
 }
 func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.MD5 && t != hash.SHA1 && t != hash.SHA256 {
+	if t != hash.MD5 {
 		return "", hash.ErrUnsupported
 	}
 	return "", nil
@@ -4427,37 +4193,6 @@ func (o *linkObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
 
-// Items returns the count of items in this directory or this
-// directory and subdirectories if known, -1 for unknown
-func (d *Directory) Items() int64 {
-	return -1
-}
-
-// SetMetadata sets metadata for a Directory
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (d *Directory) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	info, err := d.fs.updateDir(ctx, d.id, metadata)
-	if err != nil {
-		return fmt.Errorf("failed to update directory info: %w", err)
-	}
-	// Update directory from info returned
-	baseObject, err := d.fs.newBaseObject(ctx, d.remote, info)
-	if err != nil {
-		return fmt.Errorf("failed to process directory info: %w", err)
-	}
-	d.baseObject = baseObject
-	return err
-}
-
-// Hash does nothing on a directory
-//
-// This method is implemented with the incorrect type signature to
-// stop the Directory type asserting to fs.Object or fs.ObjectInfo
-func (d *Directory) Hash() {
-	// Does nothing
-}
-
 // templates for document link files
 const (
 	urlTemplate = `[InternetShortcut]{{"\r"}}
@@ -4507,8 +4242,6 @@ var (
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = (*Object)(nil)
@@ -4523,8 +4256,4 @@ var (
 	_ fs.MimeTyper       = (*linkObject)(nil)
 	_ fs.IDer            = (*linkObject)(nil)
 	_ fs.ParentIDer      = (*linkObject)(nil)
-	_ fs.Directory       = (*Directory)(nil)
-	_ fs.SetModTimer     = (*Directory)(nil)
-	_ fs.SetMetadataer   = (*Directory)(nil)
-	_ fs.ParentIDer      = (*Directory)(nil)
 )

backend/drive/drive_internal_test.go

@@ -524,49 +524,12 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	})
 }
 
-// TestIntegration/FsMkdir/FsPutFiles/Internal/Query
-func (f *Fs) InternalTestQuery(t *testing.T) {
-	ctx := context.Background()
-	var err error
-	t.Run("BadQuery", func(t *testing.T) {
-		_, err = f.query(ctx, "this is a bad query")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "failed to execute query")
-	})
-
-	t.Run("NoMatch", func(t *testing.T) {
-		results, err := f.query(ctx, fmt.Sprintf("name='%s' and name!='%s'", existingSubDir, existingSubDir))
-		require.NoError(t, err)
-		assert.Len(t, results, 0)
-	})
-
-	t.Run("GoodQuery", func(t *testing.T) {
-		pathSegments := strings.Split(existingFile, "/")
-		var parent string
-		for _, item := range pathSegments {
-			// the file name contains ' characters which must be escaped
-			escapedItem := f.opt.Enc.FromStandardName(item)
-			escapedItem = strings.ReplaceAll(escapedItem, `\`, `\\`)
-			escapedItem = strings.ReplaceAll(escapedItem, `'`, `\'`)
-
-			results, err := f.query(ctx, fmt.Sprintf("%strashed=false and name='%s'", parent, escapedItem))
-			require.NoError(t, err)
-			require.True(t, len(results) > 0)
-			for _, result := range results {
-				assert.True(t, len(result.Id) > 0)
-				assert.Equal(t, result.Name, item)
-			}
-			parent = fmt.Sprintf("'%s' in parents and ", results[0].Id)
-		}
-	})
-}
-
 // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
 func (f *Fs) InternalTestAgeQuery(t *testing.T) {
 	// Check set up for filtering
 	assert.True(t, f.Features().FilterAware)
 
-	opt := &filter.Options{}
+	opt := &filter.Opt{}
 	err := opt.MaxAge.Set("1h")
 	assert.NoError(t, err)
 	flt, err := filter.NewFilter(opt)
@@ -648,7 +611,6 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Shortcuts", f.InternalTestShortcuts)
 	t.Run("UnTrash", f.InternalTestUnTrash)
 	t.Run("CopyID", f.InternalTestCopyID)
-	t.Run("Query", f.InternalTestQuery)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
 	t.Run("ShouldRetry", f.InternalTestShouldRetry)
 }

backend/drive/metadata.go

@@ -9,8 +9,6 @@ import (
 	"sync"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/errcount"
 	"golang.org/x/sync/errgroup"
 	drive "google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
@@ -39,7 +37,7 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 		Example: "true",
 	},
 	"writers-can-share": {
-		Help:    "Whether users with only writer permission can modify the file's permissions. Not populated and ignored when setting for items in shared drives.",
+		Help:    "Whether users with only writer permission can modify the file's permissions. Not populated for items in shared drives.",
 		Type:    "boolean",
 		Example: "false",
 	},
@@ -137,30 +135,23 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
 
 // Set the permissions on the info
 func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) {
-	errs := errcount.New()
 	for _, perm := range permissions {
 		if perm.Role == "owner" {
 			// ignore owner permissions - these are set with owner
 			continue
 		}
 		cleanPermissionForWrite(perm)
-		err := f.pacer.Call(func() (bool, error) {
-			_, err := f.svc.Permissions.Create(info.Id, perm).
+		err = f.pacer.Call(func() (bool, error) {
+			_, err = f.svc.Permissions.Create(info.Id, perm).
 				SupportsAllDrives(true).
-				SendNotificationEmail(false).
 				Context(ctx).Do()
 			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
-			fs.Errorf(f, "Failed to set permission %s for %q: %v", perm.Role, perm.EmailAddress, err)
-			errs.Add(err)
+			return fmt.Errorf("failed to set permission: %w", err)
 		}
 	}
-	err = errs.Err("failed to set permission")
-	if err != nil {
-		err = fserrors.NoRetryError(err)
-	}
-	return err
+	return nil
 }
 
 // Clean attributes from permissions which we can't write
@@ -262,7 +253,7 @@ func (f *Fs) setLabels(ctx context.Context, info *drive.File, labels []*drive.La
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
-		return fmt.Errorf("failed to set labels: %w", err)
+		return fmt.Errorf("failed to set owner: %w", err)
 	}
 	return nil
 }
@@ -372,7 +363,6 @@ func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err e
 		// shared drives.
 		if o.fs.isTeamDrive && !info.HasAugmentedPermissions {
 			// Don't process permissions if there aren't any specifically set
-			fs.Debugf(o, "Ignoring %d permissions and %d permissionIds as is shared drive with hasAugmentedPermissions false", len(info.Permissions), len(info.PermissionIds))
 			info.Permissions = nil
 			info.PermissionIds = nil
 		}
@@ -537,12 +527,8 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 				return nil, err
 			}
 		case "writers-can-share":
-			if !f.isTeamDrive {
-				if err := parseBool(&updateInfo.WritersCanShare); err != nil {
-					return nil, err
-				}
-			} else {
-				fs.Debugf(f, "Ignoring %s=%s as can't set on shared drives", k, v)
+			if err := parseBool(&updateInfo.WritersCanShare); err != nil {
+				return nil, err
 			}
 		case "viewed-by-me":
 			// Can't write this
@@ -554,12 +540,7 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 			}
 			// Can't set Owner on upload so need to set afterwards
 			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setOwner(ctx, info, v)
-				if err != nil && f.opt.MetadataOwner.IsSet(rwFailOK) {
-					fs.Errorf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
+				return f.setOwner(ctx, info, v)
 			})
 		case "permissions":
 			if !f.opt.MetadataPermissions.IsSet(rwWrite) {
@@ -572,13 +553,7 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 			}
 			// Can't set Permissions on upload so need to set afterwards
 			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setPermissions(ctx, info, perms)
-				if err != nil && f.opt.MetadataPermissions.IsSet(rwFailOK) {
-					// We've already logged the permissions errors individually here
-					fs.Debugf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
+				return f.setPermissions(ctx, info, perms)
 			})
 		case "labels":
 			if !f.opt.MetadataLabels.IsSet(rwWrite) {
@@ -591,12 +566,7 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 			}
 			// Can't set Labels on upload so need to set afterwards
 			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setLabels(ctx, info, labels)
-				if err != nil && f.opt.MetadataLabels.IsSet(rwFailOK) {
-					fs.Errorf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
+				return f.setLabels(ctx, info, labels)
 			})
 		case "folder-color-rgb":
 			updateInfo.FolderColorRgb = v

backend/dropbox/dropbox.go

@@ -216,10 +216,7 @@ are supported.
 
 Note that we don't unmount the shared folder afterwards so the 
 --dropbox-shared-folders can be omitted after the first use of a particular 
-shared folder.
-
-See also --dropbox-root-namespace for an alternative way to work with shared
-folders.`,
+shared folder.`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -240,11 +237,6 @@ folders.`,
 				encoder.EncodeDel |
 				encoder.EncodeRightSpace |
 				encoder.EncodeInvalidUtf8,
-		}, {
-			Name:     "root_namespace",
-			Help:     "Specify a different Dropbox namespace ID to use as the root for all paths.",
-			Default:  "",
-			Advanced: true,
 		}}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
 	})
 }
@@ -261,7 +253,6 @@ type Options struct {
 	AsyncBatch    bool                 `config:"async_batch"`
 	PacerMinSleep fs.Duration          `config:"pacer_min_sleep"`
 	Enc           encoder.MultiEncoder `config:"encoding"`
-	RootNsid      string               `config:"root_namespace"`
 }
 
 // Fs represents a remote dropbox server
@@ -386,7 +377,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	oldToken = strings.TrimSpace(oldToken)
 	if ok && oldToken != "" && oldToken[0] != '{' {
 		fs.Infof(name, "Converting token to new format")
-		newToken := fmt.Sprintf(`{"access_token":%q,"token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
+		newToken := fmt.Sprintf(`{"access_token":"%s","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
 		err := config.SetValueAndSave(name, config.ConfigToken, newToken)
 		if err != nil {
 			return nil, fmt.Errorf("NewFS convert token: %w", err)
@@ -437,15 +428,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		members := []*team.UserSelectorArg{&user}
 		args := team.NewMembersGetInfoArgs(members)
 
-		memberIDs, err := f.team.MembersGetInfo(args)
+		memberIds, err := f.team.MembersGetInfo(args)
 		if err != nil {
 			return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
 		}
-		if len(memberIDs) == 0 || memberIDs[0].MemberInfo == nil || memberIDs[0].MemberInfo.Profile == nil {
+		if len(memberIds) == 0 || memberIds[0].MemberInfo == nil || memberIds[0].MemberInfo.Profile == nil {
 			return nil, fmt.Errorf("dropbox team member not found: %q", opt.Impersonate)
 		}
 
-		cfg.AsMemberID = memberIDs[0].MemberInfo.Profile.MemberProfile.TeamMemberId
+		cfg.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
 	}
 
 	f.srv = files.New(cfg)
@@ -511,11 +502,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	f.features.Fill(ctx, f)
 
-	if f.opt.RootNsid != "" {
-		f.ns = f.opt.RootNsid
-		fs.Debugf(f, "Overriding root namespace to %q", f.ns)
-	} else if strings.HasPrefix(root, "/") {
-		// If root starts with / then use the actual root
+	// If root starts with / then use the actual root
+	if strings.HasPrefix(root, "/") {
 		var acc *users.FullAccount
 		err = f.pacer.Call(func() (bool, error) {
 			acc, err = f.users.GetCurrentAccount()
@@ -656,7 +644,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
-// listSharedFolders lists all available shared folders mounted and not mounted
+// listSharedFoldersApi lists all available shared folders mounted and not mounted
 // we'll need the id later so we have to return them in original format
 func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
 	started := false
@@ -1243,7 +1231,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 		return nil, err
 	}
 	var total uint64
-	used := q.Used
+	var used = q.Used
 	if q.Allocation != nil {
 		if q.Allocation.Individual != nil {
 			total += q.Allocation.Individual.Allocated

backend/fichier/api.go

@@ -61,7 +61,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 			return false, err // No such user
 		case 186:
 			return false, err // IP blocked?
-		case 374, 412: // Flood detected seems to be #412 now
+		case 374:
 			fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
 			time.Sleep(30 * time.Second)
 		default:

backend/fichier/fichier.go

@@ -441,28 +441,23 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Debugf(src, "Can't move - not same remote type")
 		return nil, fs.ErrorCantMove
 	}
-	srcFs := srcObj.fs
 
 	// Find current directory ID
-	srcLeaf, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcObj.remote, false)
+	_, currentDirectoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		return nil, err
 	}
 
 	// Create temporary object
-	dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote)
+	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
 
 	// If it is in the correct directory, just rename it
 	var url string
-	if srcDirectoryID == dstDirectoryID {
-		// No rename needed
-		if srcLeaf == dstLeaf {
-			return src, nil
-		}
-		resp, err := f.renameFile(ctx, srcObj.file.URL, dstLeaf)
+	if currentDirectoryID == directoryID {
+		resp, err := f.renameFile(ctx, srcObj.file.URL, leaf)
 		if err != nil {
 			return nil, fmt.Errorf("couldn't rename file: %w", err)
 		}
@@ -471,16 +466,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		}
 		url = resp.URLs[0].URL
 	} else {
-		dstFolderID, err := strconv.Atoi(dstDirectoryID)
+		folderID, err := strconv.Atoi(directoryID)
 		if err != nil {
 			return nil, err
 		}
-		rename := dstLeaf
-		// No rename needed
-		if srcLeaf == dstLeaf {
-			rename = ""
-		}
-		resp, err := f.moveFile(ctx, srcObj.file.URL, dstFolderID, rename)
+		resp, err := f.moveFile(ctx, srcObj.file.URL, folderID, leaf)
 		if err != nil {
 			return nil, fmt.Errorf("couldn't move file: %w", err)
 		}

backend/filescom/filescom.go

@@ -1,901 +0,0 @@
-// Package filescom provides an interface to the Files.com
-// object storage system.
-package filescom
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"path"
-	"strings"
-	"time"
-
-	files_sdk "github.com/Files-com/files-sdk-go/v3"
-	"github.com/Files-com/files-sdk-go/v3/bundle"
-	"github.com/Files-com/files-sdk-go/v3/file"
-	file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
-	"github.com/Files-com/files-sdk-go/v3/folder"
-	"github.com/Files-com/files-sdk-go/v3/session"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-)
-
-/*
-Run of rclone info
-stringNeedsEscaping = []rune{
-        '/', '\x00'
-}
-maxFileLength = 512 // for 1 byte unicode characters
-maxFileLength = 512 // for 2 byte unicode characters
-maxFileLength = 512 // for 3 byte unicode characters
-maxFileLength = 512 // for 4 byte unicode characters
-canWriteUnnormalized = true
-canReadUnnormalized   = true
-canReadRenormalized   = true
-canStream = true
-*/
-
-const (
-	minSleep      = 10 * time.Millisecond
-	maxSleep      = 2 * time.Second
-	decayConstant = 2 // bigger for slower decay, exponential
-
-	folderNotEmpty = "processing-failure/folder-not-empty"
-)
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "filescom",
-		Description: "Files.com",
-		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name: "site",
-				Help: "Your site subdomain (e.g. mysite) or custom domain (e.g. myfiles.customdomain.com).",
-			}, {
-				Name: "username",
-				Help: "The username used to authenticate with Files.com.",
-			}, {
-				Name:       "password",
-				Help:       "The password used to authenticate with Files.com.",
-				IsPassword: true,
-			}, {
-				Name:      "api_key",
-				Help:      "The API key used to authenticate with Files.com.",
-				Advanced:  true,
-				Sensitive: true,
-			}, {
-				Name:     config.ConfigEncoding,
-				Help:     config.ConfigEncodingHelp,
-				Advanced: true,
-				Default: (encoder.Display |
-					encoder.EncodeBackSlash |
-					encoder.EncodeRightSpace |
-					encoder.EncodeRightCrLfHtVt |
-					encoder.EncodeInvalidUtf8),
-			}},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	Site     string               `config:"site"`
-	Username string               `config:"username"`
-	Password string               `config:"password"`
-	APIKey   string               `config:"api_key"`
-	Enc      encoder.MultiEncoder `config:"encoding"`
-}
-
-// Fs represents a remote files.com server
-type Fs struct {
-	name            string                 // name of this remote
-	root            string                 // the path we are working on
-	opt             Options                // parsed options
-	features        *fs.Features           // optional features
-	fileClient      *file.Client           // the connection to the file API
-	folderClient    *folder.Client         // the connection to the folder API
-	migrationClient *file_migration.Client // the connection to the file migration API
-	bundleClient    *bundle.Client         // the connection to the bundle API
-	pacer           *fs.Pacer              // pacer for API calls
-}
-
-// Object describes a files object
-//
-// Will definitely have info but maybe not meta
-type Object struct {
-	fs       *Fs       // what this object is part of
-	remote   string    // The remote path
-	size     int64     // size of the object
-	crc32    string    // CRC32 of the object content
-	md5      string    // MD5 of the object content
-	mimeType string    // Content-Type of the object
-	modTime  time.Time // modification time of the object
-}
-
-// ------------------------------------------------------------
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	return fmt.Sprintf("files root '%s'", f.root)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// Encode remote and turn it into an absolute path in the share
-func (f *Fs) absPath(remote string) string {
-	return f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
-}
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	429, // Too Many Requests.
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-// shouldRetry returns a boolean as to whether this err deserves to be
-// retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-
-	if apiErr, ok := err.(files_sdk.ResponseError); ok {
-		for _, e := range retryErrorCodes {
-			if apiErr.HttpCode == e {
-				fs.Debugf(nil, "Retrying API error %v", err)
-				return true, err
-			}
-		}
-	}
-
-	return fserrors.ShouldRetry(err), err
-}
-
-// readMetaDataForPath reads the metadata from the path
-func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *files_sdk.File, err error) {
-	params := files_sdk.FileFindParams{
-		Path: f.absPath(path),
-	}
-
-	var file files_sdk.File
-	err = f.pacer.Call(func() (bool, error) {
-		file, err = f.fileClient.Find(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return &file, nil
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	// Parse config into Options struct
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	root = strings.Trim(root, "/")
-
-	config, err := newClientConfig(ctx, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	f := &Fs{
-		name:            name,
-		root:            root,
-		opt:             *opt,
-		fileClient:      &file.Client{Config: config},
-		folderClient:    &folder.Client{Config: config},
-		migrationClient: &file_migration.Client{Config: config},
-		bundleClient:    &bundle.Client{Config: config},
-		pacer:           fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-	}
-	f.features = (&fs.Features{
-		CaseInsensitive:          true,
-		CanHaveEmptyDirectories:  true,
-		ReadMimeType:             true,
-		DirModTimeUpdatesOnWrite: true,
-	}).Fill(ctx, f)
-
-	if f.root != "" {
-		info, err := f.readMetaDataForPath(ctx, "")
-		if err == nil && !info.IsDir() {
-			f.root = path.Dir(f.root)
-			if f.root == "." {
-				f.root = ""
-			}
-			return f, fs.ErrorIsFile
-		}
-	}
-
-	return f, err
-}
-
-func newClientConfig(ctx context.Context, opt *Options) (config files_sdk.Config, err error) {
-	if opt.Site != "" {
-		if strings.Contains(opt.Site, ".") {
-			config.EndpointOverride = opt.Site
-		} else {
-			config.Subdomain = opt.Site
-		}
-
-		_, err = url.ParseRequestURI(config.Endpoint())
-		if err != nil {
-			err = fmt.Errorf("invalid domain or subdomain: %v", opt.Site)
-			return
-		}
-	}
-
-	config = config.Init().SetCustomClient(fshttp.NewClient(ctx))
-
-	if opt.APIKey != "" {
-		config.APIKey = opt.APIKey
-		return
-	}
-
-	if opt.Username == "" {
-		err = errors.New("username not found")
-		return
-	}
-	if opt.Password == "" {
-		err = errors.New("password not found")
-		return
-	}
-	opt.Password, err = obscure.Reveal(opt.Password)
-	if err != nil {
-		return
-	}
-
-	sessionClient := session.Client{Config: config}
-	params := files_sdk.SessionCreateParams{
-		Username: opt.Username,
-		Password: opt.Password,
-	}
-
-	thisSession, err := sessionClient.Create(params, files_sdk.WithContext(ctx))
-	if err != nil {
-		err = fmt.Errorf("couldn't create session: %w", err)
-		return
-	}
-
-	config.SessionId = thisSession.Id
-	return
-}
-
-// Return an Object from a path
-//
-// If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *files_sdk.File) (fs.Object, error) {
-	o := &Object{
-		fs:     f,
-		remote: remote,
-	}
-	var err error
-	if file != nil {
-		err = o.setMetaData(file)
-	} else {
-		err = o.readMetaData(ctx) // reads info and meta, returning an error
-	}
-	if err != nil {
-		return nil, err
-	}
-	return o, nil
-}
-
-// NewObject finds the Object at remote.  If it can't be found
-// it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(ctx, remote, nil)
-}
-
-// List the objects and directories in dir into entries.  The
-// entries can be returned in any order but should be for a
-// complete directory.
-//
-// dir should be "" to list the root, and should not have
-// trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	var it *folder.Iter
-	params := files_sdk.FolderListForParams{
-		Path: f.absPath(dir),
-	}
-
-	err = f.pacer.Call(func() (bool, error) {
-		it, err = f.folderClient.ListFor(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't list files: %w", err)
-	}
-
-	for it.Next() {
-		item := ptr(it.File())
-		remote := f.opt.Enc.ToStandardPath(item.DisplayName)
-		remote = path.Join(dir, remote)
-		if remote == dir {
-			continue
-		}
-
-		if item.IsDir() {
-			d := fs.NewDir(remote, item.ModTime())
-			entries = append(entries, d)
-		} else {
-			o, err := f.newObjectWithInfo(ctx, remote, item)
-			if err != nil {
-				return nil, err
-			}
-			entries = append(entries, o)
-		}
-	}
-	err = it.Err()
-	if files_sdk.IsNotExist(err) {
-		return nil, fs.ErrorDirNotFound
-	}
-	return
-}
-
-// Creates from the parameters passed in a half finished Object which
-// must have setMetaData called on it
-//
-// Returns the object and error.
-//
-// Used to create new objects
-func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, err error) {
-	// Create the directory for the object if it doesn't exist
-	err = f.mkParentDir(ctx, remote)
-	if err != nil {
-		return
-	}
-	// Temporary Object under construction
-	o = &Object{
-		fs:     f,
-		remote: remote,
-	}
-	return o, nil
-}
-
-// Put the object
-//
-// Copy the reader in to the new object which is returned.
-//
-// The new object may have been created if an error is returned
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	// Temporary Object under construction
-	fs := &Object{
-		fs:     f,
-		remote: src.Remote(),
-	}
-	return fs, fs.Update(ctx, in, src, options...)
-}
-
-// PutStream uploads to the remote path with the modTime given of indeterminate size
-func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.Put(ctx, in, src, options...)
-}
-
-func (f *Fs) mkdir(ctx context.Context, path string) error {
-	if path == "" || path == "." {
-		return nil
-	}
-
-	params := files_sdk.FolderCreateParams{
-		Path:         path,
-		MkdirParents: ptr(true),
-	}
-
-	err := f.pacer.Call(func() (bool, error) {
-		_, err := f.folderClient.Create(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if files_sdk.IsExist(err) {
-		return nil
-	}
-	return err
-}
-
-// Make the parent directory of remote
-func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
-	return f.mkdir(ctx, path.Dir(f.absPath(remote)))
-}
-
-// Mkdir creates the container if it doesn't exist
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	return f.mkdir(ctx, f.absPath(dir))
-}
-
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	o := Object{
-		fs:     f,
-		remote: dir,
-	}
-	return o.SetModTime(ctx, modTime)
-}
-
-// purgeCheck removes the root directory, if check is set then it
-// refuses to do so if it has anything in
-func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
-	path := f.absPath(dir)
-	if path == "" || path == "." {
-		return errors.New("can't purge root directory")
-	}
-
-	params := files_sdk.FileDeleteParams{
-		Path:      path,
-		Recursive: ptr(!check),
-	}
-
-	err := f.pacer.Call(func() (bool, error) {
-		err := f.fileClient.Delete(params, files_sdk.WithContext(ctx))
-		// Allow for eventual consistency deletion of child objects.
-		if isFolderNotEmpty(err) {
-			return true, err
-		}
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		if files_sdk.IsNotExist(err) {
-			return fs.ErrorDirNotFound
-		} else if isFolderNotEmpty(err) {
-			return fs.ErrorDirectoryNotEmpty
-		}
-
-		return fmt.Errorf("rmdir failed: %w", err)
-	}
-	return nil
-}
-
-// Rmdir deletes the root folder
-//
-// Returns an error if it isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, true)
-}
-
-// Precision return the precision of this Fs
-func (f *Fs) Precision() time.Duration {
-	return time.Second
-}
-
-// Copy src to this remote using server-side copy operations.
-//
-// This is stored with the remote path given.
-//
-// It returns the destination Object and a possible error.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.Object, err error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't copy - not same remote type")
-		return nil, fs.ErrorCantCopy
-	}
-	err = srcObj.readMetaData(ctx)
-	if err != nil {
-		return
-	}
-
-	srcPath := srcObj.fs.absPath(srcObj.remote)
-	dstPath := f.absPath(remote)
-	if strings.EqualFold(srcPath, dstPath) {
-		return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
-	}
-
-	// Create temporary object
-	dstObj, err = f.createObject(ctx, remote)
-	if err != nil {
-		return
-	}
-
-	// Copy the object
-	params := files_sdk.FileCopyParams{
-		Path:        srcPath,
-		Destination: dstPath,
-		Overwrite:   ptr(true),
-	}
-
-	var action files_sdk.FileAction
-	err = f.pacer.Call(func() (bool, error) {
-		action, err = f.fileClient.Copy(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return
-	}
-
-	err = f.waitForAction(ctx, action, "copy")
-	if err != nil {
-		return
-	}
-
-	err = dstObj.SetModTime(ctx, srcObj.modTime)
-	return
-}
-
-// Purge deletes all the files and the container
-//
-// Optional interface: Only implement this if you have a way of
-// deleting all the files quicker than just running Remove() on the
-// result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
-}
-
-// move a file or folder
-func (f *Fs) move(ctx context.Context, src *Fs, srcRemote string, dstRemote string) (info *files_sdk.File, err error) {
-	// Move the object
-	params := files_sdk.FileMoveParams{
-		Path:        src.absPath(srcRemote),
-		Destination: f.absPath(dstRemote),
-	}
-
-	var action files_sdk.FileAction
-	err = f.pacer.Call(func() (bool, error) {
-		action, err = f.fileClient.Move(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	err = f.waitForAction(ctx, action, "move")
-	if err != nil {
-		return nil, err
-	}
-
-	info, err = f.readMetaDataForPath(ctx, dstRemote)
-	return
-}
-
-func (f *Fs) waitForAction(ctx context.Context, action files_sdk.FileAction, operation string) (err error) {
-	var migration files_sdk.FileMigration
-	err = f.pacer.Call(func() (bool, error) {
-		migration, err = f.migrationClient.Wait(action, func(migration files_sdk.FileMigration) {
-			// noop
-		}, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err == nil && migration.Status != "completed" {
-		return fmt.Errorf("%v did not complete successfully: %v", operation, migration.Status)
-	}
-	return
-}
-
-// Move src to this remote using server-side move operations.
-//
-// This is stored with the remote path given.
-//
-// It returns the destination Object and a possible error.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantMove
-func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't move - not same remote type")
-		return nil, fs.ErrorCantMove
-	}
-
-	// Create temporary object
-	dstObj, err := f.createObject(ctx, remote)
-	if err != nil {
-		return nil, err
-	}
-
-	// Do the move
-	info, err := f.move(ctx, srcObj.fs, srcObj.remote, dstObj.remote)
-	if err != nil {
-		return nil, err
-	}
-
-	err = dstObj.setMetaData(info)
-	if err != nil {
-		return nil, err
-	}
-	return dstObj, nil
-}
-
-// DirMove moves src, srcRemote to this remote at dstRemote
-// using server-side move operations.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantDirMove
-//
-// If destination exists then return fs.ErrorDirExists
-func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
-	srcFs, ok := src.(*Fs)
-	if !ok {
-		fs.Debugf(srcFs, "Can't move directory - not same remote type")
-		return fs.ErrorCantDirMove
-	}
-
-	// Check if destination exists
-	_, err = f.readMetaDataForPath(ctx, dstRemote)
-	if err == nil {
-		return fs.ErrorDirExists
-	}
-
-	// Create temporary object
-	dstObj, err := f.createObject(ctx, dstRemote)
-	if err != nil {
-		return
-	}
-
-	// Do the move
-	_, err = f.move(ctx, srcFs, srcRemote, dstObj.remote)
-	return
-}
-
-// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (url string, err error) {
-	params := files_sdk.BundleCreateParams{
-		Paths: []string{f.absPath(remote)},
-	}
-	if expire < fs.DurationOff {
-		params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
-	}
-
-	var bundle files_sdk.Bundle
-	err = f.pacer.Call(func() (bool, error) {
-		bundle, err = f.bundleClient.Create(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-
-	url = bundle.Url
-	return
-}
-
-// Hashes returns the supported hash sets.
-func (f *Fs) Hashes() hash.Set {
-	return hash.NewHashSet(hash.CRC32, hash.MD5)
-}
-
-// ------------------------------------------------------------
-
-// Fs returns the parent Fs
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.remote
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// Hash returns the MD5 of an object returning a lowercase hex string
-func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	switch t {
-	case hash.CRC32:
-		if o.crc32 == "" {
-			return "", nil
-		}
-		return fmt.Sprintf("%08s", o.crc32), nil
-	case hash.MD5:
-		return o.md5, nil
-	}
-	return "", hash.ErrUnsupported
-}
-
-// Size returns the size of an object in bytes
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// setMetaData sets the metadata from info
-func (o *Object) setMetaData(file *files_sdk.File) error {
-	o.modTime = file.ModTime()
-
-	if !file.IsDir() {
-		o.size = file.Size
-		o.crc32 = file.Crc32
-		o.md5 = file.Md5
-		o.mimeType = file.MimeType
-	}
-
-	return nil
-}
-
-// readMetaData gets the metadata if it hasn't already been fetched
-//
-// it also sets the info
-func (o *Object) readMetaData(ctx context.Context) (err error) {
-	file, err := o.fs.readMetaDataForPath(ctx, o.remote)
-	if err != nil {
-		if files_sdk.IsNotExist(err) {
-			return fs.ErrorObjectNotFound
-		}
-		return err
-	}
-	if file.IsDir() {
-		return fs.ErrorIsDir
-	}
-	return o.setMetaData(file)
-}
-
-// ModTime returns the modification time of the object
-//
-// It attempts to read the objects mtime and if that isn't present the
-// LastModified returned in the http headers
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	return o.modTime
-}
-
-// SetModTime sets the modification time of the local fs object
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) {
-	params := files_sdk.FileUpdateParams{
-		Path:          o.fs.absPath(o.remote),
-		ProvidedMtime: &modTime,
-	}
-
-	var file files_sdk.File
-	err = o.fs.pacer.Call(func() (bool, error) {
-		file, err = o.fs.fileClient.Update(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return err
-	}
-	return o.setMetaData(&file)
-}
-
-// Storable returns a boolean showing whether this object storable
-func (o *Object) Storable() bool {
-	return true
-}
-
-// Open an object for read
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	// Offset and Count for range download
-	var offset, count int64
-	fs.FixRangeOption(options, o.size)
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.RangeOption:
-			offset, count = x.Decode(o.size)
-			if count < 0 {
-				count = o.size - offset
-			}
-		case *fs.SeekOption:
-			offset = x.Offset
-			count = o.size - offset
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-
-	params := files_sdk.FileDownloadParams{
-		Path: o.fs.absPath(o.remote),
-	}
-
-	headers := &http.Header{}
-	headers.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, offset+count-1))
-	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.fileClient.Download(
-			params,
-			files_sdk.WithContext(ctx),
-			files_sdk.RequestHeadersOption(headers),
-			files_sdk.ResponseBodyOption(func(closer io.ReadCloser) error {
-				in = closer
-				return err
-			}),
-		)
-		return shouldRetry(ctx, err)
-	})
-	return
-}
-
-// Returns a pointer to t - useful for returning pointers to constants
-func ptr[T any](t T) *T {
-	return &t
-}
-
-func isFolderNotEmpty(err error) bool {
-	var re files_sdk.ResponseError
-	ok := errors.As(err, &re)
-	return ok && re.Type == folderNotEmpty
-}
-
-// Update the object with the contents of the io.Reader, modTime and size
-//
-// If existing is set then it updates the object rather than creating a new one.
-//
-// The new object may have been created if an error is returned.
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	uploadOpts := []file.UploadOption{
-		file.UploadWithContext(ctx),
-		file.UploadWithReader(in),
-		file.UploadWithDestinationPath(o.fs.absPath(o.remote)),
-		file.UploadWithProvidedMtime(src.ModTime(ctx)),
-	}
-
-	err := o.fs.pacer.Call(func() (bool, error) {
-		err := o.fs.fileClient.Upload(uploadOpts...)
-		return shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return err
-	}
-
-	return o.readMetaData(ctx)
-}
-
-// Remove an object
-func (o *Object) Remove(ctx context.Context) error {
-	params := files_sdk.FileDeleteParams{
-		Path: o.fs.absPath(o.remote),
-	}
-
-	return o.fs.pacer.Call(func() (bool, error) {
-		err := o.fs.fileClient.Delete(params, files_sdk.WithContext(ctx))
-		return shouldRetry(ctx, err)
-	})
-}
-
-// MimeType of an Object if known, "" otherwise
-func (o *Object) MimeType(ctx context.Context) string {
-	return o.mimeType
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs           = (*Fs)(nil)
-	_ fs.Purger       = (*Fs)(nil)
-	_ fs.PutStreamer  = (*Fs)(nil)
-	_ fs.Copier       = (*Fs)(nil)
-	_ fs.Mover        = (*Fs)(nil)
-	_ fs.DirMover     = (*Fs)(nil)
-	_ fs.PublicLinker = (*Fs)(nil)
-	_ fs.Object       = (*Object)(nil)
-	_ fs.MimeTyper    = (*Object)(nil)
-)

backend/filescom/filescom_test.go

@@ -1,17 +0,0 @@
-// Test Files filesystem interface
-package filescom_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/filescom"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFilesCom:",
-		NilObject:  (*filescom.Object)(nil),
-	})
-}

backend/frostfs/frostfs_test.go

@@ -1,16 +0,0 @@
-package frostfs
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:      "TestFrostFS:",
-		NilObject:       (*Object)(nil),
-		SkipInvalidUTF8: true,
-	})
-}

backend/frostfs/util.go

@@ -1,326 +0,0 @@
-package frostfs
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
-	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
-	resolver "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
-	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
-	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
-	"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
-	"github.com/nspcc-dev/neo-go/cli/flags"
-	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
-	"github.com/nspcc-dev/neo-go/pkg/wallet"
-
-	"github.com/rclone/rclone/fs"
-)
-
-type endpointInfo struct {
-	Address  string
-	Priority int
-	Weight   float64
-}
-
-func publicReadWriteCCPRules() []chain.Rule {
-	return []chain.Rule{
-		{
-			Status: chain.Allow, Actions: chain.Actions{
-				Inverted: false,
-				Names: []string{
-					native.MethodPutObject,
-					native.MethodGetObject,
-					native.MethodHeadObject,
-					native.MethodDeleteObject,
-					native.MethodSearchObject,
-					native.MethodRangeObject,
-					native.MethodHashObject,
-					native.MethodPatchObject,
-				},
-			}, Resources: chain.Resources{
-				Inverted: false,
-				Names:    []string{native.ResourceFormatRootObjects},
-			}, Any: false},
-	}
-}
-
-func privateCCPRules() []chain.Rule {
-	rule := publicReadWriteCCPRules()
-	// The same as public-read-write, except that only the owner is allowed to perform the listed actions
-	rule[0].Condition = []chain.Condition{
-		{
-			Op:    chain.CondStringEquals,
-			Kind:  chain.KindRequest,
-			Key:   native.PropertyKeyActorRole,
-			Value: native.PropertyValueContainerRoleOwner,
-		},
-	}
-	return rule
-}
-
-func publicReadCCPRules() []chain.Rule {
-	rule := privateCCPRules()
-	// Add a rule that allows other users to perform reading actions.
-	rule = append(rule, chain.Rule{
-		Status: chain.Allow, Actions: chain.Actions{
-			Inverted: false,
-			Names: []string{
-				native.MethodGetObject,
-				native.MethodHeadObject,
-				native.MethodRangeObject,
-				native.MethodHashObject,
-				native.MethodSearchObject,
-			},
-		}, Resources: chain.Resources{
-			Inverted: false,
-			Names:    []string{native.ResourceFormatRootObjects},
-		}, Condition: []chain.Condition{
-			{
-				Op:    chain.CondStringEquals,
-				Kind:  chain.KindRequest,
-				Key:   native.PropertyKeyActorRole,
-				Value: native.PropertyValueContainerRoleOthers,
-			},
-		}, Any: false})
-	return rule
-}
-
-func parseContainerCreationPolicyString(policyString string) ([]chain.Rule, error) {
-	switch policyString {
-	case "private":
-		return privateCCPRules(), nil
-	case "public-read":
-		return publicReadCCPRules(), nil
-	case "public-read-write":
-		return publicReadWriteCCPRules(), nil
-	}
-	return nil, fmt.Errorf("invalid container creation policy: %s", policyString)
-}
-
-func parseEndpoints(endpointParam string) ([]endpointInfo, error) {
-	var err error
-	expectedLength := -1 // to make sure all endpoints have the same format
-
-	endpoints := strings.Split(strings.TrimSpace(endpointParam), " ")
-	res := make([]endpointInfo, 0, len(endpoints))
-	seen := make(map[string]struct{}, len(endpoints))
-
-	for _, endpoint := range endpoints {
-		endpointInfoSplit := strings.Split(endpoint, ",")
-		address := endpointInfoSplit[0]
-
-		if len(address) == 0 {
-			continue
-		}
-		if _, ok := seen[address]; ok {
-			return nil, fmt.Errorf("endpoint '%s' is already defined", address)
-		}
-		seen[address] = struct{}{}
-
-		epInfo := endpointInfo{
-			Address:  address,
-			Priority: 1,
-			Weight:   1,
-		}
-
-		if expectedLength == -1 {
-			expectedLength = len(endpointInfoSplit)
-		}
-
-		if len(endpointInfoSplit) != expectedLength {
-			return nil, fmt.Errorf("all endpoints must have the same format: '%s'", endpointParam)
-		}
-
-		switch len(endpointInfoSplit) {
-		case 1:
-		case 2:
-			epInfo.Priority, err = parsePriority(endpointInfoSplit[1])
-			if err != nil {
-				return nil, fmt.Errorf("invalid endpoint '%s': %w", endpoint, err)
-			}
-		case 3:
-			epInfo.Priority, err = parsePriority(endpointInfoSplit[1])
-			if err != nil {
-				return nil, fmt.Errorf("invalid endpoint '%s': %w", endpoint, err)
-			}
-
-			epInfo.Weight, err = parseWeight(endpointInfoSplit[2])
-			if err != nil {
-				return nil, fmt.Errorf("invalid endpoint '%s': %w", endpoint, err)
-			}
-		default:
-			return nil, fmt.Errorf("invalid endpoint format '%s'", endpoint)
-		}
-
-		res = append(res, epInfo)
-	}
-
-	return res, nil
-}
-
-func parsePriority(priorityStr string) (int, error) {
-	priority, err := strconv.Atoi(priorityStr)
-	if err != nil {
-		return 0, fmt.Errorf("invalid priority '%s': %w", priorityStr, err)
-	}
-	if priority <= 0 {
-		return 0, fmt.Errorf("priority must be positive '%s'", priorityStr)
-	}
-
-	return priority, nil
-}
-
-func parseWeight(weightStr string) (float64, error) {
-	weight, err := strconv.ParseFloat(weightStr, 64)
-	if err != nil {
-		return 0, fmt.Errorf("invalid weight '%s': %w", weightStr, err)
-	}
-	if weight <= 0 {
-		return 0, fmt.Errorf("weight must be positive '%s'", weightStr)
-	}
-
-	return weight, nil
-}
-
-func createPool(ctx context.Context, key *keys.PrivateKey, cfg *Options) (*pool.Pool, error) {
-	var prm pool.InitParameters
-	prm.SetKey(&key.PrivateKey)
-	prm.SetNodeDialTimeout(time.Duration(cfg.FrostfsConnectionTimeout))
-	prm.SetHealthcheckTimeout(time.Duration(cfg.FrostfsRequestTimeout))
-	prm.SetClientRebalanceInterval(time.Duration(cfg.FrostfsRebalanceInterval))
-	prm.SetSessionExpirationDuration(cfg.FrostfsSessionExpiration)
-
-	nodes, err := getNodePoolParams(cfg.FrostfsEndpoint)
-	if err != nil {
-		return nil, err
-	}
-	for _, node := range nodes {
-		prm.AddNode(node)
-	}
-
-	p, err := pool.NewPool(prm)
-	if err != nil {
-		return nil, fmt.Errorf("create pool: %w", err)
-	}
-
-	if err = p.Dial(ctx); err != nil {
-		return nil, fmt.Errorf("dial pool: %w", err)
-	}
-
-	return p, nil
-}
-
-func getNodePoolParams(endpointParam string) ([]pool.NodeParam, error) {
-	endpointInfos, err := parseEndpoints(endpointParam)
-	if err != nil {
-		return nil, fmt.Errorf("parse endpoints params: %w", err)
-	}
-
-	res := make([]pool.NodeParam, len(endpointInfos))
-	for i, info := range endpointInfos {
-		res[i] = pool.NewNodeParam(info.Priority, info.Address, info.Weight)
-	}
-
-	return res, nil
-}
-
-func createNNSResolver(cfg *Options) (*resolver.NNS, error) {
-	if cfg.RPCEndpoint == "" {
-		return nil, nil
-	}
-
-	var nns resolver.NNS
-	if err := nns.Dial(cfg.RPCEndpoint); err != nil {
-		return nil, fmt.Errorf("dial NNS resolver: %w", err)
-	}
-
-	return &nns, nil
-}
-
-func getAccount(cfg *Options) (*wallet.Account, error) {
-	w, err := wallet.NewWalletFromFile(cfg.Wallet)
-	if err != nil {
-		return nil, err
-	}
-
-	addr := w.GetChangeAddress()
-	if cfg.Address != "" {
-		addr, err = flags.ParseAddress(cfg.Address)
-		if err != nil {
-			return nil, fmt.Errorf("invalid address")
-		}
-	}
-	acc := w.GetAccount(addr)
-	err = acc.Decrypt(cfg.Password, w.Scrypt)
-	if err != nil {
-		return nil, err
-	}
-
-	return acc, nil
-}
-
-func newAddress(cnrID cid.ID, objID oid.ID) oid.Address {
-	var addr oid.Address
-	addr.SetContainer(cnrID)
-	addr.SetObject(objID)
-	return addr
-}
-
-func formObject(own *user.ID, cnrID cid.ID, name string, header map[string]string) *object.Object {
-	attributes := make([]object.Attribute, 0, 1+len(header))
-	filename := object.NewAttribute()
-	filename.SetKey(object.AttributeFileName)
-	filename.SetValue(name)
-
-	attributes = append(attributes, *filename)
-
-	for key, val := range header {
-		attr := object.NewAttribute()
-		attr.SetKey(key)
-		attr.SetValue(val)
-		attributes = append(attributes, *attr)
-	}
-
-	obj := object.New()
-	obj.SetOwnerID(*own)
-	obj.SetContainerID(cnrID)
-	obj.SetAttributes(attributes...)
-
-	return obj
-}
-
-func newDir(cnrID cid.ID, cnr container.Container, defaultZone string) *fs.Dir {
-	remote := cnrID.EncodeToString()
-	timestamp := container.CreatedAt(cnr)
-
-	if domain := container.ReadDomain(cnr); domain.Name() != "" {
-		if defaultZone != domain.Zone() {
-			remote = domain.Name() + "." + domain.Zone()
-		} else {
-			remote = domain.Name()
-		}
-	}
-
-	dir := fs.NewDir(remote, timestamp)
-	dir.SetID(cnrID.String())
-	return dir
-}
-
-func getContainerNameAndZone(containerStr, defaultZone string) (cnrName string, cnrZone string) {
-	defer func() {
-		if len(cnrZone) == 0 {
-			cnrZone = defaultZone
-		}
-	}()
-	if idx := strings.Index(containerStr, "."); idx >= 0 {
-		return containerStr[:idx], containerStr[idx+1:]
-	}
-	return containerStr, defaultZone
-}

backend/frostfs/util_test.go

@@ -1,205 +0,0 @@
-package frostfs
-
-import (
-	"strconv"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestGetZoneAndContainerNames(t *testing.T) {
-	for i, tc := range []struct {
-		cnrStr       string
-		defZone      string
-		expectedName string
-		expectedZone string
-	}{
-		{
-			cnrStr:       "",
-			defZone:      "def_zone",
-			expectedName: "",
-			expectedZone: "def_zone",
-		},
-		{
-			cnrStr:       "",
-			defZone:      "def_zone",
-			expectedName: "",
-			expectedZone: "def_zone",
-		},
-		{
-			cnrStr:       "cnr_name",
-			defZone:      "def_zone",
-			expectedName: "cnr_name",
-			expectedZone: "def_zone",
-		},
-		{
-			cnrStr:       "cnr_name.",
-			defZone:      "def_zone",
-			expectedName: "cnr_name",
-			expectedZone: "def_zone",
-		},
-		{
-			cnrStr:       ".cnr_zone",
-			defZone:      "def_zone",
-			expectedName: "",
-			expectedZone: "cnr_zone",
-		}, {
-			cnrStr:       ".cnr_zone",
-			defZone:      "def_zone",
-			expectedName: "",
-			expectedZone: "cnr_zone",
-		},
-	} {
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			actualName, actualZone := getContainerNameAndZone(tc.cnrStr, tc.defZone)
-			require.Equal(t, tc.expectedZone, actualZone)
-			require.Equal(t, tc.expectedName, actualName)
-		})
-	}
-}
-
-func TestParseContainerCreationPolicy(t *testing.T) {
-	for i, tc := range []struct {
-		ACLString     string
-		ExpectedError bool
-	}{
-		{
-			ACLString:     "",
-			ExpectedError: true,
-		},
-		{
-			ACLString:     "public-ready",
-			ExpectedError: true,
-		},
-		{
-			ACLString:     "public-read",
-			ExpectedError: false,
-		},
-		{
-			ACLString:     "public-read-write",
-			ExpectedError: false,
-		},
-		{
-			ACLString:     "private",
-			ExpectedError: false,
-		},
-	} {
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			rules, err := parseContainerCreationPolicyString(tc.ACLString)
-			if tc.ExpectedError {
-				require.Error(t, err)
-				require.Nil(t, rules)
-			} else {
-				require.NoError(t, err)
-				require.NotNil(t, rules)
-			}
-		})
-	}
-}
-
-func TestParseEndpoints(t *testing.T) {
-	for i, tc := range []struct {
-		EndpointsParam string
-		ExpectedError  bool
-		ExpectedResult []endpointInfo
-	}{
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080",
-			ExpectedResult: []endpointInfo{{
-				Address:  "s01.frostfs.devenv:8080",
-				Priority: 1,
-				Weight:   1,
-			}},
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,2",
-			ExpectedResult: []endpointInfo{{
-				Address:  "s01.frostfs.devenv:8080",
-				Priority: 2,
-				Weight:   1,
-			}},
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,2,3",
-			ExpectedResult: []endpointInfo{{
-				Address:  "s01.frostfs.devenv:8080",
-				Priority: 2,
-				Weight:   3,
-			}},
-		},
-		{
-			EndpointsParam: " s01.frostfs.devenv:8080  s02.frostfs.devenv:8080 ",
-			ExpectedResult: []endpointInfo{
-				{
-					Address:  "s01.frostfs.devenv:8080",
-					Priority: 1,
-					Weight:   1,
-				},
-				{
-					Address:  "s02.frostfs.devenv:8080",
-					Priority: 1,
-					Weight:   1,
-				},
-			},
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,1,1 s02.frostfs.devenv:8080,2,1 s03.frostfs.devenv:8080,2,9",
-			ExpectedResult: []endpointInfo{
-				{
-					Address:  "s01.frostfs.devenv:8080",
-					Priority: 1,
-					Weight:   1,
-				},
-				{
-					Address:  "s02.frostfs.devenv:8080",
-					Priority: 2,
-					Weight:   1,
-				},
-				{
-					Address:  "s03.frostfs.devenv:8080",
-					Priority: 2,
-					Weight:   9,
-				},
-			},
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,-1,1",
-			ExpectedError:  true,
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,,",
-			ExpectedError:  true,
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,sd,sd",
-			ExpectedError:  true,
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,1,0",
-			ExpectedError:  true,
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,1 s02.frostfs.devenv:8080",
-			ExpectedError:  true,
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,1,2 s02.frostfs.devenv:8080",
-			ExpectedError:  true,
-		},
-		{
-			EndpointsParam: "s01.frostfs.devenv:8080,1,2 s02.frostfs.devenv:8080,1",
-			ExpectedError:  true,
-		},
-	} {
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-			res, err := parseEndpoints(tc.EndpointsParam)
-			if tc.ExpectedError {
-				require.Error(t, err)
-				return
-			}
-
-			require.NoError(t, err)
-			require.Equal(t, tc.ExpectedResult, res)
-		})
-	}
-}

backend/ftp/ftp.go

@@ -85,7 +85,7 @@ to an encrypted one. Cannot be used in combination with implicit FTPS.`,
 			Default: false,
 		}, {
 			Name: "concurrency",
-			Help: strings.ReplaceAll(`Maximum number of FTP simultaneous connections, 0 for unlimited.
+			Help: strings.Replace(`Maximum number of FTP simultaneous connections, 0 for unlimited.
 
 Note that setting this is very likely to cause deadlocks so it should
 be used with care.
@@ -99,7 +99,7 @@ maximum of |--checkers| and |--transfers|.
 So for |concurrency 3| you'd use |--checkers 2 --transfers 2
 --check-first| or |--checkers 1 --transfers 1|.
 
-`, "|", "`"),
+`, "|", "`", -1),
 			Default:  0,
 			Advanced: true,
 		}, {
@@ -970,8 +970,6 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	f.putFtpConnection(&c, err)
 	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
-		case ftp.StatusRequestedFileActionOK: // some ftp servers apparently return 250 instead of 257
-			err = nil // see: https://forum.rclone.org/t/rclone-pop-up-an-i-o-error-when-creating-a-folder-in-a-mounted-ftp-drive/44368/
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
 		case 521: // dir already exists: error number according to RFC 959: issue #2363

backend/gofile/api/types.go

@@ -1,311 +0,0 @@
-// Package api has type definitions for gofile
-//
-// Converted from the API docs with help from https://mholt.github.io/json-to-go/
-package api
-
-import (
-	"fmt"
-	"time"
-)
-
-const (
-	// 2017-05-03T07:26:10-07:00
-	timeFormat = `"` + time.RFC3339 + `"`
-)
-
-// Time represents date and time information for the
-// gofile API, by using RFC3339
-type Time time.Time
-
-// MarshalJSON turns a Time into JSON (in UTC)
-func (t *Time) MarshalJSON() (out []byte, err error) {
-	timeString := (*time.Time)(t).Format(timeFormat)
-	return []byte(timeString), nil
-}
-
-// UnmarshalJSON turns JSON into a Time
-func (t *Time) UnmarshalJSON(data []byte) error {
-	newT, err := time.Parse(timeFormat, string(data))
-	if err != nil {
-		return err
-	}
-	*t = Time(newT)
-	return nil
-}
-
-// Error is returned from gofile when things go wrong
-type Error struct {
-	Status string `json:"status"`
-}
-
-// Error returns a string for the error and satisfies the error interface
-func (e Error) Error() string {
-	out := fmt.Sprintf("Error %q", e.Status)
-	return out
-}
-
-// IsError returns true if there is an error
-func (e Error) IsError() bool {
-	return e.Status != "ok"
-}
-
-// Err returns err if not nil, or e if IsError or nil
-func (e Error) Err(err error) error {
-	if err != nil {
-		return err
-	}
-	if e.IsError() {
-		return e
-	}
-	return nil
-}
-
-// Check Error satisfies the error interface
-var _ error = (*Error)(nil)
-
-// Types of things in Item
-const (
-	ItemTypeFolder = "folder"
-	ItemTypeFile   = "file"
-)
-
-// Item describes a folder or a file as returned by /contents
-type Item struct {
-	ID            string                 `json:"id"`
-	ParentFolder  string                 `json:"parentFolder"`
-	Type          string                 `json:"type"`
-	Name          string                 `json:"name"`
-	Size          int64                  `json:"size"`
-	Code          string                 `json:"code"`
-	CreateTime    int64                  `json:"createTime"`
-	ModTime       int64                  `json:"modTime"`
-	Link          string                 `json:"link"`
-	MD5           string                 `json:"md5"`
-	MimeType      string                 `json:"mimetype"`
-	ChildrenCount int                    `json:"childrenCount"`
-	DirectLinks   map[string]*DirectLink `json:"directLinks"`
-	//Public         bool     `json:"public"`
-	//ServerSelected string   `json:"serverSelected"`
-	//Thumbnail      string   `json:"thumbnail"`
-	//DownloadCount int      `json:"downloadCount"`
-	//TotalDownloadCount int64            `json:"totalDownloadCount"`
-	//TotalSize int64            `json:"totalSize"`
-	//ChildrenIDs   []string               `json:"childrenIds"`
-	Children map[string]*Item `json:"children"`
-}
-
-// ToNativeTime converts a go time to a native time
-func ToNativeTime(t time.Time) int64 {
-	return t.Unix()
-}
-
-// FromNativeTime converts native time to a go time
-func FromNativeTime(t int64) time.Time {
-	return time.Unix(t, 0)
-}
-
-// DirectLink describes a direct link to a file so it can be
-// downloaded by third parties.
-type DirectLink struct {
-	ExpireTime       int64  `json:"expireTime"`
-	SourceIpsAllowed []any  `json:"sourceIpsAllowed"`
-	DomainsAllowed   []any  `json:"domainsAllowed"`
-	Auth             []any  `json:"auth"`
-	IsReqLink        bool   `json:"isReqLink"`
-	DirectLink       string `json:"directLink"`
-}
-
-// Contents is returned from the /contents call
-type Contents struct {
-	Error
-	Data struct {
-		Item
-	} `json:"data"`
-	Metadata Metadata `json:"metadata"`
-}
-
-// Metadata is returned when paging is in use
-type Metadata struct {
-	TotalCount  int  `json:"totalCount"`
-	TotalPages  int  `json:"totalPages"`
-	Page        int  `json:"page"`
-	PageSize    int  `json:"pageSize"`
-	HasNextPage bool `json:"hasNextPage"`
-}
-
-// AccountsGetID is the result of /accounts/getid
-type AccountsGetID struct {
-	Error
-	Data struct {
-		ID string `json:"id"`
-	} `json:"data"`
-}
-
-// Stats of storage and traffic
-type Stats struct {
-	FolderCount            int64 `json:"folderCount"`
-	FileCount              int64 `json:"fileCount"`
-	Storage                int64 `json:"storage"`
-	TrafficDirectGenerated int64 `json:"trafficDirectGenerated"`
-	TrafficReqDownloaded   int64 `json:"trafficReqDownloaded"`
-	TrafficWebDownloaded   int64 `json:"trafficWebDownloaded"`
-}
-
-// AccountsGet is the result of /accounts/{id}
-type AccountsGet struct {
-	Error
-	Data struct {
-		ID                             string `json:"id"`
-		Email                          string `json:"email"`
-		Tier                           string `json:"tier"`
-		PremiumType                    string `json:"premiumType"`
-		Token                          string `json:"token"`
-		RootFolder                     string `json:"rootFolder"`
-		SubscriptionProvider           string `json:"subscriptionProvider"`
-		SubscriptionEndDate            int    `json:"subscriptionEndDate"`
-		SubscriptionLimitDirectTraffic int64  `json:"subscriptionLimitDirectTraffic"`
-		SubscriptionLimitStorage       int64  `json:"subscriptionLimitStorage"`
-		StatsCurrent                   Stats  `json:"statsCurrent"`
-		// StatsHistory                   map[int]map[int]map[int]Stats `json:"statsHistory"`
-	} `json:"data"`
-}
-
-// CreateFolderRequest is the input to /contents/createFolder
-type CreateFolderRequest struct {
-	ParentFolderID string `json:"parentFolderId"`
-	FolderName     string `json:"folderName"`
-	ModTime        int64  `json:"modTime,omitempty"`
-}
-
-// CreateFolderResponse is the output from /contents/createFolder
-type CreateFolderResponse struct {
-	Error
-	Data Item `json:"data"`
-}
-
-// DeleteRequest is the input to DELETE /contents
-type DeleteRequest struct {
-	ContentsID string `json:"contentsId"` // comma separated list of IDs
-}
-
-// DeleteResponse is the input to DELETE /contents
-type DeleteResponse struct {
-	Error
-	Data map[string]Error
-}
-
-// Server is an upload server
-type Server struct {
-	Name string `json:"name"`
-	Zone string `json:"zone"`
-}
-
-// String returns a string representation of the Server
-func (s *Server) String() string {
-	return fmt.Sprintf("%s (%s)", s.Name, s.Zone)
-}
-
-// Root returns the root URL for the server
-func (s *Server) Root() string {
-	return fmt.Sprintf("https://%s.gofile.io/", s.Name)
-}
-
-// URL returns the upload URL for the server
-func (s *Server) URL() string {
-	return fmt.Sprintf("https://%s.gofile.io/contents/uploadfile", s.Name)
-}
-
-// ServersResponse is the output from /servers
-type ServersResponse struct {
-	Error
-	Data struct {
-		Servers []Server `json:"servers"`
-	} `json:"data"`
-}
-
-// UploadResponse is returned by POST /contents/uploadfile
-type UploadResponse struct {
-	Error
-	Data Item `json:"data"`
-}
-
-// DirectLinksRequest specifies the parameters for the direct link
-type DirectLinksRequest struct {
-	ExpireTime       int64 `json:"expireTime,omitempty"`
-	SourceIpsAllowed []any `json:"sourceIpsAllowed,omitempty"`
-	DomainsAllowed   []any `json:"domainsAllowed,omitempty"`
-	Auth             []any `json:"auth,omitempty"`
-}
-
-// DirectLinksResult is returned from POST /contents/{id}/directlinks
-type DirectLinksResult struct {
-	Error
-	Data struct {
-		ExpireTime       int64  `json:"expireTime"`
-		SourceIpsAllowed []any  `json:"sourceIpsAllowed"`
-		DomainsAllowed   []any  `json:"domainsAllowed"`
-		Auth             []any  `json:"auth"`
-		IsReqLink        bool   `json:"isReqLink"`
-		ID               string `json:"id"`
-		DirectLink       string `json:"directLink"`
-	} `json:"data"`
-}
-
-// UpdateItemRequest describes the updates to be done to an item for PUT /contents/{id}/update
-//
-// The Value of the attribute to define :
-// For Attribute "name" : The name of the content (file or folder)
-// For Attribute "description" : The description displayed on the download page (folder only)
-// For Attribute "tags" : A comma-separated list of tags (folder only)
-// For Attribute "public" : either true or false (folder only)
-// For Attribute "expiry" : A unix timestamp of the expiration date (folder only)
-// For Attribute "password" : The password to set (folder only)
-type UpdateItemRequest struct {
-	Attribute string `json:"attribute"`
-	Value     any    `json:"attributeValue"`
-}
-
-// UpdateItemResponse is returned by PUT /contents/{id}/update
-type UpdateItemResponse struct {
-	Error
-	Data Item `json:"data"`
-}
-
-// MoveRequest is the input to /contents/move
-type MoveRequest struct {
-	FolderID   string `json:"folderId"`
-	ContentsID string `json:"contentsId"` // comma separated list of IDs
-}
-
-// MoveResponse is returned by POST /contents/move
-type MoveResponse struct {
-	Error
-	Data map[string]struct {
-		Error
-		Item `json:"data"`
-	} `json:"data"`
-}
-
-// CopyRequest is the input to /contents/copy
-type CopyRequest struct {
-	FolderID   string `json:"folderId"`
-	ContentsID string `json:"contentsId"` // comma separated list of IDs
-}
-
-// CopyResponse is returned by POST /contents/copy
-type CopyResponse struct {
-	Error
-	Data map[string]struct {
-		Error
-		Item `json:"data"`
-	} `json:"data"`
-}
-
-// UploadServerStatus is returned when fetching the root of an upload server
-type UploadServerStatus struct {
-	Error
-	Data struct {
-		Server string `json:"server"`
-		Test   string `json:"test"`
-	} `json:"data"`
-}

backend/gofile/gofile_test.go

@@ -1,17 +0,0 @@
-// Test Gofile filesystem interface
-package gofile_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/gofile"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestGoFile:",
-		NilObject:  (*gofile.Object)(nil),
-	})
-}

backend/googlecloudstorage/googlecloudstorage.go

@@ -697,7 +697,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			// is this a directory marker?
 			if isDirectory {
 				// Don't insert the root directory
-				if remote == f.opt.Enc.ToStandardPath(directory) {
+				if remote == directory {
 					continue
 				}
 				// process directory markers as directories

backend/googlephotos/api/types.go

@@ -56,7 +56,8 @@ type MediaItem struct {
 		CreationTime time.Time `json:"creationTime"`
 		Width        string    `json:"width"`
 		Height       string    `json:"height"`
-		Photo        struct{}  `json:"photo"`
+		Photo        struct {
+		} `json:"photo"`
 	} `json:"mediaMetadata"`
 	Filename string `json:"filename"`
 }
@@ -67,7 +68,7 @@ type MediaItems struct {
 	NextPageToken string      `json:"nextPageToken"`
 }
 
-// Content categories
+//Content categories
 // NONE	Default content category. This category is ignored when any other category is used in the filter.
 // LANDSCAPES	Media items containing landscapes.
 // RECEIPTS	Media items containing receipts.
@@ -186,5 +187,5 @@ type BatchCreateResponse struct {
 
 // BatchRemoveItems is for removing items from an album
 type BatchRemoveItems struct {
-	MediaItemIDs []string `json:"mediaItemIds"`
+	MediaItemIds []string `json:"mediaItemIds"`
 }

backend/googlephotos/googlephotos.go

@@ -280,7 +280,7 @@ func errorHandler(resp *http.Response) error {
 	if strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
 		body = []byte("Image not found or broken")
 	}
-	e := api.Error{
+	var e = api.Error{
 		Details: api.ErrorDetails{
 			Code:    resp.StatusCode,
 			Message: string(body),
@@ -620,7 +620,9 @@ func (f *Fs) listDir(ctx context.Context, prefix string, filter api.SearchFilter
 		if err != nil {
 			return err
 		}
-		entries = append(entries, entry)
+		if entry != nil {
+			entries = append(entries, entry)
+		}
 		return nil
 	})
 	if err != nil {
@@ -700,7 +702,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
 		Path:       "/albums",
 		Parameters: url.Values{},
 	}
-	request := api.CreateAlbum{
+	var request = api.CreateAlbum{
 		Album: &api.Album{
 			Title: albumTitle,
 		},
@@ -1000,7 +1002,7 @@ func (f *Fs) commitBatchAlbumID(ctx context.Context, items []uploadedItem, resul
 		Method: "POST",
 		Path:   "/mediaItems:batchCreate",
 	}
-	request := api.BatchCreateRequest{
+	var request = api.BatchCreateRequest{
 		AlbumID: albumID,
 	}
 	itemsInBatch := 0
@@ -1172,8 +1174,8 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		Path:       "/albums/" + album.ID + ":batchRemoveMediaItems",
 		NoResponse: true,
 	}
-	request := api.BatchRemoveItems{
-		MediaItemIDs: []string{o.id},
+	var request = api.BatchRemoveItems{
+		MediaItemIds: []string{o.id},
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {

backend/googlephotos/pattern.go

@@ -38,7 +38,7 @@ type dirPattern struct {
 	toEntries func(ctx context.Context, f lister, prefix string, match []string) (fs.DirEntries, error)
 }
 
-// dirPatterns is a slice of all the directory patterns
+// dirPatters is a slice of all the directory patterns
 type dirPatterns []dirPattern
 
 // patterns describes the layout of the google photos backend file system.

backend/hasher/hasher.go

@@ -164,21 +164,16 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 	}
 
 	stubFeatures := &fs.Features{
-		CanHaveEmptyDirectories:  true,
-		IsLocal:                  true,
-		ReadMimeType:             true,
-		WriteMimeType:            true,
-		SetTier:                  true,
-		GetTier:                  true,
-		ReadMetadata:             true,
-		WriteMetadata:            true,
-		UserMetadata:             true,
-		ReadDirMetadata:          true,
-		WriteDirMetadata:         true,
-		WriteDirSetModTime:       true,
-		UserDirMetadata:          true,
-		DirModTimeUpdatesOnWrite: true,
-		PartialUploads:           true,
+		CanHaveEmptyDirectories: true,
+		IsLocal:                 true,
+		ReadMimeType:            true,
+		WriteMimeType:           true,
+		SetTier:                 true,
+		GetTier:                 true,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            true,
+		PartialUploads:          true,
 	}
 	f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs)
 
@@ -346,22 +341,6 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	return errors.New("MergeDirs not supported")
 }
 
-// DirSetModTime sets the directory modtime for dir
-func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
-	if do := f.Fs.Features().DirSetModTime; do != nil {
-		return do(ctx, dir, modTime)
-	}
-	return fs.ErrorNotImplemented
-}
-
-// MkdirMetadata makes the root directory of the Fs object
-func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
-	if do := f.Fs.Features().MkdirMetadata; do != nil {
-		return do(ctx, dir, metadata)
-	}
-	return nil, fs.ErrorNotImplemented
-}
-
 // DirCacheFlush resets the directory cache - used in testing
 // as an optional interface
 func (f *Fs) DirCacheFlush() {
@@ -439,7 +418,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Shutdown the backend, closing any background tasks and any cached connections.
 func (f *Fs) Shutdown(ctx context.Context) (err error) {
-	if f.db != nil && !f.db.IsStopped() {
+	if f.db != nil {
 		err = f.db.Stop(false)
 	}
 	if do := f.Fs.Features().Shutdown; do != nil {
@@ -535,17 +514,6 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
-// SetMetadata sets metadata for an Object
-//
-// It should return fs.ErrorNotImplemented if it can't set metadata
-func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
-	do, ok := o.Object.(fs.SetMetadataer)
-	if !ok {
-		return fs.ErrorNotImplemented
-	}
-	return do.SetMetadata(ctx, metadata)
-}
-
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -562,8 +530,6 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Wrapper         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
-	_ fs.DirSetModTimer  = (*Fs)(nil)
-	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)

backend/hasher/object.go

@@ -71,14 +71,7 @@ func (o *Object) Hash(ctx context.Context, hashType hash.Type) (hashVal string,
 	f := o.f
 	if f.passHashes.Contains(hashType) {
 		fs.Debugf(o, "pass %s", hashType)
-		hashVal, err = o.Object.Hash(ctx, hashType)
-		if hashVal != "" {
-			return hashVal, err
-		}
-		if err != nil {
-			fs.Debugf(o, "error passing %s: %v", hashType, err)
-		}
-		fs.Debugf(o, "passed %s is blank -- trying other methods", hashType)
+		return o.Object.Hash(ctx, hashType)
 	}
 	if !f.suppHashes.Contains(hashType) {
 		fs.Debugf(o, "unsupp %s", hashType)

backend/hdfs/fs.go

@@ -1,4 +1,5 @@
 //go:build !plan9
+// +build !plan9
 
 package hdfs
 
@@ -149,7 +150,7 @@ func (f *Fs) Root() string {
 
 // String returns a description of the FS
 func (f *Fs) String() string {
-	return fmt.Sprintf("hdfs://%s/%s", f.opt.Namenode, f.root)
+	return fmt.Sprintf("hdfs://%s", f.opt.Namenode)
 }
 
 // Features returns the optional features of this Fs
@@ -209,8 +210,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				fs:      f,
 				remote:  remote,
 				size:    x.Size(),
-				modTime: x.ModTime(),
-			})
+				modTime: x.ModTime()})
 		}
 	}
 	return entries, nil

backend/hdfs/hdfs.go

@@ -1,4 +1,5 @@
 //go:build !plan9
+// +build !plan9
 
 // Package hdfs provides an interface to the HDFS storage system.
 package hdfs

backend/hdfs/hdfs_test.go

@@ -1,6 +1,7 @@
 // Test HDFS filesystem interface
 
 //go:build !plan9
+// +build !plan9
 
 package hdfs_test
 

backend/hdfs/hdfs_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9
+// +build plan9
 
-// Package hdfs provides an interface to the HDFS storage system.
 package hdfs

backend/hdfs/object.go

@@ -1,4 +1,5 @@
 //go:build !plan9
+// +build !plan9
 
 package hdfs
 

backend/http/http.go

@@ -89,10 +89,6 @@ that directory listings are much quicker, but rclone won't have the times or
 sizes of any files, and some files that don't exist may be in the listing.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name:    "no_escape",
-			Help:    "Do not escape URL metacharacters in path names.",
-			Default: false,
 		}},
 	}
 	fs.Register(fsi)
@@ -104,7 +100,6 @@ type Options struct {
 	NoSlash  bool            `config:"no_slash"`
 	NoHead   bool            `config:"no_head"`
 	Headers  fs.CommaSepList `config:"headers"`
-	NoEscape bool            `config:"no_escape"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -331,11 +326,6 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Join's the remote onto the base URL
 func (f *Fs) url(remote string) string {
-	if f.opt.NoEscape {
-		// Directly concatenate without escaping, no_escape behavior
-		return f.endpointURL + remote
-	}
-	// Default behavior
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 

backend/imagekit/client/url.go

@@ -56,7 +56,7 @@ func (ik *ImageKit) URL(params URLParam) (string, error) {
 		var expires = strconv.FormatInt(now+params.ExpireSeconds, 10)
 		var path = strings.Replace(resultURL, endpoint, "", 1)
 
-		path += expires
+		path = path + expires
 		mac := hmac.New(sha1.New, []byte(ik.PrivateKey))
 		mac.Write([]byte(path))
 		signature := hex.EncodeToString(mac.Sum(nil))

backend/jottacloud/jottacloud.go

@@ -1487,38 +1487,16 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorCantMove
 	}
 
-	meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
+	err := f.mkParentDir(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
-
-	if err := f.mkParentDir(ctx, remote); err != nil {
-		return nil, err
-	}
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
-	if err == nil {
-		var createTime time.Time
-		var createTimeMeta bool
-		var modTime time.Time
-		var modTimeMeta bool
-		if meta != nil {
-			createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
-			if !createTimeMeta {
-				createTime = srcObj.createTime
-			}
-			modTime, modTimeMeta = srcObj.parseFsMetadataTime(meta, "mtime")
-			if !modTimeMeta {
-				modTime = srcObj.modTime
-			}
-		}
-		if bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
-			// Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
-			fs.Debugf(src, "Server-side copied to trashed destination, restoring")
-			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
-		} else if createTimeMeta || modTimeMeta {
-			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
-		}
+	// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
+	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
+		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
+		info, err = f.createOrUpdate(ctx, remote, srcObj.createTime, srcObj.modTime, srcObj.size, srcObj.md5)
 	}
 
 	if err != nil {
@@ -1545,30 +1523,12 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorCantMove
 	}
 
-	meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
+	err := f.mkParentDir(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
-
-	if err := f.mkParentDir(ctx, remote); err != nil {
-		return nil, err
-	}
 	info, err := f.copyOrMove(ctx, "mv", srcObj.filePath(), remote)
 
-	if err == nil && meta != nil {
-		createTime, createTimeMeta := srcObj.parseFsMetadataTime(meta, "btime")
-		if !createTimeMeta {
-			createTime = srcObj.createTime
-		}
-		modTime, modTimeMeta := srcObj.parseFsMetadataTime(meta, "mtime")
-		if !modTimeMeta {
-			modTime = srcObj.modTime
-		}
-		if createTimeMeta || modTimeMeta {
-			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
-		}
-	}
-
 	if err != nil {
 		return nil, fmt.Errorf("couldn't move file: %w", err)
 	}
@@ -1826,20 +1786,6 @@ func (o *Object) readMetaData(ctx context.Context, force bool) (err error) {
 	return o.setMetaData(info)
 }
 
-// parseFsMetadataTime parses a time string from fs.Metadata with key
-func (o *Object) parseFsMetadataTime(m fs.Metadata, key string) (t time.Time, ok bool) {
-	value, ok := m[key]
-	if ok {
-		var err error
-		t, err = time.Parse(time.RFC3339Nano, value) // metadata stores RFC3339Nano timestamps
-		if err != nil {
-			fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
-			ok = false
-		}
-	}
-	return t, ok
-}
-
 // ModTime returns the modification time of the object
 //
 // It attempts to read the objects mtime and if that isn't present the
@@ -2011,11 +1957,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var createdTime string
 	var modTime string
 	if meta != nil {
-		if t, ok := o.parseFsMetadataTime(meta, "btime"); ok {
-			createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
+		if v, ok := meta["btime"]; ok {
+			t, err := time.Parse(time.RFC3339Nano, v) // metadata stores RFC3339Nano timestamps
+			if err != nil {
+				fs.Debugf(o, "failed to parse metadata btime: %q: %v", v, err)
+			} else {
+				createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
+			}
 		}
-		if t, ok := o.parseFsMetadataTime(meta, "mtime"); ok {
-			modTime = api.Rfc3339Time(t).String()
+		if v, ok := meta["mtime"]; ok {
+			t, err := time.Parse(time.RFC3339Nano, v)
+			if err != nil {
+				fs.Debugf(o, "failed to parse metadata mtime: %q: %v", v, err)
+			} else {
+				modTime = api.Rfc3339Time(t).String()
+			}
 		}
 	}
 	if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime

backend/jottacloud/jottacloud_internal_test.go

@@ -59,7 +59,7 @@ func (f *Fs) InternalTestMetadata(t *testing.T) {
 		//"utime" - read-only
 		//"content-type" - read-only
 	}
-	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, false, contents, true, "text/html", metadata)
+	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, "text/html", metadata)
 	defer func() {
 		assert.NoError(t, obj.Remove(ctx))
 	}()

backend/koofr/koofr.go

@@ -67,13 +67,13 @@ func init() {
 			Sensitive: true,
 		}, {
 			Name:       "password",
-			Help:       "Your password for rclone generate one at https://app.koofr.net/app/admin/preferences/password.",
+			Help:       "Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password).",
 			Provider:   "koofr",
 			IsPassword: true,
 			Required:   true,
 		}, {
 			Name:       "password",
-			Help:       "Your password for rclone generate one at https://storage.rcs-rds.ro/app/admin/preferences/password.",
+			Help:       "Your password for rclone (generate one at https://storage.rcs-rds.ro/app/admin/preferences/password).",
 			Provider:   "digistorage",
 			IsPassword: true,
 			Required:   true,

backend/linkbox/linkbox.go

@@ -36,7 +36,7 @@ import (
 )
 
 const (
-	maxEntitiesPerPage = 1000
+	maxEntitiesPerPage = 1024
 	minSleep           = 200 * time.Millisecond
 	maxSleep           = 2 * time.Second
 	pacerBurst         = 1
@@ -219,8 +219,7 @@ type listAllFn func(*entity) bool
 // Search is a bit fussy about which characters match
 //
 // If the name doesn't match this then do an dir list instead
-// N.B.: Linkbox doesn't support search by name that is longer than 50 chars
-var searchOK = regexp.MustCompile(`^[a-zA-Z0-9_ -.]{1,50}$`)
+var searchOK = regexp.MustCompile(`^[a-zA-Z0-9_ .]+$`)
 
 // Lists the directory required calling the user function on each item found
 //
@@ -239,7 +238,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, name string, fn listAllF
 		// If name isn't good then do an unbounded search
 		name = ""
 	}
-
 OUTER:
 	for numberOfEntities == maxEntitiesPerPage {
 		pageNumber++
@@ -260,6 +258,7 @@ OUTER:
 		err = getUnmarshaledResponse(ctx, f, opts, &responseResult)
 		if err != nil {
 			return false, fmt.Errorf("getting files failed: %w", err)
+
 		}
 
 		numberOfEntities = len(responseResult.SearchData.Entities)

backend/linkbox/linkbox_test.go

@@ -13,7 +13,5 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestLinkbox:",
 		NilObject:  (*linkbox.Object)(nil),
-		// Linkbox doesn't support leading dots for files
-		SkipLeadingDot: true,
 	})
 }

backend/local/about_unix.go

@@ -1,4 +1,5 @@
 //go:build darwin || dragonfly || freebsd || linux
+// +build darwin dragonfly freebsd linux
 
 package local
 
@@ -23,9 +24,9 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	}
 	bs := int64(s.Bsize) // nolint: unconvert
 	usage := &fs.Usage{
-		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         //nolint: unconvert // quota of bytes that can be used
-		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), //nolint: unconvert // bytes in use
-		Free:  fs.NewUsageValue(bs * int64(s.Bavail)),         //nolint: unconvert // bytes which can be uploaded before reaching the quota
+		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         // quota of bytes that can be used
+		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use
+		Free:  fs.NewUsageValue(bs * int64(s.Bavail)),         // bytes which can be uploaded before reaching the quota
 	}
 	return usage, nil
 }

backend/local/about_windows.go

@@ -1,4 +1,5 @@
 //go:build windows
+// +build windows
 
 package local