Revert "drive: supply correct scopes to when using --drive-impersonate"

This reverts commit b8280521a5.

.appveyor.yml

@@ -0,0 +1,49 @@
+version: "{build}"
+
+os: Windows Server 2012 R2
+
+clone_folder: c:\gopath\src\github.com\ncw\rclone
+
+cache:
+  - '%LocalAppData%\go-build'
+
+environment:
+  GOPATH: C:\gopath
+  CPATH: C:\Program Files (x86)\WinFsp\inc\fuse
+  ORIGPATH: '%PATH%'
+  NOCCPATH: C:\MinGW\bin;%GOPATH%\bin;%PATH%
+  PATHCC64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin;%NOCCPATH%
+  PATHCC32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%NOCCPATH%
+  PATH: '%PATHCC64%'
+  RCLONE_CONFIG_PASS:
+    secure: HbzxSy9zQ8NYWN9NNPf6ALQO9Q0mwRNqwehsLcOEHy0=
+
+install:
+- choco install winfsp -y
+- choco install zip -y
+- copy c:\MinGW\bin\mingw32-make.exe c:\MinGW\bin\make.exe
+
+build_script:
+- echo %PATH%
+- echo %GOPATH%
+- go version
+- go env
+- go install
+- go build
+- make log_since_last_release > %TEMP%\git-log.txt
+- make version > %TEMP%\version
+- set /p RCLONE_VERSION=<%TEMP%\version
+- set PATH=%PATHCC32%
+- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/386" -cgo -tags cmount %RCLONE_VERSION%
+- set PATH=%PATHCC64%
+- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/amd64" -cgo -no-clean -tags cmount %RCLONE_VERSION%
+
+test_script:
+- make GOTAGS=cmount quicktest
+
+artifacts:
+- path: rclone.exe
+- path: build/*-v*.zip
+
+deploy_script:
+- IF "%APPVEYOR_REPO_NAME%" == "ncw/rclone" IF "%APPVEYOR_PULL_REQUEST_NUMBER%" == "" make appveyor_upload

.circleci/config.yml

@@ -0,0 +1,50 @@
+---
+version: 2
+
+jobs:
+
+  build:
+    machine: true
+
+    working_directory: ~/.go_workspace/src/github.com/ncw/rclone
+
+    steps:
+    - checkout
+
+    - run:
+        name: Cross-compile rclone
+        command: |
+            docker pull rclone/xgo-cgofuse
+            go get -v github.com/karalabe/xgo
+            xgo \
+                --image=rclone/xgo-cgofuse \
+                --targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
+                -tags cmount \
+                .
+            xgo \
+                --targets=android/*,ios/* \
+                .
+
+    - run:
+        name: Prepare artifacts
+        command: |
+            mkdir -p /tmp/rclone.dist
+            cp -R rclone-* /tmp/rclone.dist
+            mkdir build
+            cp -R rclone-* build/
+
+    - run:
+        name: Build rclone
+        command: |
+            go version
+            go build
+
+    - run:
+        name: Upload artifacts
+        command: |
+          if [[ $CIRCLE_PULL_REQUEST != "" ]]; then
+              make circleci_upload
+          fi
+
+    - store_artifacts:
+        path: /tmp/rclone.dist

.forgejo/ISSUE_TEMPLATE/bug_report.md

@@ -1,45 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: community, triage, bug
-assignees: ''
-
----
-
-<!--- Provide a general summary of the issue in the Title above -->
-
-## Expected Behavior
-<!--- If you're describing a bug, tell us what should happen -->
-<!--- If you're suggesting a change/improvement, tell us how it should work -->
-
-## Current Behavior
-<!--- If describing a bug, tell us what happens instead of the expected behavior -->
-<!--- If suggesting a change/improvement, explain the difference from current behavior -->
-
-## Possible Solution
-<!--- Not obligatory -->
-<!--- If no reason/fix/additions for the bug can be suggested, -->
-<!--- uncomment the following phrase: -->
-
-<!--- No fix can be suggested by a QA engineer. Further solutions shall be up to developers. -->
-
-## Steps to Reproduce (for bugs)
-<!--- Provide a link to a live example, or an unambiguous set of steps to -->
-<!--- reproduce this bug. -->
-
-1.
-
-## Context
-<!--- How has this issue affected you? What are you trying to accomplish? -->
-<!--- Providing context helps us come up with a solution that is most useful in the real world -->
-
-## Regression
-<!-- Is this issue a regression? (Yes / No) -->
-<!-- If Yes, optionally please include version or commit id or PR# that caused this regression, if you have these details. -->
-
-## Your Environment
-<!--- Include as many relevant details about the environment you experienced the bug in -->
-* Version used:
-* Server setup and configuration:
-* Operating System and version (`uname -a`):

.forgejo/ISSUE_TEMPLATE/config.yml

@@ -1 +0,0 @@
-blank_issues_enabled: false

.forgejo/workflows/builds.yaml

@@ -1,24 +0,0 @@
-on:
-  pull_request:
-  push:
-    branches:
-      - tcl/master
-
-jobs:
-  builds:
-    name: Builds
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go_versions: [ '1.22', '1.23' ]
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '${{ matrix.go_versions }}'
-
-      - name: Build binary
-        run: make

.forgejo/workflows/dco.yml

@@ -1,20 +0,0 @@
-on: [pull_request]
-
-jobs:
-  dco:
-    name: DCO
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-         fetch-depth: 0
-
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.23'
-
-      - name: Run commit format checker
-        uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3
-        with:
-          from: 'origin/${{ github.event.pull_request.base.ref }}'

.forgejo/workflows/tests.yml

@@ -1,67 +0,0 @@
-on:
-  pull_request:
-  push:
-    branches:
-      - tcl/master
-
-jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.23'
-          cache: true
-
-      - name: Install linters
-        run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
-
-      - name: Run linters
-        run: make check
-  test:
-    name: Test
-    runs-on: oci-runner
-    strategy:
-      matrix:
-        go_versions: [ '1.23' ]
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '${{ matrix.go_versions }}'
-
-      - name: Tests for the FrostFS backend
-        env:
-          RESTIC_TEST_FUSE: false
-          AIO_IMAGE: truecloudlab/frostfs-aio
-          AIO_VERSION: 1.7.0-nightly.4
-          RCLONE_CONFIG: /config/rclone.conf
-
-        # run only tests related to FrostFS backend
-        run: |-
-          podman-service.sh
-          podman info
-          
-          mkdir /config
-          printf "[TestFrostFS]\ntype = frostfs\nendpoint = localhost:8080\nwallet = /config/wallet.json\nplacement_policy = REP 1\nrequest_timeout = 20s\nconnection_timeout = 21s" > /config/rclone.conf
-          
-          echo "Run frostfs aio container"
-          docker run -d --net=host --name aio $AIO_IMAGE:$AIO_VERSION --restart always -p 8080:8080
-                            
-          echo "Wait for frostfs to start"
-          until docker exec aio curl --fail http://localhost:8083 > /dev/null 2>&1; do sleep 0.2; done;
-          
-          echo "Issue creds"
-          docker exec aio /usr/bin/issue-creds.sh native
-          echo "Copy wallet"
-          docker cp aio:/config/user-wallet.json /config/wallet.json
-
-          echo "Start tests"
-          go test -v github.com/rclone/rclone/backend/frostfs

.gitattributes

@@ -1,11 +0,0 @@
-# Go writes go.mod and go.sum with lf even on windows
-go.mod text eol=lf
-go.sum text eol=lf
-
-# Ignore generated files in GitHub language statistics and diffs
-/MANUAL.* linguist-generated=true
-/rclone.1 linguist-generated=true
-
-# Don't fiddle with the line endings of test data
-**/testdata/** -text
-**/test/** -text

.github/ISSUE_TEMPLATE.md

@@ -10,7 +10,7 @@ instead of filing an issue for a quick response.
 
 If you are reporting a bug or asking for a new feature then please use one of the templates here:
 
-    https://github.com/rclone/rclone/issues/new
+    https://github.com/ncw/rclone/issues/new
 
 otherwise fill in the form below.
 

.github/ISSUE_TEMPLATE/Bug.md

@@ -5,31 +5,19 @@ about: Report a problem with rclone
 
 <!--
 
-We understand you are having a problem with rclone; we want to help you with that!
+Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
 
-**STOP and READ**
-**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
-Please show the effort you've put into solving the problem and please be specific.
-People are volunteering their time to help! Low effort posts are not likely to get good answers!
-
-If you think you might have found a bug, try to replicate it with the latest beta (or stable).
-The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
-
-If you can still replicate it or just got a question then please use the rclone forum:
+If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
 
     https://forum.rclone.org/
 
-for a quick response instead of filing an issue on this repo.
+instead of filing an issue for a quick response.
 
-If nothing else helps, then please fill in the info below which helps us help you.
+If you think you might have found a bug, please can you try to replicate it with the latest beta?
 
-**DO NOT REDACT** any information except passwords/keys/personal info.
+    https://beta.rclone.org/
     
-You should use 3 backticks to begin and end your paste to make it readable.
-
-Make sure to include a log obtained with '-vv'.
-
-You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
+If you can still replicate it with the latest beta, then please fill in the info below which makes our lives much easier.  A log with -vv will make our day :-)
 
 Thank you
 
@@ -37,10 +25,6 @@ The Rclone Developers
 
 -->
 
-#### The associated forum post URL from `https://forum.rclone.org`
-
-
-
 #### What is the problem you are having with rclone?
 
 
@@ -49,26 +33,18 @@ The Rclone Developers
 
 
 
-#### Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
+#### Which OS you are using and how many bits (eg Windows 7, 64 bit)
 
 
 
-#### Which cloud storage system are you using? (e.g. Google Drive)
+####  Which cloud storage system are you using? (eg Google Drive)
 
 
 
-#### The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
+#### The command you were trying to run (eg `rclone copy /tmp remote:tmp`)
 
 
 
-#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
+#### A log from the command with the `-vv` flag (eg output from `rclone -vv copy /tmp remote:tmp`)
 
 
-
-<!--- Please keep the note below for others who read your bug report. -->
-
-#### How to use GitHub
-
-* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
-* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
-* Subscribe to receive notifications on status change and new comments.

.github/ISSUE_TEMPLATE/Feature.md

@@ -7,16 +7,12 @@ about: Suggest a new feature or enhancement for rclone
 
 Welcome :-)
 
-So you've got an idea to improve rclone? We love that!
-You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
+So you've got an idea to improve rclone? We love that! You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
 
-Probably the latest beta (or stable) release has your feature, so try to update your rclone.
-The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
+Here is a checklist of things to do:
 
-If it still isn't there, here is a checklist of things to do:
-
-  1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
-  2. Discuss on the forum: https://forum.rclone.org/
+  1. Please search the old issues first for your idea and +1 or comment on an existing issue if possible.
+  2. Discuss on the forum first: https://forum.rclone.org/
   3. Make a feature request issue (this is the right place!).
   4. Be prepared to get involved making the feature :-)
 
@@ -26,9 +22,6 @@ The Rclone Developers
 
 -->
 
-#### The associated forum post URL from `https://forum.rclone.org`
-
-
 
 #### What is your current rclone version (output from `rclone version`)?
 
@@ -41,11 +34,3 @@ The Rclone Developers
 #### How do you think rclone should be changed to solve that?
 
 
-
-<!--- Please keep the note below for others who read your feature request. -->
-
-#### How to use GitHub
-
-* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
-* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
-* Subscribe to receive notifications on status change and new comments.

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Rclone Forum Community Support
-    url: https://forum.rclone.org/
-    about: Please ask and answer questions here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -22,8 +22,8 @@ Link issues and relevant forum posts here.
 
 #### Checklist
 
-- [ ] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
+- [ ] I have read the [contribution guidelines](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
 - [ ] I have added tests for all changes in this PR if appropriate.
 - [ ] I have added documentation for the changes if appropriate.
-- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
+- [ ] All commit messages are in [house style](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#commit-messages).
 - [ ] I'm done, this Pull Request is ready for review :-)

.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "daily"

.github/workflows/build.yml

@@ -1,394 +0,0 @@
----
-# Github Actions build for rclone
-# -*- compile-command: "yamllint -f parsable build.yml" -*-
-
-name: build
-
-# Trigger the workflow on push or pull request
-on:
-  push:
-    branches:
-      - '**'
-    tags:
-      - '**'
-  pull_request:
-  workflow_dispatch:
-    inputs:
-      manual:
-        description: Manual run (bypass default conditions)
-        type: boolean
-        required: true
-        default: true
-
-jobs:
-  build:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
-
-        include:
-          - job_name: linux
-            os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
-            gotags: cmount
-            build_flags: '-include "^linux/"'
-            check: true
-            quicktest: true
-            racequicktest: true
-            librclonetest: true
-            deploy: true
-
-          - job_name: linux_386
-            os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
-            goarch: 386
-            gotags: cmount
-            quicktest: true
-
-          - job_name: mac_amd64
-            os: macos-latest
-            go: '>=1.23.0-rc.1'
-            gotags: 'cmount'
-            build_flags: '-include "^darwin/amd64" -cgo'
-            quicktest: true
-            racequicktest: true
-            deploy: true
-
-          - job_name: mac_arm64
-            os: macos-latest
-            go: '>=1.23.0-rc.1'
-            gotags: 'cmount'
-            build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
-            deploy: true
-
-          - job_name: windows
-            os: windows-latest
-            go: '>=1.23.0-rc.1'
-            gotags: cmount
-            cgo: '0'
-            build_flags: '-include "^windows/"'
-            build_args: '-buildmode exe'
-            quicktest: true
-            deploy: true
-
-          - job_name: other_os
-            os: ubuntu-latest
-            go: '>=1.23.0-rc.1'
-            build_flags: '-exclude "^(windows/|darwin/|linux/)"'
-            compile_all: true
-            deploy: true
-
-          - job_name: go1.21
-            os: ubuntu-latest
-            go: '1.21'
-            quicktest: true
-            racequicktest: true
-
-          - job_name: go1.22
-            os: ubuntu-latest
-            go: '1.22'
-            quicktest: true
-            racequicktest: true
-
-    name: ${{ matrix.job_name }}
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Install Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: ${{ matrix.go }}
-          check-latest: true
-
-      - name: Set environment variables
-        shell: bash
-        run: |
-          echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
-          echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
-          echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV
-          if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi
-          if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
-
-      - name: Install Libraries on Linux
-        shell: bash
-        run: |
-          sudo modprobe fuse
-          sudo chmod 666 /dev/fuse
-          sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get install fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
-        if: matrix.os == 'ubuntu-latest'
-
-      - name: Install Libraries on macOS
-        shell: bash
-        run: |
-          # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
-          # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
-          unset HOMEBREW_NO_INSTALL_FROM_API
-          brew untap --force homebrew/core
-          brew untap --force homebrew/cask
-          brew update
-          brew install --cask macfuse
-          brew install git-annex git-annex-remote-rclone
-        if: matrix.os == 'macos-latest'
-
-      - name: Install Libraries on Windows
-        shell: powershell
-        run: |
-          $ProgressPreference = 'SilentlyContinue'
-          choco install -y winfsp zip
-          echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
-          if ($env:GOARCH -eq "386") {
-            choco install -y mingw --forcex86 --force
-            echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
-          }
-          # Copy mingw32-make.exe to make.exe so the same command line
-          # can be used on Windows as on macOS and Linux
-          $path = (get-command mingw32-make.exe).Path
-          Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe')
-        if: matrix.os == 'windows-latest'
-
-      - name: Print Go version and environment
-        shell: bash
-        run: |
-          printf "Using go at: $(which go)\n"
-          printf "Go version: $(go version)\n"
-          printf "\n\nGo environment:\n\n"
-          go env
-          printf "\n\nRclone environment:\n\n"
-          make vars
-          printf "\n\nSystem environment:\n\n"
-          env
-
-      - name: Build rclone
-        shell: bash
-        run: |
-          make
-
-      - name: Rclone version
-        shell: bash
-        run: |
-          rclone version
-
-      - name: Run tests
-        shell: bash
-        run: |
-          make quicktest
-        if: matrix.quicktest
-
-      - name: Race test
-        shell: bash
-        run: |
-          make racequicktest
-        if: matrix.racequicktest
-
-      - name: Run librclone tests
-        shell: bash
-        run: |
-          make -C librclone/ctest test
-          make -C librclone/ctest clean
-          librclone/python/test_rclone.py
-        if: matrix.librclonetest
-
-      - name: Compile all architectures test
-        shell: bash
-        run: |
-          make
-          make compile_all
-        if: matrix.compile_all
-
-      - name: Deploy built binaries
-        shell: bash
-        run: |
-          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
-          make ci_beta
-        env:
-          RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
-        # working-directory: '$(modulePath)'
-        # Deploy binaries if enabled in config && not a PR && not a fork
-        if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
-
-  lint:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
-    timeout-minutes: 30
-    name: "lint"
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Get runner parameters
-        id: get-runner-parameters
-        shell: bash
-        run: |
-          echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT
-          echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Install Go
-        id: setup-go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '>=1.23.0-rc.1'
-          check-latest: true
-          cache: false
-
-      - name: Cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/go/pkg/mod
-            ~/.cache/go-build
-            ~/.cache/golangci-lint
-          key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
-          restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
-
-      - name: Code quality test (Linux)
-        uses: golangci/golangci-lint-action@v6
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (Windows)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "windows"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (macOS)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "darwin"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (FreeBSD)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "freebsd"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Code quality test (OpenBSD)
-        uses: golangci/golangci-lint-action@v6
-        env:
-          GOOS: "openbsd"
-        with:
-          version: latest
-          skip-cache: true
-
-      - name: Install govulncheck
-        run: go install golang.org/x/vuln/cmd/govulncheck@latest
-
-      - name: Scan for vulnerabilities
-        run: govulncheck ./...
-
-  android:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
-    timeout-minutes: 30
-    name: "android-all"
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      # Upgrade together with NDK version
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '>=1.23.0-rc.1'
-
-      - name: Set global environment variables
-        shell: bash
-        run: |
-          echo "VERSION=$(make version)" >> $GITHUB_ENV
-
-      - name: build native rclone
-        run: |
-          make
-
-      - name: install gomobile
-        run: |
-          go install golang.org/x/mobile/cmd/gobind@latest
-          go install golang.org/x/mobile/cmd/gomobile@latest
-          env PATH=$PATH:~/go/bin gomobile init
-          echo "RCLONE_NDK_VERSION=21" >> $GITHUB_ENV
-
-      - name: arm-v7a gomobile build
-        run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
-
-      - name: arm-v7a Set environment variables
-        shell: bash
-        run: |
-          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
-          echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-          echo 'GOOS=android' >> $GITHUB_ENV
-          echo 'GOARCH=arm' >> $GITHUB_ENV
-          echo 'GOARM=7' >> $GITHUB_ENV
-          echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-          echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-      - name: arm-v7a build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a .
-
-      - name: arm64-v8a Set environment variables
-        shell: bash
-        run: |
-          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
-          echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-          echo 'GOOS=android' >> $GITHUB_ENV
-          echo 'GOARCH=arm64' >> $GITHUB_ENV
-          echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-          echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-      - name: arm64-v8a build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a .
-
-      - name: x86 Set environment variables
-        shell: bash
-        run: |
-          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
-          echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-          echo 'GOOS=android' >> $GITHUB_ENV
-          echo 'GOARCH=386' >> $GITHUB_ENV
-          echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-          echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-      - name: x86 build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 .
-
-      - name: x64 Set environment variables
-        shell: bash
-        run: |
-          echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
-          echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
-          echo 'GOOS=android' >> $GITHUB_ENV
-          echo 'GOARCH=amd64' >> $GITHUB_ENV
-          echo 'CGO_ENABLED=1' >> $GITHUB_ENV
-          echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV
-
-      - name: x64 build
-        run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x64 .
-
-      - name: Upload artifacts
-        run: |
-          make ci_upload
-        env:
-          RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
-        # Upload artifacts if not a PR && not a fork
-        if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'

.github/workflows/build_publish_beta_docker_image.yml

@@ -1,77 +0,0 @@
-name: Docker beta build
-
-on:
-    push:
-      branches:
-        - master
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKERHUB_USERNAME }}
-                password: ${{ secrets.DOCKERHUB_TOKEN }}
-            - name: Extract metadata (tags, labels) for Docker
-              id: meta
-              uses: docker/metadata-action@v5
-              with:
-                images: ghcr.io/${{ github.repository }}
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to GitHub Container Registry
-              uses: docker/login-action@v3
-              with:
-                registry: ghcr.io
-                # This is the user that triggered the Workflow. In this case, it will
-                # either be the user whom created the Release or manually triggered
-                # the workflow_dispatch.
-                username: ${{ github.actor }}
-                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
-                # GitHub Actions at the start of a workflow run to identify the job.
-                # This is used to authenticate against GitHub Container Registry.
-                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
-                # for more detailed information.
-                password: ${{ secrets.GITHUB_TOKEN }}
-            - name: Show disk usage
-              shell: bash
-              run: |
-                df -h .
-            - name: Build and publish image
-              uses: docker/build-push-action@v6
-              with:
-                file: Dockerfile
-                context: .
-                push: true # push the image to ghcr
-                tags: |
-                  ghcr.io/rclone/rclone:beta
-                  rclone/rclone:beta
-                labels: ${{ steps.meta.outputs.labels }}
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                cache-from: type=gha, scope=${{ github.workflow }}
-                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
-                provenance: false
-              # Eventually cache will need to be cleared if builds more frequent than once a week
-              # https://github.com/docker/build-push-action/issues/252
-            - name: Show disk usage
-              shell: bash
-              run: |
-                df -h .

.github/workflows/build_publish_release_docker_image.yml

@@ -1,89 +0,0 @@
-name: Docker release build
-
-on:
-  release:
-    types: [published]
-
-jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Get actual patch version
-              id: actual_patch_version
-              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
-            - name: Get actual minor version
-              id: actual_minor_version
-              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
-            - name: Get actual major version
-              id: actual_major_version
-              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKER_HUB_USER }}
-                password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-            - name: Build and publish image
-              uses: docker/build-push-action@v6
-              with:
-                file: Dockerfile
-                context: .
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                push: true
-                tags: |
-                  rclone/rclone:latest
-                  rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }}
-                  rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }}
-                  rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
-
-    build_docker_volume_plugin:
-        if: github.repository == 'rclone/rclone'
-        needs: build
-        runs-on: ubuntu-latest
-        name: Build docker plugin job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Build and publish docker plugin
-              shell: bash
-              run: |
-                VER=${GITHUB_REF#refs/tags/}
-                PLUGIN_USER=rclone
-                docker login --username ${{ secrets.DOCKER_HUB_USER }} \
-                             --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
-                for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
-                    export PLUGIN_USER PLUGIN_ARCH
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
-                    make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
-                done
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
-                make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}

.github/workflows/notify.yml

@@ -1,15 +0,0 @@
-name: Notify users based on issue labels
-
-on:
-  issues:
-      types: [labeled]
-
-jobs:
-  notify:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: jenschelkopf/issue-label-notification-action@1.3
-          with:
-             token: ${{ secrets.NOTIFY_ACTION_TOKEN }}
-             recipients: |
-                  Support Contract=@rclone/support

.github/workflows/winget.yml

@@ -1,14 +0,0 @@
-name: Publish to Winget
-on:
-  release:
-    types: [released]
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: vedantmgoyal2009/winget-releaser@v2
-        with:
-          identifier: Rclone.Rclone
-          installers-regex: '-windows-\w+\.zip$'
-          token: ${{ secrets.WINGET_TOKEN }}

.gitignore

@@ -1,22 +1,7 @@
 *~
 _junk/
 rclone
-rclone.exe
 build
-/docs/public/
-/docs/.hugo_build.lock
-/docs/static/img/logos/
+docs/public
 rclone.iml
 .idea
-.history
-.vscode
-*.test
-*.iml
-fuzz-build.zip
-*.orig
-*.rej
-Thumbs.db
-__pycache__
-.DS_Store
-resource_windows_*.syso
-.devcontainer

.golangci.yml

@@ -1,19 +1,20 @@
 # golangci-lint configuration options
 
+run:
+  build-tags:
+    - cmount
+
 linters:
   enable:
+    - deadcode
     - errcheck
     - goimports
-    - revive
+    - golint
     - ineffassign
+    - structcheck
+    - varcheck
     - govet
     - unconvert
-    - staticcheck
-    - gosimple
-    - stylecheck
-    - unused
-    - misspell
-    - gocritic
     #- prealloc
     #- maligned
   disable-all: true
@@ -23,122 +24,7 @@ issues:
   exclude-use-default: false
 
   # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-issues-per-linter: 0
+  max-per-linter: 0
 
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0
-
-  exclude-rules:
-
-    - linters:
-      - staticcheck
-      text: 'SA1019: "github.com/rclone/rclone/cmd/serve/httplib" is deprecated'
-
-  # don't disable the revive messages about comments on exported functions
-  include:
-    - EXC0012
-    - EXC0013
-    - EXC0014
-    - EXC0015
-
-run:
-  # timeout for analysis, e.g. 30s, 5m, default is 1m
-  timeout: 10m
-
-linters-settings:
-  revive:
-    # setting rules seems to disable all the rules, so re-enable them here
-    rules:
-      - name: blank-imports
-        disabled: false
-      - name: context-as-argument
-        disabled: false
-      - name: context-keys-type
-        disabled: false
-      - name: dot-imports
-        disabled: false
-      - name: empty-block
-        disabled: true
-      - name: error-naming
-        disabled: false
-      - name: error-return
-        disabled: false
-      - name: error-strings
-        disabled: false
-      - name: errorf
-        disabled: false
-      - name: exported
-        disabled: false
-      - name: increment-decrement
-        disabled: true
-      - name: indent-error-flow
-        disabled: false
-      - name: package-comments
-        disabled: false
-      - name: range
-        disabled: false
-      - name: receiver-naming
-        disabled: false
-      - name: redefines-builtin-id
-        disabled: true
-      - name: superfluous-else
-        disabled: true
-      - name: time-naming
-        disabled: false
-      - name: unexported-return
-        disabled: false
-      - name: unreachable-code
-        disabled: true
-      - name: unused-parameter
-        disabled: true
-      - name: var-declaration
-        disabled: false
-      - name: var-naming
-        disabled: false
-  stylecheck:
-    # Only enable the checks performed by the staticcheck stand-alone tool,
-    # as documented here: https://staticcheck.io/docs/configuration/options/#checks
-    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]
-  gocritic:
-    # Enable all default checks with some exceptions and some additions (commented).
-    # Cannot use both enabled-checks and disabled-checks, so must specify all to be used.
-    disable-all: true
-    enabled-checks:
-      #- appendAssign # Enabled by default
-      - argOrder
-      - assignOp
-      - badCall
-      - badCond
-      #- captLocal # Enabled by default
-      - caseOrder
-      - codegenComment
-      #- commentFormatting # Enabled by default
-      - defaultCaseOrder
-      - deprecatedComment
-      - dupArg
-      - dupBranchBody
-      - dupCase
-      - dupSubExpr
-      - elseif
-      #- exitAfterDefer # Enabled by default
-      - flagDeref
-      - flagName
-      #- ifElseChain # Enabled by default
-      - mapKey
-      - newDeref
-      - offBy1
-      - regexpMust
-      - ruleguard # Not enabled by default
-      #- singleCaseSwitch # Enabled by default
-      - sloppyLen
-      - sloppyTypeAssert
-      - switchTrue
-      - typeSwitchVar
-      - underef
-      - unlambda
-      - unslice
-      - valSwap
-      - wrapperFunc
-    settings:
-      ruleguard:
-        rules: "${configDir}/bin/rules.go"

.pkgr.yml

@@ -0,0 +1,2 @@
+default_dependencies: false
+cli: rclone

.travis.yml

@@ -0,0 +1,60 @@
+language: go
+sudo: required
+dist: trusty
+os:
+- linux
+go:
+- 1.8.x
+- 1.9.x
+- 1.10.x
+- 1.11.x
+- 1.12.x
+- tip
+go_import_path: github.com/ncw/rclone
+before_install:
+- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
+- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi
+install:
+- git fetch --unshallow --tags
+- make vars
+- make build_dep
+script:
+- make check
+- make quicktest
+- make compile_all
+env:
+  global:
+    - GOTAGS=cmount
+    - GO111MODULE=off
+    - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
+    - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
+    - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
+addons:
+  apt:
+    packages:
+    - fuse
+    - libfuse-dev
+    - rpm
+    - pkg-config
+cache:
+  directories:
+    - $HOME/.cache/go-build
+matrix:
+  allow_failures:
+  - go: tip
+  include:
+  - os: osx
+    go: 1.12.x
+    env: GOTAGS=""
+    cache:
+      directories:
+        - $HOME/Library/Caches/go-build
+deploy:
+  provider: script
+  script: make travis_beta
+  skip_cleanup: true
+  on:
+    repo: ncw/rclone
+    all_branches: true
+    go: 1.12.x
+    condition: $TRAVIS_PULL_REQUEST == false

CONTRIBUTING.md

@@ -1,8 +1,8 @@
-# Contributing to rclone
+# Contributing to rclone #
 
 This is a short guide on how to contribute things to rclone.
 
-## Reporting a bug
+## Reporting a bug ##
 
 If you've just got a question or aren't sure if you've found a bug
 then please use the [rclone forum](https://forum.rclone.org/) instead
@@ -12,172 +12,91 @@ When filing an issue, please include the following information if
 possible as well as a description of the problem.  Make sure you test
 with the [latest beta of rclone](https://beta.rclone.org/):
 
-- Rclone version (e.g. output from `rclone version`)
-- Which OS you are using and how many bits (e.g. Windows 10, 64 bit)
-- The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
-- A log of the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
-    - if the log contains secrets then edit the file with a text editor first to obscure them
+  * Rclone version (eg output from `rclone -V`)
+  * Which OS you are using and how many bits (eg Windows 7, 64 bit)
+  * The command you were trying to run (eg `rclone copy /tmp remote:tmp`)
+  * A log of the command with the `-vv` flag (eg output from `rclone -vv copy /tmp remote:tmp`)
+    * if the log contains secrets then edit the file with a text editor first to obscure them
 
-## Submitting a new feature or bug fix
+## Submitting a pull request ##
 
 If you find a bug that you'd like to fix, or a new feature that you'd
 like to implement then please submit a pull request via GitHub.
 
-If it is a big feature, then [make an issue](https://github.com/rclone/rclone/issues) first so it can be discussed.
+If it is a big feature then make an issue first so it can be discussed.
 
-To prepare your pull request first press the fork button on [rclone's GitHub
-page](https://github.com/rclone/rclone).
+You'll need a Go environment set up with GOPATH set.  See [the Go
+getting started docs](https://golang.org/doc/install) for more info.
 
-Then [install Git](https://git-scm.com/downloads) and set your public contribution [name](https://docs.github.com/en/github/getting-started-with-github/setting-your-username-in-git) and [email](https://docs.github.com/en/github/setting-up-and-managing-your-github-user-account/setting-your-commit-email-address#setting-your-commit-email-address-in-git).
+First in your web browser press the fork button on [rclone's GitHub
+page](https://github.com/ncw/rclone).
 
-Next open your terminal, change directory to your preferred folder and initialise your local rclone project:
+Now in your terminal
 
-    git clone https://github.com/rclone/rclone.git
-    cd rclone
+    go get -u github.com/ncw/rclone
+    cd $GOPATH/src/github.com/ncw/rclone
     git remote rename origin upstream
-      # if you have SSH keys setup in your GitHub account:
     git remote add origin git@github.com:YOURUSER/rclone.git
-      # otherwise:
-    git remote add origin https://github.com/YOURUSER/rclone.git
 
-Note that most of the terminal commands in the rest of this guide must be executed from the rclone folder created above.
-
-Now [install Go](https://golang.org/doc/install) and verify your installation:
-
-    go version
-
-Great, you can now compile and execute your own version of rclone:
-
-    go build
-    ./rclone version
-
-(Note that you can also replace `go build` with `make`, which will include a
-more accurate version number in the executable as well as enable you to specify
-more build options.) Finally make a branch to add your new feature
+Make a branch to add your new feature
 
     git checkout -b my-new-feature
 
 And get hacking.
 
-You may like one of the [popular editors/IDE's for Go](https://github.com/golang/go/wiki/IDEsAndTextEditorPlugins) and a quick view on the rclone [code organisation](#code-organisation).
+When ready - run the unit tests for the code you changed
 
-When ready - test the affected functionality and run the unit tests for the code you changed
-
-    cd folder/with/changed/files
     go test -v
 
-Note that you may need to make a test remote, e.g. `TestSwift` for some
+Note that you may need to make a test remote, eg `TestSwift` for some
 of the unit tests.
 
-This is typically enough if you made a simple bug fix, otherwise please read the rclone [testing](#testing) section too.
+Note the top level Makefile targets
+
+  * make check
+  * make test
+
+Both of these will be run by Travis when you make a pull request but
+you can do this yourself locally too.  These require some extra go
+packages which you can install with
+
+  * make build_dep
 
 Make sure you
 
-- Add [unit tests](#testing) for a new feature.
-- Add [documentation](#writing-documentation) for a new feature.
-- [Commit your changes](#committing-your-changes) using the [commit message guidelines](#commit-messages).
+  * Add [documentation](#writing-documentation) for a new feature.
+  * Follow the [commit message guidelines](#commit-messages).
+  * Add [unit tests](#testing) for a new feature
+  * squash commits down to one per feature
+  * rebase to master with `git rebase master`
 
-When you are done with that push your changes to GitHub:
+When you are done with that
 
-    git push -u origin my-new-feature
+    git push origin my-new-feature
 
-and open the GitHub website to [create your pull
+Go to the GitHub website and click [Create pull
 request](https://help.github.com/articles/creating-a-pull-request/).
 
-Your changes will then get reviewed and you might get asked to fix some stuff. If so, then make the changes in the same branch, commit and push your updates to GitHub.
+You patch will get reviewed and you might get asked to fix some stuff.
 
-You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master) or [squash your commits](#squashing-your-commits).
+If so, then make the changes in the same branch, squash the commits,
+rebase it to master then push it to GitHub with `--force`.
 
-## Using Git and GitHub
+## Enabling CI for your fork ##
 
-### Committing your changes
+The CI config files for rclone have taken care of forks of the project, so you can enable CI for your fork repo easily.
 
-Follow the guideline for [commit messages](#commit-messages) and then:
+rclone currently uses [Travis CI](https://travis-ci.org/), [AppVeyor](https://ci.appveyor.com/), and
+[Circle CI](https://circleci.com/) to build the project. To enable them for your fork, simply go into their
+websites, find your fork of rclone, and enable building there.
 
-    git checkout my-new-feature      # To switch to your branch
-    git status                       # To see the new and changed files
-    git add FILENAME                 # To select FILENAME for the commit
-    git status                       # To verify the changes to be committed
-    git commit                       # To do the commit
-    git log                          # To verify the commit. Use q to quit the log
-
-You can modify the message or changes in the latest commit using:
-
-    git commit --amend
-
-If you amend to commits that have been pushed to GitHub, then you will have to [replace your previously pushed commits](#replacing-your-previously-pushed-commits).
-
-### Replacing your previously pushed commits
-
-Note that you are about to rewrite the GitHub history of your branch. It is good practice to involve your collaborators before modifying commits that have been pushed to GitHub.
-
-Your previously pushed commits are replaced by:
-
-    git push --force origin my-new-feature 
-
-### Basing your changes on the latest master
-
-To base your changes on the latest version of the [rclone master](https://github.com/rclone/rclone/tree/master) (upstream):
-
-    git checkout master
-    git fetch upstream
-    git merge --ff-only
-    git push origin --follow-tags    # optional update of your fork in GitHub
-    git checkout my-new-feature
-    git rebase master
-
-If you rebase commits that have been pushed to GitHub, then you will have to [replace your previously pushed commits](#replacing-your-previously-pushed-commits).
-
-### Squashing your commits ###
-
-To combine your commits into one commit:
-
-    git log                          # To count the commits to squash, e.g. the last 2
-    git reset --soft HEAD~2          # To undo the 2 latest commits
-    git status                       # To check everything is as expected
-
-If everything is fine, then make the new combined commit:
-
-    git commit                       # To commit the undone commits as one
-
-otherwise, you may roll back using:
-
-    git reflog                       # To check that HEAD{1} is your previous state
-    git reset --soft 'HEAD@{1}'      # To roll back to your previous state
-
-If you squash commits that have been pushed to GitHub, then you will have to [replace your previously pushed commits](#replacing-your-previously-pushed-commits).
-
-Tip: You may like to use `git rebase -i master` if you are experienced or have a more complex situation.
-
-### GitHub Continuous Integration
-
-rclone currently uses [GitHub Actions](https://github.com/rclone/rclone/actions) to build and test the project, which should be automatically available for your fork too from the `Actions` tab in your repository.
-
-## Testing
-
-### Code quality tests
-
-If you install [golangci-lint](https://github.com/golangci/golangci-lint) then you can run the same tests as get run in the CI which can be very helpful.
-
-You can run them with `make check` or with `golangci-lint run ./...`.
-
-Using these tests ensures that the rclone codebase all uses the same coding standards. These tests also check for easy mistakes to make (like forgetting to check an error return).
-
-### Quick testing
+## Testing ##
 
 rclone's tests are run from the go testing framework, so at the top
 level you can run this to run all the tests.
 
     go test -v ./...
     
-You can also use `make`, if supported by your platform
-
-    make quicktest
-
-The quicktest is [automatically run by GitHub](#github-continuous-integration) when you push your branch to GitHub.
-
-### Backend testing
-
 rclone contains a mixture of unit tests and integration tests.
 Because it is difficult (and in some respects pointless) to test cloud
 storage systems by mocking all their interfaces, rclone unit tests can
@@ -193,89 +112,80 @@ are skipped if `TestDrive:` isn't defined.
     cd backend/drive
     go test -v
 
-You can then run the integration tests which test all of rclone's
-operations.  Normally these get run against the local file system,
+You can then run the integration tests which tests all of rclone's
+operations.  Normally these get run against the local filing system,
 but they can be run against any of the remotes.
 
     cd fs/sync
     go test -v -remote TestDrive:
-    go test -v -remote TestDrive: -fast-list
+    go test -v -remote TestDrive: -subdir
 
     cd fs/operations
     go test -v -remote TestDrive:
 
 If you want to use the integration test framework to run these tests
-altogether with an HTML report and test retries then from the
+all together with an HTML report and test retries then from the
 project root:
 
-    go install github.com/rclone/rclone/fstest/test_all
-    test_all -backends drive
-
-### Full integration testing
+    go install github.com/ncw/rclone/fstest/test_all
+    test_all -backend drive
 
 If you want to run all the integration tests against all the remotes,
 then change into the project root and run
 
-    make check
     make test
 
-The commands may require some extra go packages which you can install with
-
-    make build_dep
-
-The full integration tests are run daily on the integration test server. You can
+This command is run daily on the the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
-## Code Organisation
+## Code Organisation ##
 
 Rclone code is organised into a small number of top level directories
 with modules beneath.
 
-- backend - the rclone backends for interfacing to cloud providers -
-    - all - import this to load all the cloud providers
-    - ...providers
-- bin - scripts for use while building or maintaining rclone
-- cmd - the rclone commands
-    - all - import this to load all the commands
-    - ...commands
-- cmdtest - end-to-end tests of commands, flags, environment variables,...
-- docs - the documentation and website
-    - content - adjust these docs only - everything else is autogenerated
-    - command - these are auto-generated - edit the corresponding .go file
-- fs - main rclone definitions - minimal amount of code
-    - accounting - bandwidth limiting and statistics
-    - asyncreader - an io.Reader which reads ahead
-    - config - manage the config file and flags
-    - driveletter - detect if a name is a drive letter
-    - filter - implements include/exclude filtering
-    - fserrors - rclone specific error handling
-    - fshttp - http handling for rclone
-    - fspath - path handling for rclone
-    - hash - defines rclone's hash types and functions
-    - list - list a remote
-    - log - logging facilities
-    - march - iterates directories in lock step
-    - object - in memory Fs objects
-    - operations - primitives for sync, e.g. Copy, Move
-    - sync - sync directories
-    - walk - walk a directory
-- fstest - provides integration test framework
-    - fstests - integration tests for the backends
-    - mockdir - mocks an fs.Directory
-    - mockobject - mocks an fs.Object
-    - test_all - Runs integration tests for everything
-- graphics - the images used in the website, etc.
-- lib - libraries used by the backend
-    - atexit - register functions to run when rclone exits
-    - dircache - directory ID to name caching
-    - oauthutil - helpers for using oauth
-    - pacer - retries with backoff and paces operations
-    - readers - a selection of useful io.Readers
-    - rest - a thin abstraction over net/http for REST
-- librclone - in memory interface to rclone's API for embedding rclone
-- vfs - Virtual FileSystem layer for implementing rclone mount and similar
+  * backend - the rclone backends for interfacing to cloud providers - 
+    * all - import this to load all the cloud providers
+    * ...providers
+  * bin - scripts for use while building or maintaining rclone
+  * cmd - the rclone commands
+    * all - import this to load all the commands
+    * ...commands
+  * docs - the documentation and website
+    * content - adjust these docs only - everything else is autogenerated
+  * fs - main rclone definitions - minimal amount of code
+    * accounting - bandwidth limiting and statistics
+    * asyncreader - an io.Reader which reads ahead
+    * config - manage the config file and flags
+    * driveletter - detect if a name is a drive letter
+    * filter - implements include/exclude filtering
+    * fserrors - rclone specific error handling
+    * fshttp - http handling for rclone
+    * fspath - path handling for rclone
+    * hash - defines rclones hash types and functions
+    * list - list a remote
+    * log - logging facilities
+    * march - iterates directories in lock step
+    * object - in memory Fs objects
+    * operations - primitives for sync, eg Copy, Move
+    * sync - sync directories
+    * walk - walk a directory
+  * fstest - provides integration test framework
+    * fstests - integration tests for the backends
+    * mockdir - mocks an fs.Directory
+    * mockobject - mocks an fs.Object
+    * test_all - Runs integration tests for everything
+  * graphics - the images used in the website etc
+  * lib - libraries used by the backend
+    * atexit - register functions to run when rclone exits
+    * dircache - directory ID to name caching
+    * oauthutil - helpers for using oauth
+    * pacer - retries with backoff and paces operations
+    * readers - a selection of useful io.Readers
+    * rest - a thin abstraction over net/http for REST
+  * vendor - 3rd party code managed by `go mod`
+  * vfs - Virtual FileSystem layer for implementing rclone mount and similar
 
-## Writing Documentation
+## Writing Documentation ##
 
 If you are adding a new feature then please update the documentation.
 
@@ -284,49 +194,25 @@ If you add a new general flag (not for a backend), then document it in
 alphabetical order.
 
 If you add a new backend option/flag, then it should be documented in
-the source file in the `Help:` field.
-
-- Start with the most important information about the option,
-    as a single sentence on a single line.
-    - This text will be used for the command-line flag help.
-    - It will be combined with other information, such as any default value,
-      and the result will look odd if not written as a single sentence.
-    - It should end with a period/full stop character, which will be shown
-      in docs but automatically removed when producing the flag help.
-    - Try to keep it below 80 characters, to reduce text wrapping in the terminal.
-- More details can be added in a new paragraph, after an empty line (`"\n\n"`).
-    - Like with docs generated from Markdown, a single line break is ignored
-      and two line breaks creates a new paragraph.
-    - This text will be shown to the user in `rclone config`
-      and in the docs (where it will be added by `make backenddocs`,
-      normally run some time before next release).
-- To create options of enumeration type use the `Examples:` field.
-    - Each example value have their own `Help:` field, but they are treated
-      a bit different than the main option help text. They will be shown
-      as an unordered list, therefore a single line break is enough to
-      create a new list item. Also, for enumeration texts like name of
-      countries, it looks better without an ending period/full stop character.
+the source file in the `Help:` field.  The first line of this is used
+for the flag help, the remainder is shown to the user in `rclone
+config` and is added to the docs with `make backenddocs`.
 
 The only documentation you need to edit are the `docs/content/*.md`
-files.  The `MANUAL.*`, `rclone.1`, website, etc. are all auto-generated
+files.  The MANUAL.*, rclone.1, web site etc are all auto generated
 from those during the release process.  See the `make doc` and `make
 website` targets in the Makefile if you are interested in how.  You
 don't need to run these when adding a feature.
 
-Documentation for rclone sub commands is with their code, e.g.
-`cmd/ls/ls.go`. Write flag help strings as a single sentence on a single
-line, without a period/full stop character at the end, as it will be
-combined unmodified with other information (such as any default value).
+Documentation for rclone sub commands is with their code, eg
+`cmd/ls/ls.go`.
 
-Note that you can use [GitHub's online editor](https://help.github.com/en/github/managing-files-in-a-repository/editing-files-in-another-users-repository)
-for small changes in the docs which makes it very easy.
-
-## Making a release
+## Making a release ##
 
 There are separate instructions for making a release in the RELEASE.md
 file.
 
-## Commit messages
+## Commit messages ##
 
 Please make the first line of your commit message a summary of the
 change that a user (not a developer) of rclone would like to read, and
@@ -359,7 +245,7 @@ And here is an example of a longer one:
 ```
 mount: fix hang on errored upload
 
-In certain circumstances, if an upload failed then the mount could hang
+In certain circumstances if an upload failed then the mount could hang
 indefinitely. This was fixed by closing the read pipe after the Put
 completed.  This will cause the write side to return a pipe closed
 error fixing the hang.
@@ -367,35 +253,51 @@ error fixing the hang.
 Fixes #1498
 ```
 
-## Adding a dependency
+## Adding a dependency ##
 
 rclone uses the [go
 modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
 support in go1.11 and later to manage its dependencies.
 
-rclone can be built with modules outside of the `GOPATH`.
+**NB** you must be using go1.11 or above to add a dependency to
+rclone.  Rclone will still build with older versions of go, but we use
+the `go mod` command for dependencies which is only in go1.11 and
+above.
+
+rclone can be built with modules outside of the GOPATH, but for
+backwards compatibility with older go versions, rclone also maintains
+a `vendor` directory with all the external code rclone needs for
+building.
+
+The `vendor` directory is entirely managed by the `go mod` tool, do
+not add things manually.
 
 To add a dependency `github.com/ncw/new_dependency` see the
-instructions below.  These will fetch the dependency and add it to
-`go.mod` and `go.sum`.
+instructions below.  These will fetch the dependency, add it to
+`go.mod` and `go.sum` and vendor it for older go versions.
 
-    go get github.com/ncw/new_dependency
+    GO111MODULE=on go get github.com/ncw/new_dependency
+    GO111MODULE=on go mod vendor
 
 You can add constraints on that package when doing `go get` (see the
 go docs linked above), but don't unless you really need to.
 
-Please check in the changes generated by `go mod` including `go.mod`
-and `go.sum` in the same commit as your other changes.
+Please check in the changes generated by `go mod` including the
+`vendor` directory and `go.mod` and `go.sum` in a single commit
+separate from any other code changes with the title "vendor: add
+github.com/ncw/new_dependency".  Remember to `git add` any new files
+in `vendor`.
 
-## Updating a dependency
+## Updating a dependency ##
 
 If you need to update a dependency then run
 
-    go get golang.org/x/crypto
+    GO111MODULE=on go get -u github.com/pkg/errors
+    GO111MODULE=on go mod vendor
 
-Check in a single commit as above.
+Check in in a single commit as above.
 
-## Updating all the dependencies
+## Updating all the dependencies ##
 
 In order to update all the dependencies then run `make update`.  This
 just uses the go modules to update all the modules to their latest
@@ -404,7 +306,7 @@ stable release. Check in the changes in a single commit as above.
 This should be done early in the release cycle to pick up new versions
 of packages in time for them to get some testing.
 
-## Updating a backend
+## Updating a backend ##
 
 If you update a backend then please run the unit tests and the
 integration tests for that backend.
@@ -419,156 +321,60 @@ integration tests.
 
 The next section goes into more detail about the tests.
 
-## Writing a new backend
+## Writing a new backend ##
 
 Choose a name.  The docs here will use `remote` as an example.
 
 Note that in rclone terminology a file system backend is called a
 remote or an fs.
 
-### Research
+Research
 
-- Look at the interfaces defined in `fs/types.go`
-- Study one or more of the existing remotes
+  * Look at the interfaces defined in `fs/fs.go`
+  * Study one or more of the existing remotes
 
-### Getting going
+Getting going
 
-- Create `backend/remote/remote.go` (copy this from a similar remote)
-    - box is a good one to start from if you have a directory-based remote (and shows how to use the directory cache)
-    - b2 is a good one to start from if you have a bucket-based remote
-- Add your remote to the imports in `backend/all/all.go`
-- HTTP based remotes are easiest to maintain if they use rclone's [lib/rest](https://pkg.go.dev/github.com/rclone/rclone/lib/rest) module, but if there is a really good Go SDK from the provider then use that instead.
-- Try to implement as many optional methods as possible as it makes the remote more usable.
-- Use [lib/encoder](https://pkg.go.dev/github.com/rclone/rclone/lib/encoder) to make sure we can encode any path name and `rclone info` to help determine the encodings needed
-    - `rclone purge -v TestRemote:rclone-info`
-    - `rclone test info --all --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
-    - `go run cmd/test/info/internal/build_csv/main.go -o remote.csv remote.json`
-    - open `remote.csv` in a spreadsheet and examine
+  * Create `backend/remote/remote.go` (copy this from a similar remote)
+    * box is a good one to start from if you have a directory based remote
+    * b2 is a good one to start from if you have a bucket based remote
+  * Add your remote to the imports in `backend/all/all.go`
+  * HTTP based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead.
+  * Try to implement as many optional methods as possible as it makes the remote more usable.
 
-### Guidelines for a speedy merge
+Unit tests
 
-- **Do** use [lib/rest](https://pkg.go.dev/github.com/rclone/rclone/lib/rest) if you are implementing a REST like backend and parsing XML/JSON in the backend.
-- **Do** use rclone's Client or Transport from [fs/fshttp](https://pkg.go.dev/github.com/rclone/rclone/fs/fshttp) if your backend is HTTP based - this adds features like `--dump bodies`, `--tpslimit`, `--user-agent` without you having to code anything!
-- **Do** follow your example backend exactly - use the same code order, function names, layout, structure. **Don't** move stuff around and **Don't** delete the comments.
-- **Do not** split your backend up into `fs.go` and `object.go` (there are a few backends like that - don't follow them!)
-- **Do** put your API type definitions in a separate file - by preference `api/types.go`
-- **Remember** we have >50 backends to maintain so keeping them as similar as possible to each other is a high priority!
+  * Create a config entry called `TestRemote` for the unit tests to use
+  * Create a `backend/remote/remote_test.go` - copy and adjust your example remote
+  * Make sure all tests pass with `go test -v`
 
-### Unit tests
+Integration tests
 
-- Create a config entry called `TestRemote` for the unit tests to use
-- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
-- Make sure all tests pass with `go test -v`
-
-### Integration tests
-
-- Add your backend to `fstest/test_all/config.yaml`
-    - Once you've done that then you can use the integration test framework from the project root:
-    - go install ./...
-    - test_all -backends remote
+  * Add your backend to `fstest/test_all/config.yaml`
+      * Once you've done that then you can use the integration test framework from the project root:
+      * go install ./...
+      * test_all -backend remote
 
 Or if you want to run the integration tests manually:
 
-- Make sure integration tests pass with
-    - `cd fs/operations`
-    - `go test -v -remote TestRemote:`
-    - `cd fs/sync`
-    - `go test -v -remote TestRemote:`
-- If your remote defines `ListR` check with this also
-    - `go test -v -remote TestRemote: -fast-list`
+  * Make sure integration tests pass with
+      * `cd fs/operations`
+      * `go test -v -remote TestRemote:`
+      * `cd fs/sync`
+      * `go test -v -remote TestRemote:`
+  * If you are making a bucket based remote, then check with this also
+      * `go test -v -remote TestRemote: -subdir`
+  * And if your remote defines `ListR` this also
+      * `go test -v -remote TestRemote: -fast-list`
 
 See the [testing](#testing) section for more information on integration tests.
 
-### Backend documentation
+Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in alphabetical order but with the local file system last.
 
-Add your backend to the docs - you'll need to pick an icon for it from
-[fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in
-alphabetical order of full name of remote (e.g. `drive` is ordered as
-`Google Drive`) but with the local file system last.
-
-- `README.md` - main GitHub page
-- `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
-    - make sure this has the `autogenerated options` comments in (see your reference backend docs)
-    - update them in your backend with `bin/make_backend_docs.py remote`
-- `docs/content/overview.md` - overview docs
-- `docs/content/docs.md` - list of remotes in config section
-- `docs/content/_index.md` - front page of rclone.org
-- `docs/layouts/chrome/navbar.html` - add it to the website navigation
-- `bin/make_manual.py` - add the page to the `docs` constant
-
-Once you've written the docs, run `make serve` and check they look OK
-in the web browser and the links (internal and external) all work.
-
-## Adding a new s3 provider
-
-It is quite easy to add a new S3 provider to rclone.
-
-You'll need to modify the following files
-
-- `backend/s3/s3.go`
-    - Add the provider to `providerOption` at the top of the file
-    - Add endpoints and other config for your provider gated on the provider in `fs.RegInfo`.
-    - Exclude your provider from generic config questions (eg `region` and `endpoint).
-    - Add the provider to the `setQuirks` function - see the documentation there.
-- `docs/content/s3.md`
-    - Add the provider at the top of the page.
-    - Add a section about the provider linked from there.
-    - Add a transcript of a trial `rclone config` session
-        - Edit the transcript to remove things which might change in subsequent versions
-    - **Do not** alter or add to the autogenerated parts of `s3.md`
-    - **Do not** run `make backenddocs` or `bin/make_backend_docs.py s3`
-- `README.md` - this is the home page in github
-    - Add the provider and a link to the section you wrote in `docs/contents/s3.md`
-- `docs/content/_index.md` - this is the home page of rclone.org
-    - Add the provider and a link to the section you wrote in `docs/contents/s3.md`
-
-When adding the provider, endpoints, quirks, docs etc keep them in
-alphabetical order by `Provider` name, but with `AWS` first and
-`Other` last.
-
-Once you've written the docs, run `make serve` and check they look OK
-in the web browser and the links (internal and external) all work.
-
-Once you've written the code, test `rclone config` works to your
-satisfaction, and check the integration tests work `go test -v -remote
-NewS3Provider:`. You may need to adjust the quirks to get them to
-pass. Some providers just can't pass the tests with control characters
-in the names so if these fail and the provider doesn't support
-`urlEncodeListings` in the quirks then ignore them. Note that the
-`SetTier` test may also fail on non AWS providers.
-
-For an example of adding an s3 provider see [eb3082a1](https://github.com/rclone/rclone/commit/eb3082a1ebdb76d5625f14cedec3f5154a5e7b10).
-
-## Writing a plugin
-
-New features (backends, commands) can also be added "out-of-tree", through Go plugins.
-Changes will be kept in a dynamically loaded file instead of being compiled into the main binary.
-This is useful if you can't merge your changes upstream or don't want to maintain a fork of rclone.
-
-### Usage
-
- - Naming
-   - Plugins names must have the pattern `librcloneplugin_KIND_NAME.so`.
-   - `KIND` should be one of `backend`, `command` or `bundle`.
-   - Example: A plugin with backend support for PiFS would be called
-     `librcloneplugin_backend_pifs.so`.
- - Loading
-   - Supported on macOS & Linux as of now. ([Go issue for Windows support](https://github.com/golang/go/issues/19282))
-   - Supported on rclone v1.50 or greater.
-   - All plugins in the folder specified by variable `$RCLONE_PLUGIN_PATH` are loaded.
-   - If this variable doesn't exist, plugin support is disabled.
-   - Plugins must be compiled against the exact version of rclone to work.
-     (The rclone used during building the plugin must be the same as the source of rclone)
-
-### Building
-
-To turn your existing additions into a Go plugin, move them to an external repository
-and change the top-level package name to `main`.
-
-Check `rclone --version` and make sure that the plugin's rclone dependency and host Go version match.
-
-Then, run `go build -buildmode=plugin -o PLUGIN_NAME.so .` to build the plugin.
-
-[Go reference](https://godoc.org/github.com/rclone/rclone/lib/plugin)
-
-[Minimal example](https://gist.github.com/terorie/21b517ee347828e899e1913efc1d684f)
+  * `README.md` - main GitHub page
+  * `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
+  * `docs/content/overview.md` - overview docs
+  * `docs/content/docs.md` - list of remotes in config section
+  * `docs/content/about.md` - front page of rclone.org
+  * `docs/layouts/chrome/navbar.html` - add it to the website navigation
+  * `bin/make_manual.py` - add the page to the `docs` constant

Dockerfile

@@ -1,25 +0,0 @@
-FROM golang:alpine AS builder
-
-COPY . /go/src/github.com/rclone/rclone/
-WORKDIR /go/src/github.com/rclone/rclone/
-
-RUN apk add --no-cache make bash gawk git
-RUN \
-  CGO_ENABLED=0 \
-  make
-RUN ./rclone version
-
-# Begin final image
-FROM alpine:latest
-
-RUN apk --no-cache add ca-certificates fuse3 tzdata && \
-  echo "user_allow_other" >> /etc/fuse.conf
-
-COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
-
-RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
-
-ENTRYPOINT [ "rclone" ]
-
-WORKDIR /data
-ENV XDG_CONFIG_HOME=/config

MAINTAINERS.md

@@ -3,7 +3,7 @@
 Current active maintainers of rclone are:
 
 | Name             | GitHub ID   | Specific Responsibilities    |
-| :--------------- | :---------------- | :--------------------------  |
+| :--------------- | :---------- | :--------------------------  |
 | Nick Craig-Wood  | @ncw        | overall project health       |
 | Stefan Breunig   | @breunigs   |                              |
 | Ishuah Kariuki   | @ishuah     |                              |
@@ -11,22 +11,11 @@ Current active maintainers of rclone are:
 | Fabian Möller    | @B4dM4n     |                              |
 | Alex Chen        | @Cnly       | onedrive backend             |
 | Sandeep Ummadi   | @sandeepkru | azureblob backend            |
-| Sebastian Bünger | @buengese         | jottacloud, yandex & compress backends |
-| Ivan Andreev     | @ivandeex         | chunker & mailru backends    |
-| Max Sum          | @Max-Sum          | union backend                |
-| Fred             | @creativeprojects | seafile backend              |
-| Caleb Case       | @calebcase        | storj backend                |
-| wiserain         | @wiserain         | pikpak backend               |
-| albertony        | @albertony        |                              |
-| Chun-Hung Tseng  | @henrybear327     | Proton Drive Backend         |
-| Hideo Aoyama     | @boukendesho      | snap packaging               |
-| nielash          | @nielash          | bisync                       |
-| Dan McArdle      | @dmcardle         | gitannex                     |
-| Sam Harrison     | @childish-sambino | filescom                     |
+| Sebastian Bünger | @buengese   | jottacloud & yandex backends |
 
 **This is a work in progress Draft**
 
-This is a guide for how to be an rclone maintainer.  This is mostly a write-up of what I (@ncw) attempt to do.
+This is a guide for how to be an rclone maintainer.  This is mostly a writeup of what I (@ncw) attempt to do.
 
 ## Triaging Tickets ##
 
@@ -34,17 +23,17 @@ When a ticket comes in it should be triaged.  This means it should be classified
 
 Rclone uses the labels like this:
 
-* `bug` - a definitely verified bug
+* `bug` - a definite verified bug
 * `can't reproduce` - a problem which we can't reproduce
 * `doc fix` - a bug in the documentation - if users need help understanding the docs add this label
 * `duplicate` - normally close these and ask the user to subscribe to the original
 * `enhancement: new remote` - a new rclone backend
 * `enhancement` - a new feature
-* `FUSE` - to do with `rclone mount` command
-* `good first issue` - mark these if you find a small self-contained issue - these get shown to new visitors to the project
-* `help` wanted - mark these if you find a self-contained issue - these get shown to new visitors to the project
+* `FUSE` - do do with `rclone mount` command
+* `good first issue` - mark these if you find a small self contained issue - these get shown to new visitors to the project
+* `help` wanted - mark these if you find a self contained issue - these get shown to new visitors to the project
 * `IMPORTANT` - note to maintainers not to forget to fix this for the release
-* `maintenance` - internal enhancement, code re-organisation, etc.
+* `maintenance` - internal enhancement, code re-organisation etc
 * `Needs Go 1.XX` - waiting for that version of Go to be released
 * `question` - not a `bug` or `enhancement` - direct to the forum for next time
 * `Remote: XXX` - which rclone backend this affects
@@ -52,17 +41,17 @@ Rclone uses the labels like this:
 
 If it turns out to be a bug or an enhancement it should be tagged as such, with the appropriate other tags.  Don't forget the "good first issue" tag to give new contributors something easy to do to get going.
 
-When a ticket is tagged it should be added to a milestone, either the next release, the one after, Soon or Help Wanted.  Bugs can be added to the "Known Bugs" milestone if they aren't planned to be fixed or need to wait for something (e.g. the next go release).
+When a ticket is tagged it should be added to a milestone, either the next release, the one after, Soon or Help Wanted.  Bugs can be added to the "Known Bugs" milestone if they aren't planned to be fixed or need to wait for something (eg the next go release).
 
 The milestones have these meanings:
 
 * v1.XX - stuff we would like to fit into this release
 * v1.XX+1 - stuff we are leaving until the next release
-* Soon - stuff we think is a good idea - waiting to be scheduled for a release
+* Soon - stuff we think is a good idea - waiting to be scheduled to a release
 * Help wanted - blue sky stuff that might get moved up, or someone could help with
 * Known bugs - bugs waiting on external factors or we aren't going to fix for the moment
 
-Tickets [with no milestone](https://github.com/rclone/rclone/issues?utf8=✓&q=is%3Aissue%20is%3Aopen%20no%3Amile) are good candidates for ones that have slipped between the gaps and need following up.
+Tickets [with no milestone](https://github.com/ncw/rclone/issues?utf8=✓&q=is%3Aissue%20is%3Aopen%20no%3Amile) are good candidates for ones that have slipped between the gaps and need following up.
 
 ## Closing Tickets ##
 
@@ -72,7 +61,7 @@ Close tickets as soon as you can - make sure they are tagged with a release.  Po
 
 Try to process pull requests promptly!
 
-Merging pull requests on GitHub itself works quite well nowadays so you can squash and rebase or rebase pull requests.  rclone doesn't use merge commits.  Use the squash and rebase option if you need to edit the commit message.
+Merging pull requests on GitHub itself works quite well now-a-days so you can squash and rebase or rebase pull requests.  rclone doesn't use merge commits.  Use the squash and rebase option if you need to edit the commit message.
 
 After merging the commit, in your local master branch, do `git pull` then run `bin/update-authors.py` to update the authors file then `git push`.
 
@@ -88,15 +77,15 @@ Rclone aims for a 6-8 week release cycle.  Sometimes release cycles take longer
 
 High impact regressions should be fixed before the next release.
 
-Near the start of the release cycle, the dependencies should be updated with `make update` to give time for bugs to surface.
+Near the start of the release cycle the dependencies should be updated with `make update` to give time for bugs to surface.
 
 Towards the end of the release cycle try not to merge anything too big so let things settle down.
 
-Follow the instructions in RELEASE.md for making the release. Note that the testing part is the most time-consuming often needing several rounds of test and fix depending on exactly how many new features rclone has gained.
+Follow the instructions in RELEASE.md for making the release. Note that the testing part is the most time consuming often needing several rounds of test and fix depending on exactly how many new features rclone has gained.
 
 ## Mailing list ##
 
-There is now an invite-only mailing list for rclone developers `rclone-dev` on google groups.
+There is now an invite only mailing list for rclone developers `rclone-dev` on google groups.
 
 ## TODO ##
 

Makefile

@@ -1,160 +1,122 @@
 SHELL = bash
-# Branch we are working on
-BRANCH := $(or $(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
-# Tag of the current commit, if any.  If this is not "" then we are building a release
-RELEASE_TAG := $(shell git tag -l --points-at HEAD)
-# Version of last release (may not be on this branch)
-VERSION := $(shell cat VERSION)
-# Last tag on this branch
+BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
 LAST_TAG := $(shell git describe --tags --abbrev=0)
-# Next version
-NEXT_VERSION := $(shell echo $(VERSION) | awk -F. -v OFS=. '{print $$1,$$2+1,0}')
-NEXT_PATCH_VERSION := $(shell echo $(VERSION) | awk -F. -v OFS=. '{print $$1,$$2,$$3+1}')
-# If we are working on a release, override branch to master
-ifdef RELEASE_TAG
+ifeq ($(BRANCH),$(LAST_TAG))
 	BRANCH := master
-	LAST_TAG := $(shell git describe --abbrev=0 --tags $(VERSION)^)
 endif
-TAG_BRANCH := .$(BRANCH)
-BRANCH_PATH := branch/$(BRANCH)/
-# If building HEAD or master then unset TAG_BRANCH and BRANCH_PATH
+TAG_BRANCH := -$(BRANCH)
+BRANCH_PATH := branch/
 ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	TAG_BRANCH :=
 	BRANCH_PATH :=
 endif
-# Make version suffix -beta.NNNN.CCCCCCCC (N=Commit number, C=Commit)
-VERSION_SUFFIX := -beta.$(shell git rev-list --count HEAD).$(shell git show --no-patch --no-notes --pretty='%h' HEAD)
-# TAG is current version + commit number + commit + branch
-TAG := $(VERSION)$(VERSION_SUFFIX)$(TAG_BRANCH)
-ifdef RELEASE_TAG
-	TAG := $(RELEASE_TAG)
+TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
+NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
+ifneq ($(TAG),$(LAST_TAG))
+	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
-GO_OS := $(shell go env GOOS)
-ifdef BETA_SUBDIR
-	BETA_SUBDIR := /$(BETA_SUBDIR)
-endif
-BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
+GO_FILES := $(shell go list ./... | grep -v /vendor/ )
+# Run full tests if go >= go1.12
+FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 12)')
+BETA_PATH := $(BRANCH_PATH)$(TAG)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
-BETA_UPLOAD_ROOT := beta.rclone.org:
+BETA_UPLOAD_ROOT := memstore:beta-rclone-org
 BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
-LINTTAGS=--build-tags "$(GOTAGS)"
 endif
-LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
 
-.PHONY: rclone test_all vars version
+.PHONY: rclone vars version
 
 rclone:
-ifeq ($(GO_OS),windows)
-	go run bin/resource_windows.go -version $(TAG) -syso resource_windows_`go env GOARCH`.syso
-endif
-	go build -v $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS)
-ifeq ($(GO_OS),windows)
-	rm resource_windows_`go env GOARCH`.syso
-endif
-	mkdir -p `go env GOPATH`/bin/
-	cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
-	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
-
-test_all:
-	go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
+	touch fs/version.go
+	go install -v --ldflags "-s -X github.com/ncw/rclone/fs.Version=$(TAG)" $(BUILDTAGS)
+	cp -av `go env GOPATH`/bin/rclone .
 
 vars:
 	@echo SHELL="'$(SHELL)'"
 	@echo BRANCH="'$(BRANCH)'"
 	@echo TAG="'$(TAG)'"
-	@echo VERSION="'$(VERSION)'"
+	@echo LAST_TAG="'$(LAST_TAG)'"
+	@echo NEW_TAG="'$(NEW_TAG)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
+	@echo FULL_TESTS="'$(FULL_TESTS)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
-btest:
-	@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
-	@echo "Copied markdown of beta release to clip board"
-
-btesth:
-	@echo "<a href="$(BETA_URL)">$(TAG)</a> on branch <a href="https://github.com/rclone/rclone/tree/$(BRANCH)">$(BRANCH)</a> (uploaded in 15-30 mins)" | xclip -r -sel clip -t text/html
-	@echo "Copied beta release in HTML to clip board"
-
 version:
 	@echo '$(TAG)'
 
 # Full suite of integration tests
-test:	rclone test_all
+test:	rclone
+	go install --ldflags "-s -X github.com/ncw/rclone/fs.Version=$(TAG)" $(BUILDTAGS) github.com/ncw/rclone/fstest/test_all
 	-test_all 2>&1 | tee test_all.log
 	@echo "Written logs in test_all.log"
 
 # Quick test
 quicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
-
-racequicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
-
-compiletest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
+ifdef FULL_TESTS
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
+endif
 
 # Do source code quality checks
 check:	rclone
+ifdef FULL_TESTS
+	@# we still run go vet for -printfuncs which golangci-lint doesn't do yet
+	@# see: https://github.com/golangci/golangci-lint/issues/204
 	@echo "-- START CODE QUALITY REPORT -------------------------------"
-	@golangci-lint run $(LINTTAGS) ./...
+	@go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
+	@golangci-lint run ./...
 	@echo "-- END CODE QUALITY REPORT ---------------------------------"
+else
+	@echo Skipping source quality tests as version of go too old
+endif
 
 # Get the build dependencies
 build_dep:
-	go run bin/get-github-release.go -use-api -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+ifdef FULL_TESTS
+	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+endif
 
-# Get the release dependencies we only install on linux
-release_dep_linux:
-	go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
+# Get the release dependencies
+release_dep:
+	go get -u github.com/goreleaser/nfpm/...
+	go get -u github.com/aktau/github-release
 
 # Update dependencies
-showupdates:
-	@echo "*** Direct dependencies that could be updated ***"
-	@GO111MODULE=on go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
-
-# Update direct dependencies only
-updatedirect:
-	GO111MODULE=on go get -d $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
-	GO111MODULE=on go mod tidy
-
-# Update direct and indirect dependencies and test dependencies
 update:
-	GO111MODULE=on go get -d -u -t ./...
-	GO111MODULE=on go mod tidy
-
-# Tidy the module dependencies
-tidy:
+	GO111MODULE=on go get -u ./...
 	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
 
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
 rclone.1:	MANUAL.md
-	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
+	pandoc -s --from markdown --to man MANUAL.md -o rclone.1
 
-MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs rcdocs
+MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
-	pandoc -s --from markdown-smart --to html MANUAL.md -o MANUAL.html
+	pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html
 
 MANUAL.txt:	MANUAL.md
-	pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
+	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
-	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/
+	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/commands/
 
 backenddocs: rclone bin/make_backend_docs.py
-	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" ./bin/make_backend_docs.py
+	./bin/make_backend_docs.py
 
 rcdocs: rclone
 	bin/make_rc_docs.sh
 
 install: rclone
 	install -d ${DESTDIR}/usr/bin
-	install ${GOPATH}/bin/rclone ${DESTDIR}/usr/bin
+	install -t ${DESTDIR}/usr/bin ${GOPATH}/bin/rclone
 
 clean:
 	go clean ./...
@@ -163,27 +125,14 @@ clean:
 	rm -f rclone fs/operations/operations.test fs/sync/sync.test fs/test_all.log test.log
 
 website:
-	rm -rf docs/public
 	cd docs && hugo
-	@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
 
 upload_website:	website
-	rclone -v sync docs/public www.rclone.org:
-
-upload_test_website:	website
-	rclone -P sync docs/public test-rclone-org:
-
-validate_website: website
-	find docs/public -type f -name "*.html" | xargs tidy --mute-id yes -errors --gnu-emacs yes --drop-empty-elements no --warn-proprietary-attributes no --mute MISMATCHED_ATTRIBUTE_WARN
+	rclone -v sync docs/public memstore:www-rclone-org
 
 tarball:
 	git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
 
-vendorball:
-	go mod vendor
-	tar -zcf build/rclone-$(TAG)-vendor.tar.gz vendor
-	rm -rf vendor
-
 sign_upload:
 	cd build && md5sum rclone-v* | gpg --clearsign > MD5SUMS
 	cd build && sha1sum rclone-v* | gpg --clearsign > SHA1SUMS
@@ -195,109 +144,87 @@ check_sign:
 	cd build && gpg --verify SHA256SUMS && gpg --decrypt SHA256SUMS | sha256sum -c
 
 upload:
-	rclone -P copy build/ downloads.rclone.org:/$(TAG)
-	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
+	rclone -P copy build/ memstore:downloads-rclone-org/$(TAG)
+	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "memstore:downloads-rclone-org/$(TAG)/$$i" "memstore:downloads-rclone-org/$$j"'
 
 upload_github:
 	./bin/upload-github $(TAG)
 
 cross:	doc
-	go run bin/cross-compile.go -release current $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
+	go run bin/cross-compile.go -release current $(BUILDTAGS) $(TAG)
 
 beta:
-	go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-	rclone -v copy build/ pub.rclone.org:/$(TAG)
+	go run bin/cross-compile.go $(BUILDTAGS) $(TAG)
+	rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
 	@echo Beta release ready at https://pub.rclone.org/$(TAG)/
 
 log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-	go run bin/cross-compile.go -compile-only $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-
-ci_upload:
-	sudo chown -R $$USER build
-	find build -type l -delete
-	gzip -r9v build
-	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
-ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
+ifdef FULL_TESTS
+	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
+else
+	@echo Skipping compile all as version of go too old
 endif
-	@echo Beta release ready at $(BETA_URL)/testbuilds
 
-ci_beta:
-	git log $(LAST_TAG).. > /tmp/git-log.txt
-	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
-ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
+appveyor_upload:
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+ifndef BRANCH_PATH
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
-# Fetch the binary builds from GitHub actions
+circleci_upload:
+	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
+ifndef BRANCH_PATH
+	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
+endif
+	@echo Beta release ready at $(BETA_URL)/testbuilds
+
+BUILD_FLAGS := -exclude "^(windows|darwin)/"
+ifeq ($(TRAVIS_OS_NAME),osx)
+	BUILD_FLAGS := -include "^darwin/" -cgo
+endif
+
+travis_beta:
+ifeq ($(TRAVIS_OS_NAME),linux)
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
+endif
+	git log $(LAST_TAG).. > /tmp/git-log.txt
+	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+ifndef BRANCH_PATH
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
+endif
+	@echo Beta release ready at $(BETA_URL)
+
+# Fetch the binary builds from travis and appveyor
 fetch_binaries:
-	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
+	rclone -P sync $(BETA_UPLOAD) build/
 
 serve:	website
-	cd docs && hugo server --logLevel info -w --disableFastRender
+	cd docs && hugo server -v -w
 
-tag:	retag doc
-	bin/make_changelog.py $(LAST_TAG) $(VERSION) > docs/content/changelog.md.new
+tag:	doc
+	@echo "Old tag is $(LAST_TAG)"
+	@echo "New tag is $(NEW_TAG)"
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go
+	echo -n "$(NEW_TAG)" > docs/layouts/partials/version.html
+	git tag -s -m "Version $(NEW_TAG)" $(NEW_TAG)
+	bin/make_changelog.py $(LAST_TAG) $(NEW_TAG) > docs/content/changelog.md.new
 	mv docs/content/changelog.md.new docs/content/changelog.md
 	@echo "Edit the new changelog in docs/content/changelog.md"
 	@echo "Then commit all the changes"
-	@echo git commit -m \"Version $(VERSION)\" -a -v
-	@echo "And finally run make retag before make cross, etc."
+	@echo git commit -m \"Version $(NEW_TAG)\" -a -v
+	@echo "And finally run make retag before make cross etc"
 
 retag:
-	@echo "Version is $(VERSION)"
-	git tag -f -s -m "Version $(VERSION)" $(VERSION)
+	git tag -f -s -m "Version $(LAST_TAG)" $(LAST_TAG)
 
 startdev:
-	@echo "Version is $(VERSION)"
-	@echo "Next version is $(NEXT_VERSION)"
-	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
-	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
-
-startstable:
-	@echo "Version is $(VERSION)"
-	@echo "Next stable version is $(NEXT_PATCH_VERSION)"
-	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
-	echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_PATCH_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(LAST_TAG)-DEV\"\n" | gofmt > fs/version.go
+	git commit -m "Start $(LAST_TAG)-DEV development" fs/version.go
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe
-
-# docker volume plugin
-PLUGIN_USER ?= rclone
-PLUGIN_TAG ?= latest
-PLUGIN_BASE_TAG ?= latest
-PLUGIN_ARCH ?= amd64
-PLUGIN_IMAGE := $(PLUGIN_USER)/docker-volume-rclone:$(PLUGIN_TAG)
-PLUGIN_BASE := $(PLUGIN_USER)/rclone:$(PLUGIN_BASE_TAG)
-PLUGIN_BUILD_DIR := ./build/docker-plugin
-PLUGIN_CONTRIB_DIR := ./contrib/docker-plugin/managed
-
-docker-plugin-create:
-	docker buildx inspect |grep -q /${PLUGIN_ARCH} || \
-	docker run --rm --privileged tonistiigi/binfmt --install all
-	rm -rf ${PLUGIN_BUILD_DIR}
-	docker buildx build \
-		--no-cache --pull \
-		--build-arg BASE_IMAGE=${PLUGIN_BASE} \
-		--platform linux/${PLUGIN_ARCH} \
-		--output ${PLUGIN_BUILD_DIR}/rootfs \
-		${PLUGIN_CONTRIB_DIR}
-	cp ${PLUGIN_CONTRIB_DIR}/config.json ${PLUGIN_BUILD_DIR}
-	docker plugin rm --force ${PLUGIN_IMAGE} 2>/dev/null || true
-	docker plugin create ${PLUGIN_IMAGE} ${PLUGIN_BUILD_DIR}
-
-docker-plugin-push:
-	docker plugin push ${PLUGIN_IMAGE}
-	docker plugin rm ${PLUGIN_IMAGE}
-
-docker-plugin: docker-plugin-create docker-plugin-push

README.md

@@ -1,23 +1,4 @@
-<div align="center">
-<sup>Special thanks to our sponsor:</sup>
-<br>
-<br>
-<a href="https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103">
-  <div>
-    <img src="https://rclone.org/img/logos/warp-github.svg" width="300" alt="Warp">
-  </div>
-  <b>Warp is a modern, Rust-based terminal with AI built in so you and your team can build great software, faster.</b>
-  <div>
-    <sup>Visit warp.dev to learn more.</sup>
-  </div>
-</a>
-<br>
-<hr>
-</div>
-<br>
-
-[<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-light-mode-only)
-[<img src="https://rclone.org/img/logo_on_dark__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-dark-mode-only)
+[![Logo](https://rclone.org/img/rclone-120x120.png)](https://rclone.org/)
 
 [Website](https://rclone.org) |
 [Documentation](https://rclone.org/docs/) |
@@ -25,119 +6,60 @@
 [Contributing](CONTRIBUTING.md) |
 [Changelog](https://rclone.org/changelog/) |
 [Installation](https://rclone.org/install/) |
-[Forum](https://forum.rclone.org/)
+[Forum](https://forum.rclone.org/) | 
+[G+](https://google.com/+RcloneOrg)
 
-[![Build Status](https://github.com/rclone/rclone/workflows/build/badge.svg)](https://github.com/rclone/rclone/actions?query=workflow%3Abuild)
-[![Go Report Card](https://goreportcard.com/badge/github.com/rclone/rclone)](https://goreportcard.com/report/github.com/rclone/rclone)
-[![GoDoc](https://godoc.org/github.com/rclone/rclone?status.svg)](https://godoc.org/github.com/rclone/rclone)
-[![Docker Pulls](https://img.shields.io/docker/pulls/rclone/rclone)](https://hub.docker.com/r/rclone/rclone)
+[![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone)
+[![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/ncw/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/ncw/rclone)
+[![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master)
+[![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) 
 
 # Rclone
 
-Rclone *("rsync for cloud storage")* is a command-line program to sync files and directories to and from different cloud storage providers.
+Rclone *("rsync for cloud storage")* is a command line program to sync files and directories to and from different cloud storage providers.
 
 ## Storage providers
 
-  * 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
-  * Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
+  * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
-  * ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
-  * China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
-  * Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
-  * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
-  * Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
-  * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
-  * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
-  * Files.com [:page_facing_up:](https://rclone.org/filescom/)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
-  * GoFile [:page_facing_up:](https://rclone.org/gofile/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
-  * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
-  * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
-  * Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
-  * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
-  * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
-  * ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
-  * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
+  * Hubic [:page_facing_up:](https://rclone.org/hubic/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
-  * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
-  * Koofr [:page_facing_up:](https://rclone.org/koofr/)
-  * Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
-  * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
-  * Linkbox [:page_facing_up:](https://rclone.org/linkbox)
-  * Linode Object Storage [:page_facing_up:](https://rclone.org/s3/#linode)
-  * Magalu Object Storage [:page_facing_up:](https://rclone.org/s3/#magalu)
-  * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
-  * Memory [:page_facing_up:](https://rclone.org/memory/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
-  * Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
   * Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
   * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
   * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
   * OVH [:page_facing_up:](https://rclone.org/swift/)
-  * Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
   * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
-  * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
-  * Petabox [:page_facing_up:](https://rclone.org/s3/#petabox)
-  * PikPak [:page_facing_up:](https://rclone.org/pikpak/)
-  * Pixeldrain [:page_facing_up:](https://rclone.org/pixeldrain/)
-  * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
-  * put.io [:page_facing_up:](https://rclone.org/putio/)
-  * Proton Drive [:page_facing_up:](https://rclone.org/protondrive/)
+  * put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
-  * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
-  * Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
-  * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
-  * rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
-  * Seafile [:page_facing_up:](https://rclone.org/seafile/)
-  * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
-  * SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
-  * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
-  * Storj [:page_facing_up:](https://rclone.org/storj/)
-  * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
-  * Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
-  * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
-  * Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
-  * Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
   * The local filesystem [:page_facing_up:](https://rclone.org/local/)
   
 Please see [the full list of all storage providers and their features](https://rclone.org/overview/)
 
-### Virtual storage providers
-
-These backends adapt or modify other storage providers
-
-  * Alias: rename existing remotes [:page_facing_up:](https://rclone.org/alias/)
-  * Cache: cache remotes (DEPRECATED) [:page_facing_up:](https://rclone.org/cache/)
-  * Chunker: split large files [:page_facing_up:](https://rclone.org/chunker/)
-  * Combine: combine multiple remotes into a directory tree [:page_facing_up:](https://rclone.org/combine/)
-  * Compress: compress files [:page_facing_up:](https://rclone.org/compress/)
-  * Crypt: encrypt files [:page_facing_up:](https://rclone.org/crypt/)
-  * Hasher: hash files [:page_facing_up:](https://rclone.org/hasher/)
-  * Union: join multiple remotes to work together [:page_facing_up:](https://rclone.org/union/)
-
 ## Features
 
   * MD5/SHA-1 hashes checked at all times for file integrity
@@ -145,15 +67,11 @@ These backends adapt or modify other storage providers
   * Partial syncs supported on a whole file basis
   * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
-  * [Bisync](https://rclone.org/bisync/) (two way) to keep two directories in sync bidirectionally
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
   * Can sync to and from network, e.g. two different cloud accounts
-  * Optional large file chunking ([Chunker](https://rclone.org/chunker/))
-  * Optional transparent compression ([Compress](https://rclone.org/compress/))
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
+  * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
-  * Multi-threaded downloads to local disk
-  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDAV/FTP/SFTP/DLNA
 
 ## Installation & documentation
 
@@ -174,5 +92,5 @@ Please see the [rclone website](https://rclone.org/) for:
 License
 -------
 
-This is free software under the terms of the MIT license (check the
+This is free software under the terms of MIT the license (check the
 [COPYING file](/COPYING) included in this package).

RELEASE.md

@@ -1,175 +1,70 @@
-# Release
-
-This file describes how to make the various kinds of releases
-
-## Extra required software for making a release
-
-  * [gh the github cli](https://github.com/cli/cli) for uploading packages
+Extra required software for making a release
+  * [github-release](https://github.com/aktau/github-release) for uploading packages
   * pandoc for making the html and man pages
 
-## Making a release
-
-  * git checkout master # see below for stable branch
-  * git pull # IMPORTANT
+Making a release
   * git status - make sure everything is checked in
-  * Check GitHub actions build for master is Green
+  * Check travis & appveyor builds are green
+  * make check
   * make test # see integration test server or run locally
   * make tag
-  * edit docs/content/changelog.md # make sure to remove duplicate logs from point releases
-  * make tidy
+  * edit docs/content/changelog.md
   * make doc
   * git status - to check for new man pages - git add them
-  * git commit -a -v -m "Version v1.XX.0"
+  * git commit -a -v -m "Version v1.XX"
   * make retag
-  * git push origin # without --follow-tags so it doesn't push the tag if it fails
-  * git push --follow-tags origin
-  * # Wait for the GitHub builds to complete then...
+  * git push --tags origin master
+  * # Wait for the appveyor and travis builds to complete then...
   * make fetch_binaries
   * make tarball
-  * make vendorball
   * make sign_upload
   * make check_sign
   * make upload
   * make upload_website
   * make upload_github
-  * make startdev # make startstable for stable branch
-  * # announce with forum post, twitter post, patreon post
-
-## Update dependencies
-
-Early in the next release cycle update the dependencies.
+  * make startdev
+  * # announce with forum post, twitter post, G+ post
 
+Early in the next release cycle update the vendored dependencies
   * Review any pinned packages in go.mod and remove if possible
-  * `make updatedirect`
-  * `make GOTAGS=cmount`
-  * `make compiletest`
-  * Fix anything which doesn't compile at this point and commit changes here
-  * `git commit -a -v -m "build: update all dependencies"`
+  * make update
+  * git status
+  * git add new files
+  * git commit -a -v
 
-If the `make updatedirect` upgrades the version of go in the `go.mod`
-then go to manual mode. `go1.20` here is the lowest supported version
-in the `go.mod`.
+If `make update` fails with errors like this:
 
 ```
-go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
-go get -d $(cat /tmp/potential-upgrades)
-go mod tidy -go=1.20 -compat=1.20
+# github.com/cpuguy83/go-md2man/md2man
+../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:11:16: undefined: blackfriday.EXTENSION_NO_INTRA_EMPHASIS
+../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:12:16: undefined: blackfriday.EXTENSION_TABLES
 ```
 
-If the `go mod tidy` fails use the output from it to remove the
-package which can't be upgraded from `/tmp/potential-upgrades` when
-done
+Can be fixed with
 
-```
-git co go.mod go.sum
-```
+    * GO111MODULE=on go get -u github.com/russross/blackfriday@v1.5.2
+    * GO111MODULE=on go mod tidy
+    * GO111MODULE=on go mod vendor
  
-And try again.
 
-Optionally upgrade the direct and indirect dependencies. This is very
-likely to fail if the manual method was used abve - in that case
-ignore it as it is too time consuming to fix.
-
-  * `make update`
-  * `make GOTAGS=cmount`
-  * `make compiletest`
-  * roll back any updates which didn't compile
-  * `git commit -a -v --amend`
-  * **NB** watch out for this changing the default go version in `go.mod`
-
-Note that `make update` updates all direct and indirect dependencies
-and there can occasionally be forwards compatibility problems with
-doing that so it may be necessary to roll back dependencies to the
-version specified by `make updatedirect` in order to get rclone to
-build.
-
-Once it compiles locally, push it on a test branch and commit fixes
-until the tests pass.
-
-## Tidy beta
-
-At some point after the release run
-
-    bin/tidy-beta v1.55
-
-where the version number is that of a couple ago to remove old beta binaries.
-
-## Making a point release
-
-If rclone needs a point release due to some horrendous bug:
-
-Set vars
-
-  * BASE_TAG=v1.XX          # e.g. v1.52
-  * NEW_TAG=${BASE_TAG}.Y   # e.g. v1.52.1
-  * echo $BASE_TAG $NEW_TAG # v1.52 v1.52.1
-
-First make the release branch.  If this is a second point release then
-this will be done already.
-
-  * git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
-  * make startstable
-
-Now
-
-  * git co ${BASE_TAG}-stable
+Making a point release.  If rclone needs a point release due to some
+horrendous bug, then
+  * git branch v1.XX v1.XX-fixes
   * git cherry-pick any fixes
-  * Do the steps as above
-  * make startstable
-  * git co master
-  * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
-  * git checkout ${BASE_TAG}-stable docs/content/changelog.md
-  * git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
-  * git push
-
-## Sponsor logos
-
-If updating the website note that the sponsor logos have been moved out of the main repository.
-
-You will need to checkout `/docs/static/img/logos` from https://github.com/rclone/third-party-logos
-which is a private repo containing artwork from sponsors.
-
-## Update the website between releases
-
-Create an update website branch based off the last release
-
-    git co -b update-website
-
-If the branch already exists, double check there are no commits that need saving.
-
-Now reset the branch to the last release
-
-    git reset --hard v1.64.0
-
-Create the changes, check them in, test with `make serve` then
-
-    make upload_test_website
-
-Check out https://test.rclone.org and when happy
-
-    make upload_website
-
-Cherry pick any changes back to master and the stable branch if it is active.
-
-## Making a manual build of docker
-
-To do a basic build of rclone's docker image to debug builds locally:
-
-```
-docker buildx build --load -t rclone/rclone:testing --progress=plain .
-docker run --rm rclone/rclone:testing version
-```
-
-To test the multipatform build
-
-```
-docker buildx build -t rclone/rclone:testing --progress=plain --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 .
-```
-
-To make a full build then set the tags correctly and add `--push`
-
-Note that you can't only build one architecture - you need to build them all.
-
-```
-docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
-```
+  * Test (see above)
+  * make NEW_TAG=v1.XX.1 tag
+  * edit docs/content/changelog.md
+  * make TAG=v1.43.1 doc
+  * git commit -a -v -m "Version v1.XX.1"
+  * git tag -d -v1.XX.1
+  * git tag -s -m "Version v1.XX.1" v1.XX.1
+  * git push --tags -u origin v1.XX-fixes
+  * make BRANCH_PATH= TAG=v1.43.1 fetch_binaries
+  * make TAG=v1.43.1 tarball
+  * make TAG=v1.43.1 sign_upload
+  * make TAG=v1.43.1 check_sign
+  * make TAG=v1.43.1 upload
+  * make TAG=v1.43.1 upload_website
+  * make TAG=v1.43.1 upload_github
+  * NB this overwrites the current beta so after the release, rebuild the last travis build
+  * Announce!

VERSION

@@ -1 +0,0 @@
-v1.68.2

backend/alias/alias.go

@@ -1,27 +1,24 @@
-// Package alias implements a virtual provider to rename existing remotes.
 package alias
 
 import (
-	"context"
 	"errors"
 	"strings"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/cache"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fspath"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
+	"github.com/ncw/rclone/fs/config/configstruct"
+	"github.com/ncw/rclone/fs/fspath"
 )
 
 // Register with Fs
 func init() {
 	fsi := &fs.RegInfo{
 		Name:        "alias",
-		Description: "Alias for an existing remote",
+		Description: "Alias for a existing remote",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "remote",
-			Help:     "Remote or path to alias.\n\nCan be \"myremote:path/to/dir\", \"myremote:bucket\", \"myremote:\" or \"/local/path\".",
+			Help:     "Remote or path to alias.\nCan be \"myremote:path/to/dir\", \"myremote:bucket\", \"myremote:\" or \"/local/path\".",
 			Required: true,
 		}},
 	}
@@ -36,7 +33,7 @@ type Options struct {
 // NewFs constructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -49,5 +46,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if strings.HasPrefix(opt.Remote, name+":") {
 		return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
 	}
-	return cache.Get(ctx, fspath.JoinRootPath(opt.Remote, root))
+	fsInfo, configName, fsPath, config, err := fs.ConfigFs(opt.Remote)
+	if err != nil {
+		return nil, err
+	}
+	return fsInfo.NewFs(configName, fspath.JoinRootPath(fsPath, root), config)
 }

backend/alias/alias_internal_test.go

@@ -1,17 +1,15 @@
 package alias
 
 import (
-	"context"
 	"fmt"
 	"path"
 	"path/filepath"
 	"sort"
 	"testing"
 
-	_ "github.com/rclone/rclone/backend/local" // pull in test backend
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configfile"
+	_ "github.com/ncw/rclone/backend/local" // pull in test backend
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config"
 	"github.com/stretchr/testify/require"
 )
 
@@ -20,11 +18,11 @@ var (
 )
 
 func prepare(t *testing.T, root string) {
-	configfile.Install()
+	config.LoadConfig()
 
 	// Configure the remote
-	config.FileSetValue(remoteName, "type", "alias")
-	config.FileSetValue(remoteName, "remote", root)
+	config.FileSet(remoteName, "type", "alias")
+	config.FileSet(remoteName, "remote", root)
 }
 
 func TestNewFS(t *testing.T) {
@@ -55,24 +53,23 @@ func TestNewFS(t *testing.T) {
 			{"four/under four.txt", 9, false},
 		}},
 		{"four", "..", "", true, []testEntry{
-			{"five", -1, true},
-			{"under four.txt", 9, false},
+			{"four", -1, true},
+			{"one%.txt", 6, false},
+			{"three", -1, true},
+			{"two.html", 7, false},
 		}},
-		{"", "../../three", "", true, []testEntry{
+		{"four", "../three", "", true, []testEntry{
 			{"underthree.txt", 9, false},
 		}},
-		{"four", "../../five", "", true, []testEntry{
-			{"underfive.txt", 6, false},
-		}},
 	} {
 		what := fmt.Sprintf("test %d remoteRoot=%q, fsRoot=%q, fsList=%q", testi, test.remoteRoot, test.fsRoot, test.fsList)
 
 		remoteRoot, err := filepath.Abs(filepath.FromSlash(path.Join("test/files", test.remoteRoot)))
 		require.NoError(t, err, what)
 		prepare(t, remoteRoot)
-		f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:%s", remoteName, test.fsRoot))
+		f, err := fs.NewFs(fmt.Sprintf("%s:%s", remoteName, test.fsRoot))
 		require.NoError(t, err, what)
-		gotEntries, err := f.List(context.Background(), test.fsList)
+		gotEntries, err := f.List(test.fsList)
 		require.NoError(t, err, what)
 
 		sort.Sort(gotEntries)
@@ -81,12 +78,10 @@ func TestNewFS(t *testing.T) {
 		for i, gotEntry := range gotEntries {
 			what := fmt.Sprintf("%s, entry=%d", what, i)
 			wantEntry := test.entries[i]
-			_, isDir := gotEntry.(fs.Directory)
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			if !isDir {
 			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
-			}
+			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}
 	}
@@ -94,7 +89,7 @@ func TestNewFS(t *testing.T) {
 
 func TestNewFSNoRemote(t *testing.T) {
 	prepare(t, "")
-	f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:", remoteName))
+	f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName))
 
 	require.Error(t, err)
 	require.Nil(t, f)
@@ -102,7 +97,7 @@ func TestNewFSNoRemote(t *testing.T) {
 
 func TestNewFSInvalidRemote(t *testing.T) {
 	prepare(t, "not_existing_test_remote:")
-	f, err := fs.NewFs(context.Background(), fmt.Sprintf("%s:", remoteName))
+	f, err := fs.NewFs(fmt.Sprintf("%s:", remoteName))
 
 	require.Error(t, err)
 	require.Nil(t, f)

backend/all/all.go

@@ -1,66 +1,31 @@
-// Package all imports all the backends
 package all
 
 import (
 	// Active file systems
-	_ "github.com/rclone/rclone/backend/alias"
-	_ "github.com/rclone/rclone/backend/azureblob"
-	_ "github.com/rclone/rclone/backend/azurefiles"
-	_ "github.com/rclone/rclone/backend/b2"
-	_ "github.com/rclone/rclone/backend/box"
-	_ "github.com/rclone/rclone/backend/cache"
-	_ "github.com/rclone/rclone/backend/chunker"
-	_ "github.com/rclone/rclone/backend/combine"
-	_ "github.com/rclone/rclone/backend/compress"
-	_ "github.com/rclone/rclone/backend/crypt"
-	_ "github.com/rclone/rclone/backend/drive"
-	_ "github.com/rclone/rclone/backend/dropbox"
-	_ "github.com/rclone/rclone/backend/fichier"
-	_ "github.com/rclone/rclone/backend/filefabric"
-	_ "github.com/rclone/rclone/backend/filescom"
-	_ "github.com/rclone/rclone/backend/frostfs"
-	_ "github.com/rclone/rclone/backend/ftp"
-	_ "github.com/rclone/rclone/backend/gofile"
-	_ "github.com/rclone/rclone/backend/googlecloudstorage"
-	_ "github.com/rclone/rclone/backend/googlephotos"
-	_ "github.com/rclone/rclone/backend/hasher"
-	_ "github.com/rclone/rclone/backend/hdfs"
-	_ "github.com/rclone/rclone/backend/hidrive"
-	_ "github.com/rclone/rclone/backend/http"
-	_ "github.com/rclone/rclone/backend/imagekit"
-	_ "github.com/rclone/rclone/backend/internetarchive"
-	_ "github.com/rclone/rclone/backend/jottacloud"
-	_ "github.com/rclone/rclone/backend/koofr"
-	_ "github.com/rclone/rclone/backend/linkbox"
-	_ "github.com/rclone/rclone/backend/local"
-	_ "github.com/rclone/rclone/backend/mailru"
-	_ "github.com/rclone/rclone/backend/mega"
-	_ "github.com/rclone/rclone/backend/memory"
-	_ "github.com/rclone/rclone/backend/netstorage"
-	_ "github.com/rclone/rclone/backend/onedrive"
-	_ "github.com/rclone/rclone/backend/opendrive"
-	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
-	_ "github.com/rclone/rclone/backend/pcloud"
-	_ "github.com/rclone/rclone/backend/pikpak"
-	_ "github.com/rclone/rclone/backend/pixeldrain"
-	_ "github.com/rclone/rclone/backend/premiumizeme"
-	_ "github.com/rclone/rclone/backend/protondrive"
-	_ "github.com/rclone/rclone/backend/putio"
-	_ "github.com/rclone/rclone/backend/qingstor"
-	_ "github.com/rclone/rclone/backend/quatrix"
-	_ "github.com/rclone/rclone/backend/s3"
-	_ "github.com/rclone/rclone/backend/seafile"
-	_ "github.com/rclone/rclone/backend/sftp"
-	_ "github.com/rclone/rclone/backend/sharefile"
-	_ "github.com/rclone/rclone/backend/sia"
-	_ "github.com/rclone/rclone/backend/smb"
-	_ "github.com/rclone/rclone/backend/storj"
-	_ "github.com/rclone/rclone/backend/sugarsync"
-	_ "github.com/rclone/rclone/backend/swift"
-	_ "github.com/rclone/rclone/backend/ulozto"
-	_ "github.com/rclone/rclone/backend/union"
-	_ "github.com/rclone/rclone/backend/uptobox"
-	_ "github.com/rclone/rclone/backend/webdav"
-	_ "github.com/rclone/rclone/backend/yandex"
-	_ "github.com/rclone/rclone/backend/zoho"
+	_ "github.com/ncw/rclone/backend/alias"
+	_ "github.com/ncw/rclone/backend/amazonclouddrive"
+	_ "github.com/ncw/rclone/backend/azureblob"
+	_ "github.com/ncw/rclone/backend/b2"
+	_ "github.com/ncw/rclone/backend/box"
+	_ "github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/crypt"
+	_ "github.com/ncw/rclone/backend/drive"
+	_ "github.com/ncw/rclone/backend/dropbox"
+	_ "github.com/ncw/rclone/backend/ftp"
+	_ "github.com/ncw/rclone/backend/googlecloudstorage"
+	_ "github.com/ncw/rclone/backend/http"
+	_ "github.com/ncw/rclone/backend/hubic"
+	_ "github.com/ncw/rclone/backend/jottacloud"
+	_ "github.com/ncw/rclone/backend/local"
+	_ "github.com/ncw/rclone/backend/mega"
+	_ "github.com/ncw/rclone/backend/onedrive"
+	_ "github.com/ncw/rclone/backend/opendrive"
+	_ "github.com/ncw/rclone/backend/pcloud"
+	_ "github.com/ncw/rclone/backend/qingstor"
+	_ "github.com/ncw/rclone/backend/s3"
+	_ "github.com/ncw/rclone/backend/sftp"
+	_ "github.com/ncw/rclone/backend/swift"
+	_ "github.com/ncw/rclone/backend/union"
+	_ "github.com/ncw/rclone/backend/webdav"
+	_ "github.com/ncw/rclone/backend/yandex"
 )

backend/amazonclouddrive/amazonclouddrive_test.go

@@ -0,0 +1,20 @@
+// Test AmazonCloudDrive filesystem interface
+
+// +build acd
+
+package amazonclouddrive_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/backend/amazonclouddrive"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.NilObject = fs.Object((*amazonclouddrive.Object)(nil))
+	fstests.RemoteName = "TestAmazonCloudDrive:"
+	fstests.Run(t)
+}

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,go1.8
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,16 +1,14 @@
 // Test AzureBlob filesystem interface
 
-//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,go1.8
 
 package azureblob
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/stretchr/testify/assert"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
@@ -18,28 +16,9 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:  "TestAzureBlob:",
 		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool", "Cold"},
+		TiersToTest: []string{"Hot", "Cool"},
 		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: defaultChunkSize,
-		},
-	})
-}
-
-// TestIntegration2 runs integration tests against the remote
-func TestIntegration2(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	name := "TestAzureBlob"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:  name + ":",
-		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool", "Cold"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: defaultChunkSize,
-		},
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "directory_markers", Value: "true"},
+			MaxChunkSize: maxChunkSize,
 		},
 	})
 }
@@ -48,29 +27,11 @@ func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadChunkSize(cs)
 }
 
+func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadCutoff(cs)
+}
+
 var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
 )
-
-func TestValidateAccessTier(t *testing.T) {
-	tests := map[string]struct {
-		accessTier string
-		want       bool
-	}{
-		"hot":     {"hot", true},
-		"HOT":     {"HOT", true},
-		"Hot":     {"Hot", true},
-		"cool":    {"cool", true},
-		"cold":    {"cold", true},
-		"archive": {"archive", true},
-		"empty":   {"", false},
-		"unknown": {"unknown", false},
-	}
-
-	for name, test := range tests {
-		t.Run(name, func(t *testing.T) {
-			got := validateAccessTier(test.accessTier)
-			assert.Equal(t, test.want, got)
-		})
-	}
-}

backend/azureblob/azureblob_unsupported.go

@@ -1,7 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || solaris || js
+// +build plan9 solaris !go1.8
 
-// Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob

backend/azurefiles/azurefiles_internal_test.go

@@ -1,69 +0,0 @@
-//go:build !plan9 && !js
-
-package azurefiles
-
-import (
-	"context"
-	"math/rand"
-	"strings"
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/stretchr/testify/assert"
-)
-
-func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("Authentication", f.InternalTestAuth)
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)
-
-func (f *Fs) InternalTestAuth(t *testing.T) {
-	t.Skip("skipping since this requires authentication credentials which are not part of repo")
-	shareName := "test-rclone-oct-2023"
-	testCases := []struct {
-		name    string
-		options *Options
-	}{
-		{
-			name: "ConnectionString",
-			options: &Options{
-				ShareName:        shareName,
-				ConnectionString: "",
-			},
-		},
-		{
-			name: "AccountAndKey",
-			options: &Options{
-				ShareName: shareName,
-				Account:   "",
-				Key:       "",
-			}},
-		{
-			name: "SASUrl",
-			options: &Options{
-				ShareName: shareName,
-				SASURL:    "",
-			}},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			fs, err := newFsFromOptions(context.TODO(), "TestAzureFiles", "", tc.options)
-			assert.NoError(t, err)
-			dirName := randomString(10)
-			assert.NoError(t, fs.Mkdir(context.TODO(), dirName))
-		})
-	}
-}
-
-const chars = "abcdefghijklmnopqrstuvwzyxABCDEFGHIJKLMNOPQRSTUVWZYX"
-
-func randomString(charCount int) string {
-	strBldr := strings.Builder{}
-	for i := 0; i < charCount; i++ {
-		randPos := rand.Int63n(52)
-		strBldr.WriteByte(chars[randPos])
-	}
-	return strBldr.String()
-}

backend/azurefiles/azurefiles_test.go

@@ -1,17 +0,0 @@
-//go:build !plan9 && !js
-
-package azurefiles
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-func TestIntegration(t *testing.T) {
-	var objPtr *Object
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestAzureFiles:",
-		NilObject:  objPtr,
-	})
-}

backend/azurefiles/azurefiles_unsupported.go

@@ -1,7 +0,0 @@
-// Build for azurefiles for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-//go:build plan9 || js
-
-// Package azurefiles provides an interface to Microsoft Azure Files
-package azurefiles

backend/b2/api/types.go

@@ -1,13 +1,13 @@
-// Package api provides types used by the Backblaze B2 API.
 package api
 
 import (
 	"fmt"
+	"path"
 	"strconv"
+	"strings"
 	"time"
 
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/version"
+	"github.com/ncw/rclone/fs/fserrors"
 )
 
 // Error describes a B2 error response
@@ -37,14 +37,6 @@ type Bucket struct {
 	AccountID string `json:"accountId"`
 	Name      string `json:"bucketName"`
 	Type      string `json:"bucketType"`
-	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
-}
-
-// LifecycleRule is a single lifecycle rule
-type LifecycleRule struct {
-	DaysFromHidingToDeleting  *int   `json:"daysFromHidingToDeleting"`
-	DaysFromUploadingToHiding *int   `json:"daysFromUploadingToHiding"`
-	FileNamePrefix            string `json:"fileNamePrefix"`
 }
 
 // Timestamp is a UTC time when this file was uploaded. It is a base
@@ -58,7 +50,7 @@ type Timestamp time.Time
 // MarshalJSON turns a Timestamp into JSON (in UTC)
 func (t *Timestamp) MarshalJSON() (out []byte, err error) {
 	timestamp := (*time.Time)(t).UTC().UnixNano()
-	return []byte(strconv.FormatInt(timestamp/1e6, 10)), nil
+	return []byte(strconv.FormatInt(timestamp/1E6, 10)), nil
 }
 
 // UnmarshalJSON turns JSON into a Timestamp
@@ -67,21 +59,20 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
 	if err != nil {
 		return err
 	}
-	*t = Timestamp(time.Unix(timestamp/1e3, (timestamp%1e3)*1e6).UTC())
+	*t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6).UTC())
 	return nil
 }
 
-// HasVersion returns true if it looks like the passed filename has a timestamp on it.
-//
-// Note that the passed filename's timestamp may still be invalid even if this
-// function returns true.
-func HasVersion(remote string) bool {
-	return version.Match(remote)
-}
+const versionFormat = "-v2006-01-02-150405.000"
 
 // AddVersion adds the timestamp as a version string into the filename passed in.
 func (t Timestamp) AddVersion(remote string) string {
-	return version.Add(remote, time.Time(t))
+	ext := path.Ext(remote)
+	base := remote[:len(remote)-len(ext)]
+	s := time.Time(t).Format(versionFormat)
+	// Replace the '.' with a '-'
+	s = strings.Replace(s, ".", "-", -1)
+	return base + s + ext
 }
 
 // RemoveVersion removes the timestamp from a filename as a version string.
@@ -89,10 +80,25 @@ func (t Timestamp) AddVersion(remote string) string {
 // It returns the new file name and a timestamp, or the old filename
 // and a zero timestamp.
 func RemoveVersion(remote string) (t Timestamp, newRemote string) {
-	time, newRemote := version.Remove(remote)
-	t = Timestamp(time)
+	newRemote = remote
+	ext := path.Ext(remote)
+	base := remote[:len(remote)-len(ext)]
+	if len(base) < len(versionFormat) {
 		return
 	}
+	versionStart := len(base) - len(versionFormat)
+	// Check it ends in -xxx
+	if base[len(base)-4] != '-' {
+		return
+	}
+	// Replace with .xxx for parsing
+	base = base[:len(base)-4] + "." + base[len(base)-3:]
+	newT, err := time.Parse(versionFormat, base[versionStart:])
+	if err != nil {
+		return
+	}
+	return Timestamp(newT), base[:versionStart] + ext
+}
 
 // IsZero returns true if the timestamp is uninitialized
 func (t Timestamp) IsZero() bool {
@@ -183,21 +189,6 @@ type GetUploadURLResponse struct {
 	AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when uploading files to this bucket, see b2_upload_file.
 }
 
-// GetDownloadAuthorizationRequest is passed to b2_get_download_authorization
-type GetDownloadAuthorizationRequest struct {
-	BucketID               string `json:"bucketId"`                       // The ID of the bucket that you want to upload to.
-	FileNamePrefix         string `json:"fileNamePrefix"`                 // The file name prefix of files the download authorization token will allow access to.
-	ValidDurationInSeconds int64  `json:"validDurationInSeconds"`         // The number of seconds before the authorization token will expire. The minimum value is 1 second. The maximum value is 604800 which is one week in seconds.
-	B2ContentDisposition   string `json:"b2ContentDisposition,omitempty"` // optional - If this is present, download requests using the returned authorization must include the same value for b2ContentDisposition.
-}
-
-// GetDownloadAuthorizationResponse is received from b2_get_download_authorization
-type GetDownloadAuthorizationResponse struct {
-	BucketID           string `json:"bucketId"`           // The unique ID of the bucket.
-	FileNamePrefix     string `json:"fileNamePrefix"`     // The file name prefix of files the download authorization token will allow access to.
-	AuthorizationToken string `json:"authorizationToken"` // The authorizationToken that must be used when downloading files, see b2_download_file_by_name.
-}
-
 // FileInfo is received from b2_upload_file, b2_get_file_info and b2_finish_large_file
 type FileInfo struct {
 	ID              string            `json:"fileId"`          // The unique identifier for this version of this file. Used with b2_get_file_info, b2_download_file_by_id, and b2_delete_file_version.
@@ -217,7 +208,6 @@ type CreateBucketRequest struct {
 	AccountID string `json:"accountId"`
 	Name      string `json:"bucketName"`
 	Type      string `json:"bucketType"`
-	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
 }
 
 // DeleteBucketRequest is used to create a bucket
@@ -248,7 +238,7 @@ type GetFileInfoRequest struct {
 // If the original source of the file being uploaded has a last
 // modified time concept, Backblaze recommends using
 // src_last_modified_millis as the name, and a string holding the base
-// 10 number of milliseconds since midnight, January 1, 1970
+// 10 number number of milliseconds since midnight, January 1, 1970
 // UTC. This fits in a 64 bit integer such as the type "long" in the
 // programming language Java. It is intended to be compatible with
 // Java's time long. For example, it can be passed directly into the
@@ -321,30 +311,3 @@ type CancelLargeFileResponse struct {
 	AccountID string `json:"accountId"` // The identifier for the account.
 	BucketID  string `json:"bucketId"`  // The unique ID of the bucket.
 }
-
-// CopyFileRequest is as passed to b2_copy_file
-type CopyFileRequest struct {
-	SourceID          string            `json:"sourceFileId"`                  // The ID of the source file being copied.
-	Name              string            `json:"fileName"`                      // The name of the new file being created.
-	Range             string            `json:"range,omitempty"`               // The range of bytes to copy. If not provided, the whole source file will be copied.
-	MetadataDirective string            `json:"metadataDirective,omitempty"`   // The strategy for how to populate metadata for the new file: COPY or REPLACE
-	ContentType       string            `json:"contentType,omitempty"`         // The MIME type of the content of the file (REPLACE only)
-	Info              map[string]string `json:"fileInfo,omitempty"`            // This field stores the metadata that will be stored with the file. (REPLACE only)
-	DestBucketID      string            `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
-}
-
-// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponse
-type CopyPartRequest struct {
-	SourceID    string `json:"sourceFileId"`    // The ID of the source file being copied.
-	LargeFileID string `json:"largeFileId"`     // The ID of the large file the part will belong to, as returned by b2_start_large_file.
-	PartNumber  int64  `json:"partNumber"`      // Which part this is (starting from 1)
-	Range       string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
-}
-
-// UpdateBucketRequest describes a request to modify a B2 bucket
-type UpdateBucketRequest struct {
-	ID             string          `json:"bucketId"`
-	AccountID      string          `json:"accountId"`
-	Type           string          `json:"bucketType,omitempty"`
-	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
-}

backend/b2/api/types_test.go

@@ -4,8 +4,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/backend/b2/api"
-	"github.com/rclone/rclone/fstest"
+	"github.com/ncw/rclone/backend/b2/api"
+	"github.com/ncw/rclone/fstest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -13,6 +13,7 @@ import (
 var (
 	emptyT api.Timestamp
 	t0     = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123456789Z"))
+	t0r    = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123000000Z"))
 	t1     = api.Timestamp(fstest.Time("2001-02-03T04:05:06.123000000Z"))
 )
 
@@ -35,6 +36,40 @@ func TestTimestampUnmarshalJSON(t *testing.T) {
 	assert.Equal(t, (time.Time)(t1), (time.Time)(tActual))
 }
 
+func TestTimestampAddVersion(t *testing.T) {
+	for _, test := range []struct {
+		t        api.Timestamp
+		in       string
+		expected string
+	}{
+		{t0, "potato.txt", "potato-v1970-01-01-010101-123.txt"},
+		{t1, "potato", "potato-v2001-02-03-040506-123"},
+		{t1, "", "-v2001-02-03-040506-123"},
+	} {
+		actual := test.t.AddVersion(test.in)
+		assert.Equal(t, test.expected, actual, test.in)
+	}
+}
+
+func TestTimestampRemoveVersion(t *testing.T) {
+	for _, test := range []struct {
+		in             string
+		expectedT      api.Timestamp
+		expectedRemote string
+	}{
+		{"potato.txt", emptyT, "potato.txt"},
+		{"potato-v1970-01-01-010101-123.txt", t0r, "potato.txt"},
+		{"potato-v2001-02-03-040506-123", t1, "potato"},
+		{"-v2001-02-03-040506-123", t1, ""},
+		{"potato-v2A01-02-03-040506-123", emptyT, "potato-v2A01-02-03-040506-123"},
+		{"potato-v2001-02-03-040506=123", emptyT, "potato-v2001-02-03-040506=123"},
+	} {
+		actualT, actualRemote := api.RemoveVersion(test.in)
+		assert.Equal(t, test.expectedT, actualT, test.in)
+		assert.Equal(t, test.expectedRemote, actualRemote, test.in)
+	}
+}
+
 func TestTimestampIsZero(t *testing.T) {
 	assert.True(t, emptyT.IsZero())
 	assert.False(t, t0.IsZero())
@@ -42,11 +77,11 @@ func TestTimestampIsZero(t *testing.T) {
 }
 
 func TestTimestampEqual(t *testing.T) {
-	assert.False(t, emptyT.Equal(emptyT)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
+	assert.False(t, emptyT.Equal(emptyT))
 	assert.False(t, t0.Equal(emptyT))
 	assert.False(t, emptyT.Equal(t0))
 	assert.False(t, t0.Equal(t1))
 	assert.False(t, t1.Equal(t0))
-	assert.True(t, t0.Equal(t0)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
-	assert.True(t, t1.Equal(t1)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
+	assert.True(t, t0.Equal(t0))
+	assert.True(t, t1.Equal(t1))
 }

backend/b2/b2_internal_test.go

@@ -1,29 +1,14 @@
 package b2
 
 import (
-	"context"
-	"crypto/sha1"
-	"fmt"
-	"path"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/backend/b2/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/cache"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/bucket"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/rclone/rclone/lib/version"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
+	"github.com/ncw/rclone/fstest"
 )
 
 // Test b2 string encoding
-// https://www.backblaze.com/docs/cloud-storage-native-api-string-encoding
+// https://www.backblaze.com/b2/docs/string_encoding.html
 
 var encodeTest = []struct {
 	fullyEncoded     string
@@ -183,304 +168,3 @@ func TestParseTimeString(t *testing.T) {
 	}
 
 }
-
-// Return a map of the headers in the options with keys stripped of the "x-bz-info-" prefix
-func OpenOptionToMetaData(options []fs.OpenOption) map[string]string {
-	var headers = make(map[string]string)
-	for _, option := range options {
-		k, v := option.Header()
-		k = strings.ToLower(k)
-		if strings.HasPrefix(k, headerPrefix) {
-			headers[k[len(headerPrefix):]] = v
-		}
-	}
-
-	return headers
-}
-
-func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string, chunkSize string) {
-	what := fmt.Sprintf("Size%s/UploadCutoff%s/ChunkSize%s", size, uploadCutoff, chunkSize)
-	t.Run(what, func(t *testing.T) {
-		ctx := context.Background()
-
-		ss := fs.SizeSuffix(0)
-		err := ss.Set(size)
-		require.NoError(t, err)
-		original := random.String(int(ss))
-
-		contents := fstest.Gz(t, original)
-		mimeType := "text/html"
-
-		if chunkSize != "" {
-			ss := fs.SizeSuffix(0)
-			err := ss.Set(chunkSize)
-			require.NoError(t, err)
-			_, err = f.SetUploadChunkSize(ss)
-			require.NoError(t, err)
-		}
-
-		if uploadCutoff != "" {
-			ss := fs.SizeSuffix(0)
-			err := ss.Set(uploadCutoff)
-			require.NoError(t, err)
-			_, err = f.SetUploadCutoff(ss)
-			require.NoError(t, err)
-		}
-
-		item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z"))
-		btime := time.Now()
-		metadata := fs.Metadata{
-			// Just mtime for now - limit to milliseconds since x-bz-info-src_last_modified_millis can't support any
-
-			"mtime": "2009-05-06T04:05:06.499Z",
-		}
-
-		// Need to specify HTTP options with the header prefix since they are passed as-is
-		options := []fs.OpenOption{
-			&fs.HTTPOption{Key: "X-Bz-Info-a", Value: "1"},
-			&fs.HTTPOption{Key: "X-Bz-Info-b", Value: "2"},
-		}
-
-		obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, mimeType, metadata, options...)
-		defer func() {
-			assert.NoError(t, obj.Remove(ctx))
-		}()
-		o := obj.(*Object)
-		gotMetadata, err := o.getMetaData(ctx)
-		require.NoError(t, err)
-
-		// X-Bz-Info-a & X-Bz-Info-b
-		optMetadata := OpenOptionToMetaData(options)
-		for k, v := range optMetadata {
-			got := gotMetadata.Info[k]
-			assert.Equal(t, v, got, k)
-		}
-
-		// mtime
-		for k, v := range metadata {
-			got := o.meta[k]
-			assert.Equal(t, v, got, k)
-		}
-
-		assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
-
-		// Modification time from the x-bz-info-src_last_modified_millis header
-		var mtime api.Timestamp
-		err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey]))
-		if err != nil {
-			fs.Debugf(o, "Bad "+timeHeader+" header: %v", err)
-		}
-		assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time")
-
-		// Upload time
-		gotBtime := time.Time(gotMetadata.UploadTimestamp)
-		dt := gotBtime.Sub(btime)
-		assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt))
-
-		t.Run("GzipEncoding", func(t *testing.T) {
-			// Test that the gzipped file we uploaded can be
-			// downloaded
-			checkDownload := func(wantContents string, wantSize int64, wantHash string) {
-				gotContents := fstests.ReadObject(ctx, t, o, -1)
-				assert.Equal(t, wantContents, gotContents)
-				assert.Equal(t, wantSize, o.Size())
-				gotHash, err := o.Hash(ctx, hash.SHA1)
-				require.NoError(t, err)
-				assert.Equal(t, wantHash, gotHash)
-			}
-
-			t.Run("NoDecompress", func(t *testing.T) {
-				checkDownload(contents, int64(len(contents)), sha1Sum(t, contents))
-			})
-		})
-	})
-}
-
-func (f *Fs) InternalTestMetadata(t *testing.T) {
-	// 1 kB regular file
-	f.internalTestMetadata(t, "1kiB", "", "")
-
-	// 10 MiB large file
-	f.internalTestMetadata(t, "10MiB", "6MiB", "6MiB")
-}
-
-func sha1Sum(t *testing.T, s string) string {
-	hash := sha1.Sum([]byte(s))
-	return fmt.Sprintf("%x", hash)
-}
-
-// This is adapted from the s3 equivalent.
-func (f *Fs) InternalTestVersions(t *testing.T) {
-	ctx := context.Background()
-
-	// Small pause to make the LastModified different since AWS
-	// only seems to track them to 1 second granularity
-	time.Sleep(2 * time.Second)
-
-	// Create an object
-	const dirName = "versions"
-	const fileName = dirName + "/" + "test-versions.txt"
-	contents := random.String(100)
-	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
-	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
-	defer func() {
-		assert.NoError(t, obj.Remove(ctx))
-	}()
-	objMetadata, err := obj.(*Object).getMetaData(ctx)
-	require.NoError(t, err)
-
-	// Small pause
-	time.Sleep(2 * time.Second)
-
-	// Remove it
-	assert.NoError(t, obj.Remove(ctx))
-
-	// Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity
-	time.Sleep(2 * time.Second)
-
-	// And create it with different size and contents
-	newContents := random.String(101)
-	newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z"))
-	newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true)
-	newObjMetadata, err := newObj.(*Object).getMetaData(ctx)
-	require.NoError(t, err)
-
-	t.Run("Versions", func(t *testing.T) {
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-
-		// Read the contents
-		entries, err := f.List(ctx, dirName)
-		require.NoError(t, err)
-		tests := 0
-		var fileNameVersion string
-		for _, entry := range entries {
-			t.Log(entry)
-			remote := entry.Remote()
-			if remote == fileName {
-				t.Run("ReadCurrent", func(t *testing.T) {
-					assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
-				})
-				tests++
-			} else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName {
-				t.Run("ReadVersion", func(t *testing.T) {
-					assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
-				})
-				assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time")
-				fileNameVersion = remote
-				tests++
-			}
-		}
-		assert.Equal(t, 2, tests, "object missing from listing")
-
-		// Check we can read the object with a version suffix
-		t.Run("NewObject", func(t *testing.T) {
-			o, err := f.NewObject(ctx, fileNameVersion)
-			require.NoError(t, err)
-			require.NotNil(t, o)
-			assert.Equal(t, int64(100), o.Size(), o.Remote())
-		})
-
-		// Check we can make a NewFs from that object with a version suffix
-		t.Run("NewFs", func(t *testing.T) {
-			newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion)
-			// Make sure --b2-versions is set in the config of the new remote
-			fs.Debugf(nil, "oldPath = %q", newPath)
-			lastColon := strings.LastIndex(newPath, ":")
-			require.True(t, lastColon >= 0)
-			newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:]
-			fs.Debugf(nil, "newPath = %q", newPath)
-			fNew, err := cache.Get(ctx, newPath)
-			// This should return pointing to a file
-			require.Equal(t, fs.ErrorIsFile, err)
-			require.NotNil(t, fNew)
-			// With the directory above
-			assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew)))
-		})
-	})
-
-	t.Run("VersionAt", func(t *testing.T) {
-		// We set --b2-version-at for this test so make sure we reset it at the end
-		defer func() {
-			f.opt.VersionAt = fs.Time{}
-		}()
-
-		var (
-			firstObjectTime  = time.Time(objMetadata.UploadTimestamp)
-			secondObjectTime = time.Time(newObjMetadata.UploadTimestamp)
-		)
-
-		for _, test := range []struct {
-			what     string
-			at       time.Time
-			want     []fstest.Item
-			wantErr  error
-			wantSize int64
-		}{
-			{
-				what:    "Before",
-				at:      firstObjectTime.Add(-time.Second),
-				want:    fstests.InternalTestFiles,
-				wantErr: fs.ErrorObjectNotFound,
-			},
-			{
-				what:     "AfterOne",
-				at:       firstObjectTime.Add(time.Second),
-				want:     append([]fstest.Item{item}, fstests.InternalTestFiles...),
-				wantSize: 100,
-			},
-			{
-				what:    "AfterDelete",
-				at:      secondObjectTime.Add(-time.Second),
-				want:    fstests.InternalTestFiles,
-				wantErr: fs.ErrorObjectNotFound,
-			},
-			{
-				what:     "AfterTwo",
-				at:       secondObjectTime.Add(time.Second),
-				want:     append([]fstest.Item{newItem}, fstests.InternalTestFiles...),
-				wantSize: 101,
-			},
-		} {
-			t.Run(test.what, func(t *testing.T) {
-				f.opt.VersionAt = fs.Time(test.at)
-				t.Run("List", func(t *testing.T) {
-					fstest.CheckListing(t, f, test.want)
-				})
-				// b2 NewObject doesn't work with VersionAt
-				//t.Run("NewObject", func(t *testing.T) {
-				//	gotObj, gotErr := f.NewObject(ctx, fileName)
-				//	assert.Equal(t, test.wantErr, gotErr)
-				//	if gotErr == nil {
-				//		assert.Equal(t, test.wantSize, gotObj.Size())
-				//	}
-				//})
-			})
-		}
-	})
-
-	t.Run("Cleanup", func(t *testing.T) {
-		require.NoError(t, f.cleanUp(ctx, true, false, 0))
-		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
-		fstest.CheckListing(t, f, items)
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-		fstest.CheckListing(t, f, items)
-	})
-
-	// Purge gets tested later
-}
-
-// -run TestIntegration/FsMkdir/FsPutFiles/Internal
-func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("Metadata", f.InternalTestMetadata)
-	t.Run("Versions", f.InternalTestVersions)
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)

backend/b2/b2_test.go

@@ -4,8 +4,8 @@ package b2
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
@@ -28,12 +28,7 @@ func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadCutoff(cs)
 }
 
-func (f *Fs) SetCopyCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setCopyCutoff(cs)
-}
-
 var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
 	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
-	_ fstests.SetCopyCutoffer     = (*Fs)(nil)
 )

backend/b2/upload.go

@@ -1,11 +1,11 @@
 // Upload large files for b2
 //
-// Docs - https://www.backblaze.com/docs/cloud-storage-large-files
+// Docs - https://www.backblaze.com/b2/docs/large_files.html
 
 package b2
 
 import (
-	"context"
+	"bytes"
 	"crypto/sha1"
 	"encoding/hex"
 	"fmt"
@@ -14,15 +14,12 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/rclone/rclone/backend/b2/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/fs/chunksize"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/pool"
-	"github.com/rclone/rclone/lib/rest"
-	"golang.org/x/sync/errgroup"
+	"github.com/ncw/rclone/backend/b2/api"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/rest"
+	"github.com/pkg/errors"
 )
 
 type hashAppendingReader struct {
@@ -72,109 +69,78 @@ func newHashAppendingReader(in io.Reader, h gohash.Hash) *hashAppendingReader {
 type largeUpload struct {
 	f        *Fs                             // parent Fs
 	o        *Object                         // object being uploaded
-	doCopy    bool                            // doing copy rather than upload
-	what      string                          // text name of operation for logs
 	in       io.Reader                       // read the data from here
 	wrap     accounting.WrapFn               // account parts being transferred
 	id       string                          // ID of the file being uploaded
 	size     int64                           // total size
-	parts     int                             // calculated number of parts, if known
-	sha1smu   sync.Mutex                      // mutex to protect sha1s
+	parts    int64                           // calculated number of parts, if known
 	sha1s    []string                        // slice of SHA1s for each part
 	uploadMu sync.Mutex                      // lock for upload variable
 	uploads  []*api.GetUploadPartURLResponse // result of get upload URL calls
-	chunkSize int64                           // chunk size to use
-	src       *Object                         // if copying, object we are reading from
-	info      *api.FileInfo                   // final response with info about the object
 }
 
 // newLargeUpload starts an upload of object o from in with metadata in src
-//
-// If newInfo is set then metadata from that will be used instead of reading it from src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File, options ...fs.OpenOption) (up *largeUpload, err error) {
+func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *largeUpload, err error) {
+	remote := o.remote
 	size := src.Size()
-	parts := 0
-	chunkSize := defaultChunkSize
+	parts := int64(0)
+	sha1SliceSize := int64(maxParts)
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
-		chunkSize = chunksize.Calculator(o, size, maxParts, defaultChunkSize)
-		parts = int(size / int64(chunkSize))
-		if size%int64(chunkSize) != 0 {
+		parts = size / int64(o.fs.opt.ChunkSize)
+		if size%int64(o.fs.opt.ChunkSize) != 0 {
 			parts++
 		}
+		if parts > maxParts {
+			return nil, errors.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts)
 		}
-	bucket, bucketPath := o.split()
-	bucketID, err := f.getBucketID(ctx, bucket)
+		sha1SliceSize = parts
+	}
+
+	modTime := src.ModTime()
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_start_large_file",
+	}
+	bucketID, err := f.getBucketID()
 	if err != nil {
 		return nil, err
 	}
 	var request = api.StartLargeFileRequest{
 		BucketID:    bucketID,
-		Name:     f.opt.Enc.FromStandardPath(bucketPath),
-	}
-	optionsToSend := make([]fs.OpenOption, 0, len(options))
-	if newInfo == nil {
-		modTime, err := o.getModTime(ctx, src, options)
-		if err != nil {
-			return nil, err
-		}
-
-		request.ContentType = fs.MimeType(ctx, src)
-		request.Info = map[string]string{
+		Name:        o.fs.root + remote,
+		ContentType: fs.MimeType(src),
+		Info: map[string]string{
 			timeKey: timeString(modTime),
-		}
-		// Custom upload headers - remove header prefix since they are sent in the body
-		for _, option := range options {
-			k, v := option.Header()
-			k = strings.ToLower(k)
-			if strings.HasPrefix(k, headerPrefix) {
-				request.Info[k[len(headerPrefix):]] = v
-			} else {
-				optionsToSend = append(optionsToSend, option)
-			}
+		},
 	}
 	// Set the SHA1 if known
-		if !o.fs.opt.DisableCheckSum || doCopy {
-			if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
+	if !o.fs.opt.DisableCheckSum {
+		if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
 			request.Info[sha1Key] = calculatedSha1
 		}
 	}
-	} else {
-		request.ContentType = newInfo.ContentType
-		request.Info = newInfo.Info
-	}
-	opts := rest.Opts{
-		Method:  "POST",
-		Path:    "/b2_start_large_file",
-		Options: optionsToSend,
-	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
-		return f.shouldRetry(ctx, resp, err)
+		resp, err := f.srv.CallJSON(&opts, &request, &response)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
 	}
+	// unwrap the accounting from the input, we use wrap to put it
+	// back on after the buffering
+	in, wrap := accounting.UnWrap(in)
 	up = &largeUpload{
 		f:     f,
 		o:     o,
-		doCopy:    doCopy,
-		what:      "upload",
+		in:    in,
+		wrap:  wrap,
 		id:    response.ID,
 		size:  size,
 		parts: parts,
-		sha1s:     make([]string, 0, 16),
-		chunkSize: int64(chunkSize),
-	}
-	// unwrap the accounting from the input, we use wrap to put it
-	// back on after the buffering
-	if doCopy {
-		up.what = "copy"
-		up.src = src.(*Object)
-	} else {
-		up.in, up.wrap = accounting.UnWrap(in)
+		sha1s: make([]string, sha1SliceSize),
 	}
 	return up, nil
 }
@@ -182,15 +148,10 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
 //
 // This should be returned with returnUploadURL when finished
-func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadPartURLResponse, err error) {
+func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err error) {
 	up.uploadMu.Lock()
-	if len(up.uploads) > 0 {
-		upload, up.uploads = up.uploads[0], up.uploads[1:]
-		up.uploadMu.Unlock()
-		return upload, nil
-	}
-	up.uploadMu.Unlock()
-
+	defer up.uploadMu.Unlock()
+	if len(up.uploads) == 0 {
 		opts := rest.Opts{
 			Method: "POST",
 			Path:   "/b2_get_upload_part_url",
@@ -198,12 +159,15 @@ func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadP
 		var request = api.GetUploadPartURLRequest{
 			ID: up.id,
 		}
-	err = up.f.pacer.Call(func() (bool, error) {
-		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &upload)
-		return up.f.shouldRetry(ctx, resp, err)
+		err := up.f.pacer.Call(func() (bool, error) {
+			resp, err := up.f.srv.CallJSON(&opts, &request, &upload)
+			return up.f.shouldRetry(resp, err)
 		})
 		if err != nil {
-		return nil, fmt.Errorf("failed to get upload URL: %w", err)
+			return nil, errors.Wrap(err, "failed to get upload URL")
+		}
+	} else {
+		upload, up.uploads = up.uploads[0], up.uploads[1:]
 	}
 	return upload, nil
 }
@@ -218,48 +182,26 @@ func (up *largeUpload) returnUploadURL(upload *api.GetUploadPartURLResponse) {
 	up.uploadMu.Unlock()
 }
 
-// Add an sha1 to the being built up sha1s
-func (up *largeUpload) addSha1(chunkNumber int, sha1 string) {
-	up.sha1smu.Lock()
-	defer up.sha1smu.Unlock()
-	if len(up.sha1s) < chunkNumber+1 {
-		up.sha1s = append(up.sha1s, make([]string, chunkNumber+1-len(up.sha1s))...)
-	}
-	up.sha1s[chunkNumber] = sha1
+// clearUploadURL clears the current UploadURL and the AuthorizationToken
+func (up *largeUpload) clearUploadURL() {
+	up.uploadMu.Lock()
+	up.uploads = nil
+	up.uploadMu.Unlock()
 }
 
-// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
-func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (size int64, err error) {
-	// Only account after the checksum reads have been done
-	if do, ok := reader.(pool.DelayAccountinger); ok {
-		// To figure out this number, do a transfer and if the accounted size is 0 or a
-		// multiple of what it should be, increase or decrease this number.
-		do.DelayAccounting(1)
-	}
-
-	err = up.f.pacer.Call(func() (bool, error) {
-		// Discover the size by seeking to the end
-		size, err = reader.Seek(0, io.SeekEnd)
-		if err != nil {
-			return false, err
-		}
-
-		// rewind the reader on retry and after reading size
-		_, err = reader.Seek(0, io.SeekStart)
-		if err != nil {
-			return false, err
-		}
-
-		fs.Debugf(up.o, "Sending chunk %d length %d", chunkNumber, size)
+// Transfer a chunk
+func (up *largeUpload) transferChunk(part int64, body []byte) error {
+	err := up.f.pacer.Call(func() (bool, error) {
+		fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body))
 
 		// Get upload URL
-		upload, err := up.getUploadURL(ctx)
+		upload, err := up.getUploadURL()
 		if err != nil {
 			return false, err
 		}
 
-		in := newHashAppendingReader(reader, sha1.New())
-		sizeWithHash := size + int64(in.AdditionalLength())
+		in := newHashAppendingReader(bytes.NewReader(body), sha1.New())
+		size := int64(len(body)) + int64(in.AdditionalLength())
 
 		// Authorization
 		//
@@ -275,7 +217,7 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
 		// The number of bytes in the file being uploaded. Note that
 		// this header is required; you cannot leave it out and just
 		// use chunked encoding.  The minimum size of every part but
-		// the last one is 100 MB (100,000,000 bytes)
+		// the last one is 100MB.
 		//
 		// X-Bz-Content-Sha1
 		//
@@ -289,18 +231,18 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
 			Body:    up.wrap(in),
 			ExtraHeaders: map[string]string{
 				"Authorization":    upload.AuthorizationToken,
-				"X-Bz-Part-Number": fmt.Sprintf("%d", chunkNumber+1),
+				"X-Bz-Part-Number": fmt.Sprintf("%d", part),
 				sha1Header:         "hex_digits_at_end",
 			},
-			ContentLength: &sizeWithHash,
+			ContentLength: &size,
 		}
 
 		var response api.UploadPartResponse
 
-		resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)
-		retry, err := up.f.shouldRetry(ctx, resp, err)
+		resp, err := up.f.srv.CallJSON(&opts, nil, &response)
+		retry, err := up.f.shouldRetry(resp, err)
 		if err != nil {
-			fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", chunkNumber, retry, err, err)
+			fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err)
 		}
 		// On retryable error clear PartUploadURL
 		if retry {
@@ -308,52 +250,20 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
 			upload = nil
 		}
 		up.returnUploadURL(upload)
-		up.addSha1(chunkNumber, in.HexSum())
+		up.sha1s[part-1] = in.HexSum()
 		return retry, err
 	})
 	if err != nil {
-		fs.Debugf(up.o, "Error sending chunk %d: %v", chunkNumber, err)
+		fs.Debugf(up.o, "Error sending chunk %d: %v", part, err)
 	} else {
-		fs.Debugf(up.o, "Done sending chunk %d", chunkNumber)
-	}
-	return size, err
-}
-
-// Copy a chunk
-func (up *largeUpload) copyChunk(ctx context.Context, part int, partSize int64) error {
-	err := up.f.pacer.Call(func() (bool, error) {
-		fs.Debugf(up.o, "Copying chunk %d length %d", part, partSize)
-		opts := rest.Opts{
-			Method: "POST",
-			Path:   "/b2_copy_part",
-		}
-		offset := int64(part) * up.chunkSize // where we are in the source file
-		var request = api.CopyPartRequest{
-			SourceID:    up.src.id,
-			LargeFileID: up.id,
-			PartNumber:  int64(part + 1),
-			Range:       fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
-		}
-		var response api.UploadPartResponse
-		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
-		retry, err := up.f.shouldRetry(ctx, resp, err)
-		if err != nil {
-			fs.Debugf(up.o, "Error copying chunk %d (retry=%v): %v: %#v", part, retry, err, err)
-		}
-		up.addSha1(part, response.SHA1)
-		return retry, err
-	})
-	if err != nil {
-		fs.Debugf(up.o, "Error copying chunk %d: %v", part, err)
-	} else {
-		fs.Debugf(up.o, "Done copying chunk %d", part)
+		fs.Debugf(up.o, "Done sending chunk %d", part)
 	}
 	return err
 }
 
-// Close closes off the large upload
-func (up *largeUpload) Close(ctx context.Context) error {
-	fs.Debugf(up.o, "Finishing large file %s with %d parts", up.what, up.parts)
+// finish closes off the large upload
+func (up *largeUpload) finish() error {
+	fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_finish_large_file",
@@ -364,19 +274,17 @@ func (up *largeUpload) Close(ctx context.Context) error {
 	}
 	var response api.FileInfo
 	err := up.f.pacer.Call(func() (bool, error) {
-		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
-		return up.f.shouldRetry(ctx, resp, err)
+		resp, err := up.f.srv.CallJSON(&opts, &request, &response)
+		return up.f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
 	}
-	up.info = &response
-	return nil
+	return up.o.decodeMetaDataFileInfo(&response)
 }
 
-// Abort aborts the large upload
-func (up *largeUpload) Abort(ctx context.Context) error {
-	fs.Debugf(up.o, "Cancelling large file %s", up.what)
+// cancel aborts the large upload
+func (up *largeUpload) cancel() error {
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_cancel_large_file",
@@ -386,119 +294,142 @@ func (up *largeUpload) Abort(ctx context.Context) error {
 	}
 	var response api.CancelLargeFileResponse
 	err := up.f.pacer.Call(func() (bool, error) {
-		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
-		return up.f.shouldRetry(ctx, resp, err)
+		resp, err := up.f.srv.CallJSON(&opts, &request, &response)
+		return up.f.shouldRetry(resp, err)
 	})
+	return err
+}
+
+func (up *largeUpload) managedTransferChunk(wg *sync.WaitGroup, errs chan error, part int64, buf []byte) {
+	wg.Add(1)
+	go func(part int64, buf []byte) {
+		defer wg.Done()
+		defer up.f.putUploadBlock(buf)
+		err := up.transferChunk(part, buf)
 		if err != nil {
-		fs.Errorf(up.o, "Failed to cancel large file %s: %v", up.what, err)
+			select {
+			case errs <- err:
+			default:
+			}
+		}
+	}(part, buf)
+}
+
+func (up *largeUpload) finishOrCancelOnError(err error, errs chan error) error {
+	if err == nil {
+		select {
+		case err = <-errs:
+		default:
+		}
+	}
+	if err != nil {
+		fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err)
+		cancelErr := up.cancel()
+		if cancelErr != nil {
+			fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr)
 		}
 		return err
 	}
+	return up.finish()
+}
 
 // Stream uploads the chunks from the input, starting with a required initial
 // chunk. Assumes the file size is unknown and will upload until the input
 // reaches EOF.
-//
-// Note that initialUploadBlock must be returned to f.putBuf()
-func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock *pool.RW) (err error) {
-	defer atexit.OnError(&err, func() { _ = up.Abort(ctx) })()
+func (up *largeUpload) Stream(initialUploadBlock []byte) (err error) {
 	fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
-	var (
-		g, gCtx      = errgroup.WithContext(ctx)
-		hasMoreParts = true
-	)
-	up.size = initialUploadBlock.Size()
-	up.parts = 0
-	for part := 0; hasMoreParts; part++ {
-		// Get a block of memory from the pool and token which limits concurrency.
-		var rw *pool.RW
-		if part == 0 {
-			rw = initialUploadBlock
-		} else {
-			rw = up.f.getRW(false)
+	errs := make(chan error, 1)
+	hasMoreParts := true
+	var wg sync.WaitGroup
+
+	// Transfer initial chunk
+	up.size = int64(len(initialUploadBlock))
+	up.managedTransferChunk(&wg, errs, 1, initialUploadBlock)
+
+outer:
+	for part := int64(2); hasMoreParts; part++ {
+		// Check any errors
+		select {
+		case err = <-errs:
+			break outer
+		default:
 		}
 
-		// Fail fast, in case an errgroup managed function returns an error
-		// gCtx is cancelled. There is no point in uploading all the other parts.
-		if gCtx.Err() != nil {
-			up.f.putRW(rw)
-			break
-		}
+		// Get a block of memory
+		buf := up.f.getUploadBlock()
 
 		// Read the chunk
-		var n int64
-		if part == 0 {
-			n = rw.Size()
-		} else {
-			n, err = io.CopyN(rw, up.in, up.chunkSize)
-			if err == io.EOF {
-				if n == 0 {
-					fs.Debugf(up.o, "Not sending empty chunk after EOF - ending.")
-					up.f.putRW(rw)
-					break
-				} else {
-					fs.Debugf(up.o, "Read less than a full chunk %d, making this the last one.", n)
-				}
+		var n int
+		n, err = io.ReadFull(up.in, buf)
+		if err == io.ErrUnexpectedEOF {
+			fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
+			buf = buf[:n]
 			hasMoreParts = false
+			err = nil
+		} else if err == io.EOF {
+			fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
+			up.f.putUploadBlock(buf)
+			err = nil
+			break outer
 		} else if err != nil {
 			// other kinds of errors indicate failure
-				up.f.putRW(rw)
-				return err
-			}
+			up.f.putUploadBlock(buf)
+			break outer
 		}
 
 		// Keep stats up to date
-		up.parts += 1
-		up.size += n
+		up.parts = part
+		up.size += int64(n)
 		if part > maxParts {
-			up.f.putRW(rw)
-			return fmt.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
+			err = errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
+			break outer
 		}
 
-		part := part // for the closure
-		g.Go(func() (err error) {
-			defer up.f.putRW(rw)
-			_, err = up.WriteChunk(gCtx, part, rw)
-			return err
-		})
+		// Transfer the chunk
+		up.managedTransferChunk(&wg, errs, part, buf)
 	}
-	err = g.Wait()
-	if err != nil {
-		return err
-	}
-	return up.Close(ctx)
+	wg.Wait()
+	up.sha1s = up.sha1s[:up.parts]
+
+	return up.finishOrCancelOnError(err, errs)
 }
 
-// Copy the chunks from the source to the destination
-func (up *largeUpload) Copy(ctx context.Context) (err error) {
-	defer atexit.OnError(&err, func() { _ = up.Abort(ctx) })()
-	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
-	var (
-		g, gCtx   = errgroup.WithContext(ctx)
-		remaining = up.size
-	)
-	g.SetLimit(up.f.opt.UploadConcurrency)
-	for part := 0; part < up.parts; part++ {
-		// Fail fast, in case an errgroup managed function returns an error
-		// gCtx is cancelled. There is no point in copying all the other parts.
-		if gCtx.Err() != nil {
-			break
+// Upload uploads the chunks from the input
+func (up *largeUpload) Upload() error {
+	fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id)
+	remaining := up.size
+	errs := make(chan error, 1)
+	var wg sync.WaitGroup
+	var err error
+outer:
+	for part := int64(1); part <= up.parts; part++ {
+		// Check any errors
+		select {
+		case err = <-errs:
+			break outer
+		default:
 		}
 
 		reqSize := remaining
-		if reqSize >= up.chunkSize {
-			reqSize = up.chunkSize
+		if reqSize >= int64(up.f.opt.ChunkSize) {
+			reqSize = int64(up.f.opt.ChunkSize)
 		}
 
-		part := part // for the closure
-		g.Go(func() (err error) {
-			return up.copyChunk(gCtx, part, reqSize)
-		})
+		// Get a block of memory
+		buf := up.f.getUploadBlock()[:reqSize]
+
+		// Read the chunk
+		_, err = io.ReadFull(up.in, buf)
+		if err != nil {
+			up.f.putUploadBlock(buf)
+			break outer
+		}
+
+		// Transfer the chunk
+		up.managedTransferChunk(&wg, errs, part, buf)
 		remaining -= reqSize
 	}
-	err = g.Wait()
-	if err != nil {
-		return err
-	}
-	return up.Close(ctx)
+	wg.Wait()
+
+	return up.finishOrCancelOnError(err, errs)
 }

backend/box/api/types.go

@@ -14,7 +14,7 @@ const (
 	timeFormat = `"` + time.RFC3339 + `"`
 )
 
-// Time represents date and time information for the
+// Time represents represents date and time information for the
 // box API, by using RFC3339
 type Time time.Time
 
@@ -39,7 +39,7 @@ type Error struct {
 	Type        string `json:"type"`
 	Status      int    `json:"status"`
 	Code        string `json:"code"`
-	ContextInfo json.RawMessage `json:"context_info"`
+	ContextInfo json.RawMessage
 	HelpURL     string `json:"help_url"`
 	Message     string `json:"message"`
 	RequestID   string `json:"request_id"`
@@ -52,7 +52,7 @@ func (e *Error) Error() string {
 		out += ": " + e.Message
 	}
 	if e.ContextInfo != nil {
-		out += fmt.Sprintf(" (%s)", string(e.ContextInfo))
+		out += fmt.Sprintf(" (%+v)", e.ContextInfo)
 	}
 	return out
 }
@@ -61,9 +61,9 @@ func (e *Error) Error() string {
 var _ error = (*Error)(nil)
 
 // ItemFields are the fields needed for FileInfo
-var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status,shared_link,owned_by"
+var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status,shared_link"
 
-// Types of things in Item/ItemMini
+// Types of things in Item
 const (
 	ItemTypeFolder    = "folder"
 	ItemTypeFile      = "file"
@@ -72,21 +72,11 @@ const (
 	ItemStatusDeleted = "deleted"
 )
 
-// ItemMini is a subset of the elements in a full Item returned by some API calls
-type ItemMini struct {
-	Type       string `json:"type"`
-	ID         string `json:"id"`
-	SequenceID int64  `json:"sequence_id,string"`
-	Etag       string `json:"etag"`
-	SHA1       string `json:"sha1"`
-	Name       string `json:"name"`
-}
-
 // Item describes a folder or a file as returned by Get Folder Items and others
 type Item struct {
 	Type              string  `json:"type"`
 	ID                string  `json:"id"`
-	SequenceID        int64    `json:"sequence_id,string"`
+	SequenceID        string  `json:"sequence_id"`
 	Etag              string  `json:"etag"`
 	SHA1              string  `json:"sha1"`
 	Name              string  `json:"name"`
@@ -96,17 +86,10 @@ type Item struct {
 	ContentCreatedAt  Time    `json:"content_created_at"`
 	ContentModifiedAt Time    `json:"content_modified_at"`
 	ItemStatus        string  `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted
-	Parent            ItemMini `json:"parent"`
 	SharedLink        struct {
 		URL    string `json:"url,omitempty"`
 		Access string `json:"access,omitempty"`
 	} `json:"shared_link"`
-	OwnedBy struct {
-		Type  string `json:"type"`
-		ID    string `json:"id"`
-		Name  string `json:"name"`
-		Login string `json:"login"`
-	} `json:"owned_by"`
 }
 
 // ModTime returns the modification time of the item
@@ -124,7 +107,6 @@ type FolderItems struct {
 	Entries    []Item `json:"entries"`
 	Offset     int    `json:"offset"`
 	Limit      int    `json:"limit"`
-	NextMarker *string `json:"next_marker,omitempty"`
 	Order      []struct {
 		By        string `json:"by"`
 		Direction string `json:"direction"`
@@ -150,26 +132,6 @@ type UploadFile struct {
 	ContentModifiedAt Time   `json:"content_modified_at"`
 }
 
-// PreUploadCheck is the request for upload preflight check
-type PreUploadCheck struct {
-	Name   string `json:"name"`
-	Parent Parent `json:"parent"`
-	Size   *int64 `json:"size,omitempty"`
-}
-
-// PreUploadCheckResponse is the response from upload preflight check
-// if successful
-type PreUploadCheckResponse struct {
-	UploadToken string `json:"upload_token"`
-	UploadURL   string `json:"upload_url"`
-}
-
-// PreUploadCheckConflict is returned in the ContextInfo error field
-// from PreUploadCheck when the error code is "item_name_in_use"
-type PreUploadCheckConflict struct {
-	Conflicts ItemMini `json:"conflicts"`
-}
-
 // UpdateFileModTime is used in Update File Info
 type UpdateFileModTime struct {
 	ContentModifiedAt Time `json:"content_modified_at"`
@@ -240,70 +202,3 @@ type CommitUpload struct {
 		ContentModifiedAt Time `json:"content_modified_at"`
 	} `json:"attributes"`
 }
-
-// ConfigJSON defines the shape of a box config.json
-type ConfigJSON struct {
-	BoxAppSettings AppSettings `json:"boxAppSettings"`
-	EnterpriseID   string      `json:"enterpriseID"`
-}
-
-// AppSettings defines the shape of the boxAppSettings within box config.json
-type AppSettings struct {
-	ClientID     string  `json:"clientID"`
-	ClientSecret string  `json:"clientSecret"`
-	AppAuth      AppAuth `json:"appAuth"`
-}
-
-// AppAuth defines the shape of the appAuth within boxAppSettings in config.json
-type AppAuth struct {
-	PublicKeyID string `json:"publicKeyID"`
-	PrivateKey  string `json:"privateKey"`
-	Passphrase  string `json:"passphrase"`
-}
-
-// User is returned from /users/me
-type User struct {
-	Type          string    `json:"type"`
-	ID            string    `json:"id"`
-	Name          string    `json:"name"`
-	Login         string    `json:"login"`
-	CreatedAt     time.Time `json:"created_at"`
-	ModifiedAt    time.Time `json:"modified_at"`
-	Language      string    `json:"language"`
-	Timezone      string    `json:"timezone"`
-	SpaceAmount   int64     `json:"space_amount"`
-	SpaceUsed     int64     `json:"space_used"`
-	MaxUploadSize int64     `json:"max_upload_size"`
-	Status        string    `json:"status"`
-	JobTitle      string    `json:"job_title"`
-	Phone         string    `json:"phone"`
-	Address       string    `json:"address"`
-	AvatarURL     string    `json:"avatar_url"`
-}
-
-// FileTreeChangeEventTypes are the events that can require cache invalidation
-var FileTreeChangeEventTypes = map[string]struct{}{
-	"ITEM_COPY":                 {},
-	"ITEM_CREATE":               {},
-	"ITEM_MAKE_CURRENT_VERSION": {},
-	"ITEM_MODIFY":               {},
-	"ITEM_MOVE":                 {},
-	"ITEM_RENAME":               {},
-	"ITEM_TRASH":                {},
-	"ITEM_UNDELETE_VIA_TRASH":   {},
-	"ITEM_UPLOAD":               {},
-}
-
-// Event is an array element in the response returned from /events
-type Event struct {
-	EventType string `json:"event_type"`
-	EventID   string `json:"event_id"`
-	Source    Item   `json:"source"`
-}
-
-// Events is returned from /events
-type Events struct {
-	ChunkSize          int64   `json:"chunk_size"`
-	Entries            []Event `json:"entries"`
-	NextStreamPosition int64   `json:"next_stream_position"`
-}

backend/box/box_test.go

@@ -4,8 +4,8 @@ package box_test
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/box"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/box"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote

backend/box/upload.go

@@ -1,14 +1,12 @@
-// multipart upload for box
+// multpart upload for box
 
 package box
 
 import (
 	"bytes"
-	"context"
 	"crypto/sha1"
 	"encoding/base64"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -16,15 +14,15 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/rclone/backend/box/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/rest"
+	"github.com/ncw/rclone/backend/box/api"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/lib/rest"
+	"github.com/pkg/errors"
 )
 
 // createUploadSession creates an upload session for the object
-func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) {
+func (o *Object) createUploadSession(leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) {
 	opts := rest.Opts{
 		Method:  "POST",
 		Path:    "/files/upload_sessions",
@@ -39,12 +37,12 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri
 	} else {
 		opts.Path = "/files/upload_sessions"
 		request.FolderID = directoryID
-		request.FileName = o.fs.opt.Enc.FromStandardName(leaf)
+		request.FileName = replaceReservedChars(leaf)
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
-		return shouldRetry(ctx, resp, err)
+		resp, err = o.fs.srv.CallJSON(&opts, &request, &response)
+		return shouldRetry(resp, err)
 	})
 	return
 }
@@ -55,7 +53,7 @@ func sha1Digest(digest []byte) string {
 }
 
 // uploadPart uploads a part in an upload session
-func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
+func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) {
 	chunkSize := int64(len(chunk))
 	sha1sum := sha1.Sum(chunk)
 	opts := rest.Opts{
@@ -65,7 +63,6 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 		ContentType:   "application/octet-stream",
 		ContentLength: &chunkSize,
 		ContentRange:  fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
-		Options:       options,
 		ExtraHeaders: map[string]string{
 			"Digest": sha1Digest(sha1sum[:]),
 		},
@@ -73,8 +70,8 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts.Body = wrap(bytes.NewReader(chunk))
-		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
-		return shouldRetry(ctx, resp, err)
+		resp, err = o.fs.srv.CallJSON(&opts, nil, &response)
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -83,7 +80,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 }
 
 // commitUpload finishes an upload session
-func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) {
+func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) {
 	opts := rest.Opts{
 		Method:  "POST",
 		Path:    "/files/upload_sessions/" + SessionID + "/commit",
@@ -100,22 +97,22 @@ func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api
 	var body []byte
 	var resp *http.Response
 	// For discussion of this value see:
-	// https://github.com/rclone/rclone/issues/2054
+	// https://github.com/ncw/rclone/issues/2054
 	maxTries := o.fs.opt.CommitRetries
 	const defaultDelay = 10
 	var tries int
 outer:
 	for tries = 0; tries < maxTries; tries++ {
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
+			resp, err = o.fs.srv.CallJSON(&opts, &request, nil)
 			if err != nil {
-				return shouldRetry(ctx, resp, err)
+				return shouldRetry(resp, err)
 			}
 			body, err = rest.ReadBody(resp)
-			return shouldRetry(ctx, resp, err)
+			return shouldRetry(resp, err)
 		})
 		delay := defaultDelay
-		var why string
+		why := "unknown"
 		if err != nil {
 			// Sometimes we get 400 Error with
 			// parts_mismatch immediately after uploading
@@ -140,7 +137,7 @@ outer:
 					}
 				}
 			default:
-				return nil, fmt.Errorf("unknown HTTP status return %q (%d)", resp.Status, resp.StatusCode)
+				return nil, errors.Errorf("unknown HTTP status return %q (%d)", resp.Status, resp.StatusCode)
 			}
 		}
 		fs.Debugf(o, "commit multipart upload failed %d/%d - trying again in %d seconds (%s)", tries+1, maxTries, delay, why)
@@ -151,13 +148,13 @@ outer:
 	}
 	err = json.Unmarshal(body, &result)
 	if err != nil {
-		return nil, fmt.Errorf("couldn't decode commit response: %q: %w", body, err)
+		return nil, errors.Wrapf(err, "couldn't decode commit response: %q", body)
 	}
 	return result, nil
 }
 
 // abortUpload cancels an upload session
-func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error) {
+func (o *Object) abortUpload(SessionID string) (err error) {
 	opts := rest.Opts{
 		Method:     "DELETE",
 		Path:       "/files/upload_sessions/" + SessionID,
@@ -166,30 +163,32 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error)
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
+		resp, err = o.fs.srv.Call(&opts)
+		return shouldRetry(resp, err)
 	})
 	return err
 }
 
 // uploadMultipart uploads a file using multipart upload
-func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
+func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) {
 	// Create upload session
-	session, err := o.createUploadSession(ctx, leaf, directoryID, size)
+	session, err := o.createUploadSession(leaf, directoryID, size)
 	if err != nil {
-		return fmt.Errorf("multipart upload create session failed: %w", err)
+		return errors.Wrap(err, "multipart upload create session failed")
 	}
 	chunkSize := session.PartSize
 	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", session.TotalParts, fs.SizeSuffix(chunkSize))
 
 	// Cancel the session if something went wrong
-	defer atexit.OnError(&err, func() {
+	defer func() {
+		if err != nil {
 			fs.Debugf(o, "Cancelling multipart upload: %v", err)
-		cancelErr := o.abortUpload(ctx, session.ID)
+			cancelErr := o.abortUpload(session.ID)
 			if cancelErr != nil {
-			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
+				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
 			}
-	})()
+		}
+	}()
 
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
@@ -222,7 +221,7 @@ outer:
 		// Read the chunk
 		_, err = io.ReadFull(in, buf)
 		if err != nil {
-			err = fmt.Errorf("multipart upload failed to read source: %w", err)
+			err = errors.Wrap(err, "multipart upload failed to read source")
 			break outer
 		}
 
@@ -236,9 +235,9 @@ outer:
 			defer wg.Done()
 			defer o.fs.uploadToken.Put()
 			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
-			partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
+			partResponse, err := o.uploadPart(session.ID, position, size, buf, wrap)
 			if err != nil {
-				err = fmt.Errorf("multipart upload failed to upload part: %w", err)
+				err = errors.Wrap(err, "multipart upload failed to upload part")
 				select {
 				case errs <- err:
 				default:
@@ -264,13 +263,13 @@ outer:
 	}
 
 	// Finalise the upload session
-	result, err := o.commitUpload(ctx, session.ID, parts, modTime, hash.Sum(nil))
+	result, err := o.commitUpload(session.ID, parts, modTime, hash.Sum(nil))
 	if err != nil {
-		return fmt.Errorf("multipart upload failed to finalize: %w", err)
+		return errors.Wrap(err, "multipart upload failed to finalize")
 	}
 
 	if result.TotalCount != 1 || len(result.Entries) != 1 {
-		return fmt.Errorf("multipart upload failed %v - not sure why", o)
+		return errors.Errorf("multipart upload failed %v - not sure why", o)
 	}
 	return o.setMetaData(&result.Entries[0])
 }

backend/cache/cache_mount_unix_test.go

@@ -0,0 +1,78 @@
+// +build !plan9,!windows
+
+package cache_test
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"bazil.org/fuse"
+	fusefs "bazil.org/fuse/fs"
+	"github.com/ncw/rclone/cmd/mount"
+	"github.com/ncw/rclone/cmd/mountlib"
+	"github.com/ncw/rclone/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	device := f.Name() + ":" + f.Root()
+	var options = []fuse.MountOption{
+		fuse.MaxReadahead(uint32(mountlib.MaxReadAhead)),
+		fuse.Subtype("rclone"),
+		fuse.FSName(device), fuse.VolumeName(device),
+		fuse.NoAppleDouble(),
+		fuse.NoAppleXattr(),
+		//fuse.AllowOther(),
+	}
+	err := os.MkdirAll(r.mntDir, os.ModePerm)
+	require.NoError(t, err)
+	c, err := fuse.Mount(r.mntDir, options...)
+	require.NoError(t, err)
+	filesys := mount.NewFS(f)
+	server := fusefs.New(c, nil)
+
+	// Serve the mount point in the background returning error to errChan
+	r.unmountRes = make(chan error, 1)
+	go func() {
+		err := server.Serve(filesys)
+		closeErr := c.Close()
+		if err == nil {
+			err = closeErr
+		}
+		r.unmountRes <- err
+	}()
+
+	// check if the mount process has an error to report
+	<-c.Ready
+	require.NoError(t, c.MountError)
+
+	r.unmountFn = func() error {
+		// Shutdown the VFS
+		filesys.VFS.Shutdown()
+		return fuse.Unmount(r.mntDir)
+	}
+
+	r.vfs = filesys.VFS
+	r.isMounted = true
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	var err error
+
+	for i := 0; i < 4; i++ {
+		err = r.unmountFn()
+		if err != nil {
+			//log.Printf("signal to umount failed - retrying: %v", err)
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		break
+	}
+	require.NoError(t, err)
+	err = <-r.unmountRes
+	require.NoError(t, err)
+	err = r.vfs.CleanUp()
+	require.NoError(t, err)
+	r.isMounted = false
+}

backend/cache/cache_mount_windows_test.go

@@ -0,0 +1,124 @@
+// +build windows
+
+package cache_test
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/billziss-gh/cgofuse/fuse"
+	"github.com/ncw/rclone/cmd/cmount"
+	"github.com/ncw/rclone/cmd/mountlib"
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+)
+
+// waitFor runs fn() until it returns true or the timeout expires
+func waitFor(fn func() bool) (ok bool) {
+	const totalWait = 10 * time.Second
+	const individualWait = 10 * time.Millisecond
+	for i := 0; i < int(totalWait/individualWait); i++ {
+		ok = fn()
+		if ok {
+			return ok
+		}
+		time.Sleep(individualWait)
+	}
+	return false
+}
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	// FIXME implement cmount
+	t.Skip("windows not supported yet")
+
+	device := f.Name() + ":" + f.Root()
+	options := []string{
+		"-o", "fsname=" + device,
+		"-o", "subtype=rclone",
+		"-o", fmt.Sprintf("max_readahead=%d", mountlib.MaxReadAhead),
+		"-o", "uid=-1",
+		"-o", "gid=-1",
+		"-o", "allow_other",
+		// This causes FUSE to supply O_TRUNC with the Open
+		// call which is more efficient for cmount.  However
+		// it does not work with cgofuse on Windows with
+		// WinFSP so cmount must work with or without it.
+		"-o", "atomic_o_trunc",
+		"--FileSystemName=rclone",
+	}
+
+	fsys := cmount.NewFS(f)
+	host := fuse.NewFileSystemHost(fsys)
+
+	// Serve the mount point in the background returning error to errChan
+	r.unmountRes = make(chan error, 1)
+	go func() {
+		var err error
+		ok := host.Mount(r.mntDir, options)
+		if !ok {
+			err = errors.New("mount failed")
+		}
+		r.unmountRes <- err
+	}()
+
+	// unmount
+	r.unmountFn = func() error {
+		// Shutdown the VFS
+		fsys.VFS.Shutdown()
+		if host.Unmount() {
+			if !waitFor(func() bool {
+				_, err := os.Stat(r.mntDir)
+				return err != nil
+			}) {
+				t.Fatalf("mountpoint %q didn't disappear after unmount - continuing anyway", r.mntDir)
+			}
+			return nil
+		}
+		return errors.New("host unmount failed")
+	}
+
+	// Wait for the filesystem to become ready, checking the file
+	// system didn't blow up before starting
+	select {
+	case err := <-r.unmountRes:
+		require.NoError(t, err)
+	case <-time.After(time.Second * 3):
+	}
+
+	// Wait for the mount point to be available on Windows
+	// On Windows the Init signal comes slightly before the mount is ready
+	if !waitFor(func() bool {
+		_, err := os.Stat(r.mntDir)
+		return err == nil
+	}) {
+		t.Errorf("mountpoint %q didn't became available on mount", r.mntDir)
+	}
+
+	r.vfs = fsys.VFS
+	r.isMounted = true
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	// FIXME implement cmount
+	t.Skip("windows not supported yet")
+	var err error
+
+	for i := 0; i < 4; i++ {
+		err = r.unmountFn()
+		if err != nil {
+			//log.Printf("signal to umount failed - retrying: %v", err)
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		break
+	}
+	require.NoError(t, err)
+	err = <-r.unmountRes
+	require.NoError(t, err)
+	err = r.vfs.CleanUp()
+	require.NoError(t, err)
+	r.isMounted = false
+}

backend/cache/cache_test.go

@@ -1,15 +1,15 @@
 // Test Cache filesystem interface
 
-//go:build !plan9 && !js && !race
+// +build !plan9
 
 package cache_test
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/cache"
-	_ "github.com/rclone/rclone/backend/local"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
@@ -17,9 +17,5 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestCache:",
 		NilObject:  (*cache.Object)(nil),
-		UnimplementableFsMethods:        []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter", "DirSetModTime", "MkdirMetadata"},
-		UnimplementableObjectMethods:    []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata", "SetMetadata"},
-		UnimplementableDirectoryMethods: []string{"Metadata", "SetMetadata", "SetModTime"},
-		SkipInvalidUTF8:                 true, // invalid UTF-8 confuses the cache
 	})
 }

backend/cache/cache_unsupported.go

@@ -1,7 +1,6 @@
 // Build for cache for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || js
+// +build plan9
 
-// Package cache implements a virtual provider to cache existing remotes.
 package cache

backend/cache/cache_upload_test.go

@@ -1,9 +1,8 @@
-//go:build !plan9 && !js && !race
+// +build !plan9
 
 package cache_test
 
 import (
-	"context"
 	"fmt"
 	"math/rand"
 	"os"
@@ -12,16 +11,18 @@ import (
 	"testing"
 	"time"
 
-	"github.com/rclone/rclone/backend/cache"
-	_ "github.com/rclone/rclone/backend/drive"
-	"github.com/rclone/rclone/fs"
+	"github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/drive"
+	"github.com/ncw/rclone/fs"
 	"github.com/stretchr/testify/require"
 )
 
 func TestInternalUploadTempDirCreated(t *testing.T) {
 	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
-	runInstance.newCacheFs(t, remoteName, id, false, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id)})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
 	require.NoError(t, err)
@@ -60,7 +61,9 @@ func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltD
 func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "0s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
@@ -68,21 +71,25 @@ func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
 
 func TestInternalUploadMoveExistingFile(t *testing.T) {
 	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir(context.Background(), "one")
+	err := rootFs.Mkdir("one")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "one/test")
+	err = rootFs.Mkdir("one/test")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "second")
+	err = rootFs.Mkdir("second")
 	require.NoError(t, err)
 
 	// create some rand test data
@@ -110,14 +117,16 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
 
 func TestInternalUploadTempPathCleaned(t *testing.T) {
 	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir(context.Background(), "one")
+	err := rootFs.Mkdir("one")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "one/test")
+	err = rootFs.Mkdir("one/test")
 	require.NoError(t, err)
-	err = rootFs.Mkdir(context.Background(), "second")
+	err = rootFs.Mkdir("second")
 	require.NoError(t, err)
 
 	// create some rand test data
@@ -151,19 +160,21 @@ func TestInternalUploadTempPathCleaned(t *testing.T) {
 
 func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir(context.Background(), "test")
+	err := rootFs.Mkdir("test")
 	require.NoError(t, err)
 	minSize := 5242880
 	maxSize := 10485760
 	totalFiles := 10
-	randInstance := rand.New(rand.NewSource(time.Now().Unix()))
+	rand.Seed(time.Now().Unix())
 
 	lastFile := ""
 	for i := 0; i < totalFiles; i++ {
-		size := int64(randInstance.Intn(maxSize-minSize) + minSize)
+		size := int64(rand.Intn(maxSize-minSize) + minSize)
 		testReader := runInstance.randomReader(t, size)
 		remote := "test/" + strconv.Itoa(i) + ".bin"
 		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
@@ -200,7 +211,9 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 func TestInternalUploadTempFileOperations(t *testing.T) {
 	id := "tiutfo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 
@@ -220,9 +233,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.dirMove(t, rootFs, "test", "second")
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.Error(t, err)
-		_, err = rootFs.NewObject(context.Background(), "second/one")
+		_, err = rootFs.NewObject("second/one")
 		require.NoError(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -243,7 +256,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.rm(t, rootFs, "test")
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "directory not empty")
-	_, err = rootFs.NewObject(context.Background(), "test/one")
+	_, err = rootFs.NewObject("test/one")
 	require.NoError(t, err)
 	// validate that it exists in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -257,9 +270,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	if err != errNotSupported {
 		require.NoError(t, err)
 		// try to read from it
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.Error(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/second")
+		_, err = rootFs.NewObject("test/second")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -276,9 +289,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/third")
+		_, err = rootFs.NewObject("test/third")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -293,7 +306,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	// test Remove -- allowed
 	err = runInstance.rm(t, rootFs, "test/one")
 	require.NoError(t, err)
-	_, err = rootFs.NewObject(context.Background(), "test/one")
+	_, err = rootFs.NewObject("test/one")
 	require.Error(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -305,7 +318,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	require.NoError(t, err)
 	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
 	require.NoError(t, err)
-	obj2, err := rootFs.NewObject(context.Background(), "test/one")
+	obj2, err := rootFs.NewObject("test/one")
 	require.NoError(t, err)
 	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
 	require.Equal(t, "one content updated", string(data2))
@@ -328,7 +341,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 
@@ -351,7 +366,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	err = runInstance.dirMove(t, rootFs, "test", "second")
 	if err != errNotSupported {
 		require.Error(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.NoError(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -363,7 +378,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	// test Rmdir
 	err = runInstance.rm(t, rootFs, "test")
 	require.Error(t, err)
-	_, err = rootFs.NewObject(context.Background(), "test/one")
+	_, err = rootFs.NewObject("test/one")
 	require.NoError(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -374,9 +389,9 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	if err != errNotSupported {
 		require.Error(t, err)
 		// try to read from it
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/second")
+		_, err = rootFs.NewObject("test/second")
 		require.Error(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -389,9 +404,9 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/one")
+		_, err = rootFs.NewObject("test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject(context.Background(), "test/third")
+		_, err = rootFs.NewObject("test/third")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -406,7 +421,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	// test Remove
 	err = runInstance.rm(t, rootFs, "test/one")
 	require.Error(t, err)
-	_, err = rootFs.NewObject(context.Background(), "test/one")
+	_, err = rootFs.NewObject("test/one")
 	require.NoError(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))

backend/cache/directory.go

@@ -1,13 +1,12 @@
-//go:build !plan9 && !js
+// +build !plan9
 
 package cache
 
 import (
-	"context"
 	"path"
 	"time"
 
-	"github.com/rclone/rclone/fs"
+	"github.com/ncw/rclone/fs"
 )
 
 // Directory is a generic dir that stores basic information about it
@@ -56,7 +55,7 @@ func ShallowDirectory(f *Fs, remote string) *Directory {
 }
 
 // DirectoryFromOriginal builds one from a generic fs.Directory
-func DirectoryFromOriginal(ctx context.Context, f *Fs, d fs.Directory) *Directory {
+func DirectoryFromOriginal(f *Fs, d fs.Directory) *Directory {
 	var cd *Directory
 	fullRemote := path.Join(f.Root(), d.Remote())
 
@@ -68,7 +67,7 @@ func DirectoryFromOriginal(ctx context.Context, f *Fs, d fs.Directory) *Director
 		CacheFs:      f,
 		Name:         name,
 		Dir:          dir,
-		CacheModTime: d.ModTime(ctx).UnixNano(),
+		CacheModTime: d.ModTime().UnixNano(),
 		CacheSize:    d.Size(),
 		CacheItems:   d.Items(),
 		CacheType:    "Directory",
@@ -101,8 +100,17 @@ func (d *Directory) abs() string {
 	return cleanPath(path.Join(d.Dir, d.Name))
 }
 
+// parentRemote returns the absolute path parent remote
+func (d *Directory) parentRemote() string {
+	absPath := d.abs()
+	if absPath == "" {
+		return ""
+	}
+	return cleanPath(path.Dir(absPath))
+}
+
 // ModTime returns the cached ModTime
-func (d *Directory) ModTime(ctx context.Context) time.Time {
+func (d *Directory) ModTime() time.Time {
 	return time.Unix(0, d.CacheModTime)
 }
 

backend/cache/handle.go

@@ -1,10 +1,8 @@
-//go:build !plan9 && !js
+// +build !plan9
 
 package cache
 
 import (
-	"context"
-	"errors"
 	"fmt"
 	"io"
 	"path"
@@ -13,8 +11,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/operations"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/operations"
+	"github.com/pkg/errors"
 )
 
 var uploaderMap = make(map[string]*backgroundWriter)
@@ -41,7 +40,6 @@ func initBackgroundUploader(fs *Fs) (*backgroundWriter, error) {
 
 // Handle is managing the read/write/seek operations on an open handle
 type Handle struct {
-	ctx            context.Context
 	cachedObject   *Object
 	cfs            *Fs
 	memory         *Memory
@@ -60,9 +58,8 @@ type Handle struct {
 }
 
 // NewObjectHandle returns a new Handle for an existing Object
-func NewObjectHandle(ctx context.Context, o *Object, cfs *Fs) *Handle {
+func NewObjectHandle(o *Object, cfs *Fs) *Handle {
 	r := &Handle{
-		ctx:           ctx,
 		cachedObject:  o,
 		cfs:           cfs,
 		offset:        0,
@@ -118,7 +115,7 @@ func (r *Handle) startReadWorkers() {
 	r.scaleWorkers(totalWorkers)
 }
 
-// scaleWorkers will increase the worker pool count by the provided amount
+// scaleOutWorkers will increase the worker pool count by the provided amount
 func (r *Handle) scaleWorkers(desired int) {
 	current := r.workers
 	if current == desired {
@@ -208,7 +205,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 	offset := chunkStart % int64(r.cacheFs().opt.ChunkSize)
 
 	// we align the start offset of the first chunk to a likely chunk in the storage
-	chunkStart -= offset
+	chunkStart = chunkStart - offset
 	r.queueOffset(chunkStart)
 	found := false
 
@@ -242,7 +239,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 			return nil, io.ErrUnexpectedEOF
 		}
 
-		return nil, fmt.Errorf("chunk not found %v", chunkStart)
+		return nil, errors.Errorf("chunk not found %v", chunkStart)
 	}
 
 	// first chunk will be aligned with the start
@@ -322,12 +319,12 @@ func (r *Handle) Seek(offset int64, whence int) (int64, error) {
 		fs.Debugf(r, "moving offset end (%v) from %v to %v", r.cachedObject.Size(), r.offset, r.cachedObject.Size()+offset)
 		r.offset = r.cachedObject.Size() + offset
 	default:
-		err = fmt.Errorf("cache: unimplemented seek whence %v", whence)
+		err = errors.Errorf("cache: unimplemented seek whence %v", whence)
 	}
 
 	chunkStart := r.offset - (r.offset % int64(r.cacheFs().opt.ChunkSize))
 	if chunkStart >= int64(r.cacheFs().opt.ChunkSize) {
-		chunkStart -= int64(r.cacheFs().opt.ChunkSize)
+		chunkStart = chunkStart - int64(r.cacheFs().opt.ChunkSize)
 	}
 	r.queueOffset(chunkStart)
 
@@ -354,7 +351,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 	r := w.rc
 	if w.rc == nil {
 		r, err = w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) {
-			return w.r.cachedObject.Object.Open(w.r.ctx, &fs.RangeOption{Start: offset, End: end - 1})
+			return w.r.cachedObject.Object.Open(&fs.RangeOption{Start: offset, End: end - 1})
 		})
 		if err != nil {
 			return nil, err
@@ -364,7 +361,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 
 	if !closeOpen {
 		if do, ok := r.(fs.RangeSeeker); ok {
-			_, err = do.RangeSeek(w.r.ctx, offset, io.SeekStart, end-offset)
+			_, err = do.RangeSeek(offset, io.SeekStart, end-offset)
 			return r, err
 		} else if do, ok := r.(io.Seeker); ok {
 			_, err = do.Seek(offset, io.SeekStart)
@@ -374,7 +371,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 
 	_ = w.rc.Close()
 	return w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) {
-		r, err = w.r.cachedObject.Object.Open(w.r.ctx, &fs.RangeOption{Start: offset, End: end - 1})
+		r, err = w.r.cachedObject.Object.Open(&fs.RangeOption{Start: offset, End: end - 1})
 		if err != nil {
 			return nil, err
 		}
@@ -415,9 +412,11 @@ func (w *worker) run() {
 					continue
 				}
 			}
-		} else if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
+		} else {
+			if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
 				continue
 			}
+		}
 
 		chunkEnd := chunkStart + int64(w.r.cacheFs().opt.ChunkSize)
 		// TODO: Remove this comment if it proves to be reliable for #1896
@@ -450,7 +449,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
 	// we seem to be getting only errors so we abort
 	if err != nil {
 		fs.Errorf(w, "object open failed %v: %v", chunkStart, err)
-		err = w.r.cachedObject.refreshFromSource(w.r.ctx, true)
+		err = w.r.cachedObject.refreshFromSource(true)
 		if err != nil {
 			fs.Errorf(w, "%v", err)
 		}
@@ -463,7 +462,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
 	sourceRead, err = io.ReadFull(w.rc, data)
 	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 		fs.Errorf(w, "failed to read chunk %v: %v", chunkStart, err)
-		err = w.r.cachedObject.refreshFromSource(w.r.ctx, true)
+		err = w.r.cachedObject.refreshFromSource(true)
 		if err != nil {
 			fs.Errorf(w, "%v", err)
 		}
@@ -589,7 +588,7 @@ func (b *backgroundWriter) run() {
 		remote := b.fs.cleanRootFromPath(absPath)
 		b.notify(remote, BackgroundUploadStarted, nil)
 		fs.Infof(remote, "background upload: started upload")
-		err = operations.MoveFile(context.TODO(), b.fs.UnWrap(), b.fs.tempFs, remote, remote)
+		err = operations.MoveFile(b.fs.UnWrap(), b.fs.tempFs, remote, remote)
 		if err != nil {
 			b.notify(remote, BackgroundUploadError, err)
 			_ = b.fs.cache.rollbackPendingUpload(absPath)
@@ -599,14 +598,14 @@ func (b *backgroundWriter) run() {
 		// clean empty dirs up to root
 		thisDir := cleanPath(path.Dir(remote))
 		for thisDir != "" {
-			thisList, err := b.fs.tempFs.List(context.TODO(), thisDir)
+			thisList, err := b.fs.tempFs.List(thisDir)
 			if err != nil {
 				break
 			}
 			if len(thisList) > 0 {
 				break
 			}
-			err = b.fs.tempFs.Rmdir(context.TODO(), thisDir)
+			err = b.fs.tempFs.Rmdir(thisDir)
 			fs.Debugf(thisDir, "cleaned from temp path")
 			if err != nil {
 				break

backend/cache/object.go

@@ -1,18 +1,17 @@
-//go:build !plan9 && !js
+// +build !plan9
 
 package cache
 
 import (
-	"context"
-	"fmt"
 	"io"
 	"path"
 	"sync"
 	"time"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/readers"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/readers"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -33,7 +32,6 @@ type Object struct {
 	CacheStorable bool                 `json:"storable"` // says whether this object can be stored
 	CacheType     string               `json:"cacheType"`
 	CacheTs       time.Time            `json:"cacheTs"`
-	cacheHashesMu sync.Mutex
 	CacheHashes   map[hash.Type]string // all supported hashes cached
 
 	refreshMutex sync.Mutex
@@ -70,7 +68,7 @@ func NewObject(f *Fs, remote string) *Object {
 }
 
 // ObjectFromOriginal builds one from a generic fs.Object
-func ObjectFromOriginal(ctx context.Context, f *Fs, o fs.Object) *Object {
+func ObjectFromOriginal(f *Fs, o fs.Object) *Object {
 	var co *Object
 	fullRemote := cleanPath(path.Join(f.Root(), o.Remote()))
 	dir, name := path.Split(fullRemote)
@@ -94,19 +92,17 @@ func ObjectFromOriginal(ctx context.Context, f *Fs, o fs.Object) *Object {
 		CacheType: cacheType,
 		CacheTs:   time.Now(),
 	}
-	co.updateData(ctx, o)
+	co.updateData(o)
 	return co
 }
 
-func (o *Object) updateData(ctx context.Context, source fs.Object) {
+func (o *Object) updateData(source fs.Object) {
 	o.Object = source
-	o.CacheModTime = source.ModTime(ctx).UnixNano()
+	o.CacheModTime = source.ModTime().UnixNano()
 	o.CacheSize = source.Size()
 	o.CacheStorable = source.Storable()
 	o.CacheTs = time.Now()
-	o.cacheHashesMu.Lock()
 	o.CacheHashes = make(map[hash.Type]string)
-	o.cacheHashesMu.Unlock()
 }
 
 // Fs returns its FS info
@@ -134,20 +130,20 @@ func (o *Object) abs() string {
 }
 
 // ModTime returns the cached ModTime
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	_ = o.refresh(ctx)
+func (o *Object) ModTime() time.Time {
+	_ = o.refresh()
 	return time.Unix(0, o.CacheModTime)
 }
 
 // Size returns the cached Size
 func (o *Object) Size() int64 {
-	_ = o.refresh(context.TODO())
+	_ = o.refresh()
 	return o.CacheSize
 }
 
 // Storable returns the cached Storable
 func (o *Object) Storable() bool {
-	_ = o.refresh(context.TODO())
+	_ = o.refresh()
 	return o.CacheStorable
 }
 
@@ -155,18 +151,18 @@ func (o *Object) Storable() bool {
 // all these conditions must be true to ignore a refresh
 // 1. cache ts didn't expire yet
 // 2. is not pending a notification from the wrapped fs
-func (o *Object) refresh(ctx context.Context) error {
+func (o *Object) refresh() error {
 	isNotified := o.CacheFs.isNotifiedRemote(o.Remote())
 	isExpired := time.Now().After(o.CacheTs.Add(time.Duration(o.CacheFs.opt.InfoAge)))
 	if !isExpired && !isNotified {
 		return nil
 	}
 
-	return o.refreshFromSource(ctx, true)
+	return o.refreshFromSource(true)
 }
 
 // refreshFromSource requests the original FS for the object in case it comes from a cached entry
-func (o *Object) refreshFromSource(ctx context.Context, force bool) error {
+func (o *Object) refreshFromSource(force bool) error {
 	o.refreshMutex.Lock()
 	defer o.refreshMutex.Unlock()
 	var err error
@@ -176,33 +172,29 @@ func (o *Object) refreshFromSource(ctx context.Context, force bool) error {
 		return nil
 	}
 	if o.isTempFile() {
-		liveObject, err = o.ParentFs.NewObject(ctx, o.Remote())
-		if err != nil {
-			err = fmt.Errorf("in parent fs %v: %w", o.ParentFs, err)
-		}
+		liveObject, err = o.ParentFs.NewObject(o.Remote())
+		err = errors.Wrapf(err, "in parent fs %v", o.ParentFs)
 	} else {
-		liveObject, err = o.CacheFs.Fs.NewObject(ctx, o.Remote())
-		if err != nil {
-			err = fmt.Errorf("in cache fs %v: %w", o.CacheFs.Fs, err)
-		}
+		liveObject, err = o.CacheFs.Fs.NewObject(o.Remote())
+		err = errors.Wrapf(err, "in cache fs %v", o.CacheFs.Fs)
 	}
 	if err != nil {
 		fs.Errorf(o, "error refreshing object in : %v", err)
 		return err
 	}
-	o.updateData(ctx, liveObject)
+	o.updateData(liveObject)
 	o.persist()
 
 	return nil
 }
 
 // SetModTime sets the ModTime of this object
-func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
-	if err := o.refreshFromSource(ctx, false); err != nil {
+func (o *Object) SetModTime(t time.Time) error {
+	if err := o.refreshFromSource(false); err != nil {
 		return err
 	}
 
-	err := o.Object.SetModTime(ctx, t)
+	err := o.Object.SetModTime(t)
 	if err != nil {
 		return err
 	}
@@ -215,19 +207,19 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
 }
 
 // Open is used to request a specific part of the file using fs.RangeOption
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
 	var err error
 
 	if o.Object == nil {
-		err = o.refreshFromSource(ctx, true)
+		err = o.refreshFromSource(true)
 	} else {
-		err = o.refresh(ctx)
+		err = o.refresh()
 	}
 	if err != nil {
 		return nil, err
 	}
 
-	cacheReader := NewObjectHandle(ctx, o, o.CacheFs)
+	cacheReader := NewObjectHandle(o, o.CacheFs)
 	var offset, limit int64 = 0, -1
 	for _, option := range options {
 		switch x := option.(type) {
@@ -246,8 +238,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 }
 
 // Update will change the object data
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	if err := o.refreshFromSource(ctx, false); err != nil {
+func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	if err := o.refreshFromSource(false); err != nil {
 		return err
 	}
 	// pause background uploads if active
@@ -256,13 +248,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		defer o.CacheFs.backgroundRunner.play()
 		// don't allow started uploads
 		if o.isTempFile() && o.tempFileStartedUpload() {
-			return fmt.Errorf("%v is currently uploading, can't update", o)
+			return errors.Errorf("%v is currently uploading, can't update", o)
 		}
 	}
 	fs.Debugf(o, "updating object contents with size %v", src.Size())
 
 	// FIXME use reliable upload
-	err := o.Object.Update(ctx, in, src, options...)
+	err := o.Object.Update(in, src, options...)
 	if err != nil {
 		fs.Errorf(o, "error updating source: %v", err)
 		return err
@@ -273,11 +265,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// advertise to ChangeNotify if wrapped doesn't do that
 	o.CacheFs.notifyChangeUpstreamIfNeeded(o.Remote(), fs.EntryObject)
 
-	o.CacheModTime = src.ModTime(ctx).UnixNano()
+	o.CacheModTime = src.ModTime().UnixNano()
 	o.CacheSize = src.Size()
-	o.cacheHashesMu.Lock()
 	o.CacheHashes = make(map[hash.Type]string)
-	o.cacheHashesMu.Unlock()
 	o.CacheTs = time.Now()
 	o.persist()
 
@@ -285,8 +275,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 // Remove deletes the object from both the cache and the source
-func (o *Object) Remove(ctx context.Context) error {
-	if err := o.refreshFromSource(ctx, false); err != nil {
+func (o *Object) Remove() error {
+	if err := o.refreshFromSource(false); err != nil {
 		return err
 	}
 	// pause background uploads if active
@@ -295,10 +285,10 @@ func (o *Object) Remove(ctx context.Context) error {
 		defer o.CacheFs.backgroundRunner.play()
 		// don't allow started uploads
 		if o.isTempFile() && o.tempFileStartedUpload() {
-			return fmt.Errorf("%v is currently uploading, can't delete", o)
+			return errors.Errorf("%v is currently uploading, can't delete", o)
 		}
 	}
-	err := o.Object.Remove(ctx)
+	err := o.Object.Remove()
 	if err != nil {
 		return err
 	}
@@ -316,27 +306,24 @@ func (o *Object) Remove(ctx context.Context) error {
 
 // Hash requests a hash of the object and stores in the cache
 // since it might or might not be called, this is lazy loaded
-func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
-	_ = o.refresh(ctx)
-	o.cacheHashesMu.Lock()
+func (o *Object) Hash(ht hash.Type) (string, error) {
+	_ = o.refresh()
 	if o.CacheHashes == nil {
 		o.CacheHashes = make(map[hash.Type]string)
 	}
+
 	cachedHash, found := o.CacheHashes[ht]
-	o.cacheHashesMu.Unlock()
 	if found {
 		return cachedHash, nil
 	}
-	if err := o.refreshFromSource(ctx, false); err != nil {
+	if err := o.refreshFromSource(false); err != nil {
 		return "", err
 	}
-	liveHash, err := o.Object.Hash(ctx, ht)
+	liveHash, err := o.Object.Hash(ht)
 	if err != nil {
 		return "", err
 	}
-	o.cacheHashesMu.Lock()
 	o.CacheHashes[ht] = liveHash
-	o.cacheHashesMu.Unlock()
 
 	o.persist()
 	fs.Debugf(o, "object hash cached: %v", liveHash)

backend/cache/plex.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !js
+// +build !plan9
 
 package cache
 
@@ -7,15 +7,15 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/ncw/rclone/fs"
 	cache "github.com/patrickmn/go-cache"
-	"github.com/rclone/rclone/fs"
 	"golang.org/x/net/websocket"
 )
 
@@ -166,7 +166,7 @@ func (p *plexConnector) listenWebsocket() {
 								continue
 							}
 							var data []byte
-							data, err = io.ReadAll(resp.Body)
+							data, err = ioutil.ReadAll(resp.Body)
 							if err != nil {
 								continue
 							}
@@ -212,7 +212,7 @@ func (p *plexConnector) authenticate() error {
 	var data map[string]interface{}
 	err = json.NewDecoder(resp.Body).Decode(&data)
 	if err != nil {
-		return fmt.Errorf("failed to obtain token: %w", err)
+		return fmt.Errorf("failed to obtain token: %v", err)
 	}
 	tokenGen, ok := get(data, "user", "authToken")
 	if !ok {

backend/cache/storage_memory.go

@@ -1,15 +1,15 @@
-//go:build !plan9 && !js
+// +build !plan9
 
 package cache
 
 import (
-	"fmt"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/ncw/rclone/fs"
 	cache "github.com/patrickmn/go-cache"
-	"github.com/rclone/rclone/fs"
+	"github.com/pkg/errors"
 )
 
 // Memory is a wrapper of transient storage for a go-cache store
@@ -52,7 +52,7 @@ func (m *Memory) GetChunk(cachedObject *Object, offset int64) ([]byte, error) {
 		return data, nil
 	}
 
-	return nil, fmt.Errorf("couldn't get cached object data at offset %v", offset)
+	return nil, errors.Errorf("couldn't get cached object data at offset %v", offset)
 }
 
 // AddChunk adds a new chunk of a cached object
@@ -75,7 +75,10 @@ func (m *Memory) CleanChunksByAge(chunkAge time.Duration) {
 
 // CleanChunksByNeed will cleanup chunks after the FS passes a specific chunk
 func (m *Memory) CleanChunksByNeed(offset int64) {
-	for key := range m.db.Items() {
+	var items map[string]cache.Item
+
+	items = m.db.Items()
+	for key := range items {
 		sepIdx := strings.LastIndex(key, "-")
 		keyOffset, err := strconv.ParseInt(key[sepIdx+1:], 10, 64)
 		if err != nil {

backend/cache/storage_persistent.go

@@ -1,13 +1,13 @@
-//go:build !plan9 && !js
+// +build !plan9
 
 package cache
 
 import (
 	"bytes"
-	"context"
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
@@ -15,9 +15,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/walk"
-	bolt "go.etcd.io/bbolt"
+	bolt "github.com/coreos/bbolt"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/walk"
+	"github.com/pkg/errors"
 )
 
 // Constants
@@ -117,11 +118,11 @@ func (b *Persistent) connect() error {
 
 	err = os.MkdirAll(b.dataPath, os.ModePerm)
 	if err != nil {
-		return fmt.Errorf("failed to create a data directory %q: %w", b.dataPath, err)
+		return errors.Wrapf(err, "failed to create a data directory %q", b.dataPath)
 	}
 	b.db, err = bolt.Open(b.dbPath, 0644, &bolt.Options{Timeout: b.features.DbWaitTime})
 	if err != nil {
-		return fmt.Errorf("failed to open a cache connection to %q: %w", b.dbPath, err)
+		return errors.Wrapf(err, "failed to open a cache connection to %q", b.dbPath)
 	}
 	if b.features.PurgeDb {
 		b.Purge()
@@ -173,7 +174,7 @@ func (b *Persistent) GetDir(remote string) (*Directory, error) {
 	err := b.db.View(func(tx *bolt.Tx) error {
 		bucket := b.getBucket(remote, false, tx)
 		if bucket == nil {
-			return fmt.Errorf("couldn't open bucket (%v)", remote)
+			return errors.Errorf("couldn't open bucket (%v)", remote)
 		}
 
 		data := bucket.Get([]byte("."))
@@ -181,7 +182,7 @@ func (b *Persistent) GetDir(remote string) (*Directory, error) {
 			return json.Unmarshal(data, cd)
 		}
 
-		return fmt.Errorf("%v not found", remote)
+		return errors.Errorf("%v not found", remote)
 	})
 
 	return cd, err
@@ -206,7 +207,7 @@ func (b *Persistent) AddBatchDir(cachedDirs []*Directory) error {
 			bucket = b.getBucket(cachedDirs[0].Dir, true, tx)
 		}
 		if bucket == nil {
-			return fmt.Errorf("couldn't open bucket (%v)", cachedDirs[0].Dir)
+			return errors.Errorf("couldn't open bucket (%v)", cachedDirs[0].Dir)
 		}
 
 		for _, cachedDir := range cachedDirs {
@@ -223,7 +224,7 @@ func (b *Persistent) AddBatchDir(cachedDirs []*Directory) error {
 
 			encoded, err := json.Marshal(cachedDir)
 			if err != nil {
-				return fmt.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
+				return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
 			}
 			err = b.Put([]byte("."), encoded)
 			if err != nil {
@@ -241,17 +242,17 @@ func (b *Persistent) GetDirEntries(cachedDir *Directory) (fs.DirEntries, error)
 	err := b.db.View(func(tx *bolt.Tx) error {
 		bucket := b.getBucket(cachedDir.abs(), false, tx)
 		if bucket == nil {
-			return fmt.Errorf("couldn't open bucket (%v)", cachedDir.abs())
+			return errors.Errorf("couldn't open bucket (%v)", cachedDir.abs())
 		}
 
 		val := bucket.Get([]byte("."))
 		if val != nil {
 			err := json.Unmarshal(val, cachedDir)
 			if err != nil {
-				return fmt.Errorf("error during unmarshalling obj: %w", err)
+				return errors.Errorf("error during unmarshalling obj: %v", err)
 			}
 		} else {
-			return fmt.Errorf("missing cached dir: %v", cachedDir)
+			return errors.Errorf("missing cached dir: %v", cachedDir)
 		}
 
 		c := bucket.Cursor()
@@ -266,7 +267,7 @@ func (b *Persistent) GetDirEntries(cachedDir *Directory) (fs.DirEntries, error)
 				// we try to find a cached meta for the dir
 				currentBucket := c.Bucket().Bucket(k)
 				if currentBucket == nil {
-					return fmt.Errorf("couldn't open bucket (%v)", string(k))
+					return errors.Errorf("couldn't open bucket (%v)", string(k))
 				}
 
 				metaKey := currentBucket.Get([]byte("."))
@@ -315,7 +316,7 @@ func (b *Persistent) RemoveDir(fp string) error {
 		err = b.db.Update(func(tx *bolt.Tx) error {
 			bucket := b.getBucket(cleanPath(parentDir), false, tx)
 			if bucket == nil {
-				return fmt.Errorf("couldn't open bucket (%v)", fp)
+				return errors.Errorf("couldn't open bucket (%v)", fp)
 			}
 			// delete the cached dir
 			err := bucket.DeleteBucket([]byte(cleanPath(dirName)))
@@ -375,13 +376,13 @@ func (b *Persistent) GetObject(cachedObject *Object) (err error) {
 	return b.db.View(func(tx *bolt.Tx) error {
 		bucket := b.getBucket(cachedObject.Dir, false, tx)
 		if bucket == nil {
-			return fmt.Errorf("couldn't open parent bucket for %v", cachedObject.Dir)
+			return errors.Errorf("couldn't open parent bucket for %v", cachedObject.Dir)
 		}
 		val := bucket.Get([]byte(cachedObject.Name))
 		if val != nil {
 			return json.Unmarshal(val, cachedObject)
 		}
-		return fmt.Errorf("couldn't find object (%v)", cachedObject.Name)
+		return errors.Errorf("couldn't find object (%v)", cachedObject.Name)
 	})
 }
 
@@ -390,16 +391,16 @@ func (b *Persistent) AddObject(cachedObject *Object) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		bucket := b.getBucket(cachedObject.Dir, true, tx)
 		if bucket == nil {
-			return fmt.Errorf("couldn't open parent bucket for %v", cachedObject)
+			return errors.Errorf("couldn't open parent bucket for %v", cachedObject)
 		}
 		// cache Object Info
 		encoded, err := json.Marshal(cachedObject)
 		if err != nil {
-			return fmt.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
+			return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
 		}
 		err = bucket.Put([]byte(cachedObject.Name), encoded)
 		if err != nil {
-			return fmt.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
+			return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
 		}
 		return nil
 	})
@@ -411,7 +412,7 @@ func (b *Persistent) RemoveObject(fp string) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		bucket := b.getBucket(cleanPath(parentDir), false, tx)
 		if bucket == nil {
-			return fmt.Errorf("couldn't open parent bucket for %v", cleanPath(parentDir))
+			return errors.Errorf("couldn't open parent bucket for %v", cleanPath(parentDir))
 		}
 		err := bucket.Delete([]byte(cleanPath(objName)))
 		if err != nil {
@@ -443,7 +444,7 @@ func (b *Persistent) HasEntry(remote string) bool {
 	err := b.db.View(func(tx *bolt.Tx) error {
 		bucket := b.getBucket(dir, false, tx)
 		if bucket == nil {
-			return fmt.Errorf("couldn't open parent bucket for %v", remote)
+			return errors.Errorf("couldn't open parent bucket for %v", remote)
 		}
 		if f := bucket.Bucket([]byte(name)); f != nil {
 			return nil
@@ -452,9 +453,12 @@ func (b *Persistent) HasEntry(remote string) bool {
 			return nil
 		}
 
-		return fmt.Errorf("couldn't find object (%v)", remote)
+		return errors.Errorf("couldn't find object (%v)", remote)
 	})
-	return err == nil
+	if err == nil {
+		return true
+	}
+	return false
 }
 
 // HasChunk confirms the existence of a single chunk of an object
@@ -471,7 +475,7 @@ func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error
 	var data []byte
 
 	fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10))
-	data, err := os.ReadFile(fp)
+	data, err := ioutil.ReadFile(fp)
 	if err != nil {
 		return nil, err
 	}
@@ -484,7 +488,7 @@ func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error {
 	_ = os.MkdirAll(path.Join(b.dataPath, fp), os.ModePerm)
 
 	filePath := path.Join(b.dataPath, fp, strconv.FormatInt(offset, 10))
-	err := os.WriteFile(filePath, data, os.ModePerm)
+	err := ioutil.WriteFile(filePath, data, os.ModePerm)
 	if err != nil {
 		return err
 	}
@@ -549,7 +553,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
 	err := b.db.Update(func(tx *bolt.Tx) error {
 		dataTsBucket := tx.Bucket([]byte(DataTsBucket))
 		if dataTsBucket == nil {
-			return fmt.Errorf("couldn't open (%v) bucket", DataTsBucket)
+			return errors.Errorf("Couldn't open (%v) bucket", DataTsBucket)
 		}
 		// iterate through ts
 		c := dataTsBucket.Cursor()
@@ -727,7 +731,7 @@ func (b *Persistent) GetChunkTs(path string, offset int64) (time.Time, error) {
 				return nil
 			}
 		}
-		return fmt.Errorf("not found %v-%v", path, offset)
+		return errors.Errorf("not found %v-%v", path, offset)
 	})
 
 	return t, err
@@ -762,12 +766,37 @@ func (b *Persistent) iterateBuckets(buk *bolt.Bucket, bucketFn func(name string)
 	return err
 }
 
+func (b *Persistent) dumpRoot() string {
+	var itBuckets func(buk *bolt.Bucket) map[string]interface{}
+
+	itBuckets = func(buk *bolt.Bucket) map[string]interface{} {
+		m := make(map[string]interface{})
+		c := buk.Cursor()
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			if v == nil {
+				buk2 := buk.Bucket(k)
+				m[string(k)] = itBuckets(buk2)
+			} else {
+				m[string(k)] = "-"
+			}
+		}
+		return m
+	}
+	var mm map[string]interface{}
+	_ = b.db.View(func(tx *bolt.Tx) error {
+		mm = itBuckets(tx.Bucket([]byte(RootBucket)))
+		return nil
+	})
+	raw, _ := json.MarshalIndent(mm, "", "  ")
+	return string(raw)
+}
+
 // addPendingUpload adds a new file to the pending queue of uploads
 func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket))
 		if err != nil {
-			return fmt.Errorf("couldn't bucket for %v", tempBucket)
+			return errors.Errorf("couldn't bucket for %v", tempBucket)
 		}
 		tempObj := &tempUploadInfo{
 			DestPath: destPath,
@@ -778,11 +807,11 @@ func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 		// cache Object Info
 		encoded, err := json.Marshal(tempObj)
 		if err != nil {
-			return fmt.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
+			return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
 		}
 		err = bucket.Put([]byte(destPath), encoded)
 		if err != nil {
-			return fmt.Errorf("couldn't cache object (%v) info: %v", destPath, err)
+			return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 		}
 
 		return nil
@@ -797,7 +826,7 @@ func (b *Persistent) getPendingUpload(inRoot string, waitTime time.Duration) (de
 	err = b.db.Update(func(tx *bolt.Tx) error {
 		bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket))
 		if err != nil {
-			return fmt.Errorf("couldn't bucket for %v", tempBucket)
+			return errors.Errorf("couldn't bucket for %v", tempBucket)
 		}
 
 		c := bucket.Cursor()
@@ -830,7 +859,7 @@ func (b *Persistent) getPendingUpload(inRoot string, waitTime time.Duration) (de
 			return nil
 		}
 
-		return fmt.Errorf("no pending upload found")
+		return errors.Errorf("no pending upload found")
 	})
 
 	return destPath, err
@@ -841,14 +870,14 @@ func (b *Persistent) SearchPendingUpload(remote string) (started bool, err error
 	err = b.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(tempBucket))
 		if bucket == nil {
-			return fmt.Errorf("couldn't bucket for %v", tempBucket)
+			return errors.Errorf("couldn't bucket for %v", tempBucket)
 		}
 
 		var tempObj = &tempUploadInfo{}
 		v := bucket.Get([]byte(remote))
 		err = json.Unmarshal(v, tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload (%v) not found %v", remote, err)
+			return errors.Errorf("pending upload (%v) not found %v", remote, err)
 		}
 
 		started = tempObj.Started
@@ -863,7 +892,7 @@ func (b *Persistent) searchPendingUploadFromDir(dir string) (remotes []string, e
 	err = b.db.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(tempBucket))
 		if bucket == nil {
-			return fmt.Errorf("couldn't bucket for %v", tempBucket)
+			return errors.Errorf("couldn't bucket for %v", tempBucket)
 		}
 
 		c := bucket.Cursor()
@@ -893,22 +922,22 @@ func (b *Persistent) rollbackPendingUpload(remote string) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket))
 		if err != nil {
-			return fmt.Errorf("couldn't bucket for %v", tempBucket)
+			return errors.Errorf("couldn't bucket for %v", tempBucket)
 		}
 		var tempObj = &tempUploadInfo{}
 		v := bucket.Get([]byte(remote))
 		err = json.Unmarshal(v, tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload (%v) not found: %w", remote, err)
+			return errors.Errorf("pending upload (%v) not found %v", remote, err)
 		}
 		tempObj.Started = false
 		v2, err := json.Marshal(tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated: %w", err)
+			return errors.Errorf("pending upload not updated %v", err)
 		}
 		err = bucket.Put([]byte(tempObj.DestPath), v2)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated: %w", err)
+			return errors.Errorf("pending upload not updated %v", err)
 		}
 		return nil
 	})
@@ -921,7 +950,7 @@ func (b *Persistent) removePendingUpload(remote string) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket))
 		if err != nil {
-			return fmt.Errorf("couldn't bucket for %v", tempBucket)
+			return errors.Errorf("couldn't bucket for %v", tempBucket)
 		}
 		return bucket.Delete([]byte(remote))
 	})
@@ -936,17 +965,17 @@ func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUpload
 	return b.db.Update(func(tx *bolt.Tx) error {
 		bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket))
 		if err != nil {
-			return fmt.Errorf("couldn't bucket for %v", tempBucket)
+			return errors.Errorf("couldn't bucket for %v", tempBucket)
 		}
 
 		var tempObj = &tempUploadInfo{}
 		v := bucket.Get([]byte(remote))
 		err = json.Unmarshal(v, tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload (%v) not found %v", remote, err)
+			return errors.Errorf("pending upload (%v) not found %v", remote, err)
 		}
 		if tempObj.Started {
-			return fmt.Errorf("pending upload already started %v", remote)
+			return errors.Errorf("pending upload already started %v", remote)
 		}
 		err = fn(tempObj)
 		if err != nil {
@@ -964,19 +993,28 @@ func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUpload
 		}
 		v2, err := json.Marshal(tempObj)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated: %w", err)
+			return errors.Errorf("pending upload not updated %v", err)
 		}
 		err = bucket.Put([]byte(tempObj.DestPath), v2)
 		if err != nil {
-			return fmt.Errorf("pending upload not updated: %w", err)
+			return errors.Errorf("pending upload not updated %v", err)
 		}
 
 		return nil
 	})
 }
 
+// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already)
+// TO BE USED IN TESTING ONLY
+func (b *Persistent) SetPendingUploadToStarted(remote string) error {
+	return b.updatePendingUpload(remote, func(item *tempUploadInfo) error {
+		item.Started = true
+		return nil
+	})
+}
+
 // ReconcileTempUploads will recursively look for all the files in the temp directory and add them to the queue
-func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) error {
+func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		_ = tx.DeleteBucket([]byte(tempBucket))
 		bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket))
@@ -985,7 +1023,7 @@ func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) erro
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.ListR(ctx, cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)
@@ -1009,11 +1047,11 @@ func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) erro
 			// cache Object Info
 			encoded, err := json.Marshal(tempObj)
 			if err != nil {
-				return fmt.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
+				return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
 			}
 			err = bucket.Put([]byte(destPath), encoded)
 			if err != nil {
-				return fmt.Errorf("couldn't cache object (%v) info: %v", destPath, err)
+				return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 			}
 			fs.Debugf(cacheFs, "reconciled temporary upload: %v", destPath)
 		}
@@ -1022,6 +1060,19 @@ func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) erro
 	})
 }
 
+// PurgeTempUploads will remove all the pending uploads from the queue
+// TO BE USED IN TESTING ONLY
+func (b *Persistent) PurgeTempUploads() {
+	b.tempQueueMux.Lock()
+	defer b.tempQueueMux.Unlock()
+
+	_ = b.db.Update(func(tx *bolt.Tx) error {
+		_ = tx.DeleteBucket([]byte(tempBucket))
+		_, _ = tx.CreateBucketIfNotExists([]byte(tempBucket))
+		return nil
+	})
+}
+
 // Close should be called when the program ends gracefully
 func (b *Persistent) Close() {
 	b.cleanupMux.Lock()

backend/cache/utils_test.go

@@ -1,26 +0,0 @@
-//go:build !plan9 && !js
-// +build !plan9,!js
-
-package cache
-
-import bolt "go.etcd.io/bbolt"
-
-// PurgeTempUploads will remove all the pending uploads from the queue
-func (b *Persistent) PurgeTempUploads() {
-	b.tempQueueMux.Lock()
-	defer b.tempQueueMux.Unlock()
-
-	_ = b.db.Update(func(tx *bolt.Tx) error {
-		_ = tx.DeleteBucket([]byte(tempBucket))
-		_, _ = tx.CreateBucketIfNotExists([]byte(tempBucket))
-		return nil
-	})
-}
-
-// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already)
-func (b *Persistent) SetPendingUploadToStarted(remote string) error {
-	return b.updatePendingUpload(remote, func(item *tempUploadInfo) error {
-		item.Started = true
-		return nil
-	})
-}

backend/chunker/chunker_internal_test.go

@@ -1,939 +0,0 @@
-package chunker
-
-import (
-	"bytes"
-	"context"
-	"flag"
-	"fmt"
-	"io"
-	"path"
-	"regexp"
-	"strings"
-	"testing"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/fspath"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/object"
-	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// Command line flags
-var (
-	UploadKilobytes = flag.Int("upload-kilobytes", 0, "Upload size in Kilobytes, set this to test large uploads")
-)
-
-// test that chunking does not break large uploads
-func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
-	t.Run(fmt.Sprintf("PutLarge%dk", kilobytes), func(t *testing.T) {
-		fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
-			ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
-			Path:    fmt.Sprintf("chunker-upload-%dk", kilobytes),
-			Size:    int64(kilobytes) * int64(fs.Kibi),
-		})
-	})
-}
-
-type settings map[string]interface{}
-
-func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
-	fsName := strings.Split(f.Name(), "{")[0] // strip off hash
-	configMap := configmap.Simple{}
-	for key, val := range opts {
-		configMap[key] = fmt.Sprintf("%v", val)
-	}
-	rpath := fspath.JoinRootPath(f.Root(), path)
-	remote := fmt.Sprintf("%s,%s:%s", fsName, configMap.String(), rpath)
-	fixFs, err := fs.NewFs(ctx, remote)
-	require.NoError(t, err)
-	return fixFs
-}
-
-var mtime1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
-
-func testPutFile(ctx context.Context, t *testing.T, f fs.Fs, name, contents, message string, check bool) fs.Object {
-	item := fstest.Item{Path: name, ModTime: mtime1}
-	obj := fstests.PutTestContents(ctx, t, f, &item, contents, check)
-	assert.NotNil(t, obj, message)
-	return obj
-}
-
-// test chunk name parser
-func testChunkNameFormat(t *testing.T, f *Fs) {
-	saveOpt := f.opt
-	defer func() {
-		// restore original settings (f is pointer, f.opt is struct)
-		f.opt = saveOpt
-		_ = f.setChunkNameFormat(f.opt.NameFormat)
-	}()
-
-	assertFormat := func(pattern, wantDataFormat, wantCtrlFormat, wantNameRegexp string) {
-		err := f.setChunkNameFormat(pattern)
-		assert.NoError(t, err)
-		assert.Equal(t, wantDataFormat, f.dataNameFmt)
-		assert.Equal(t, wantCtrlFormat, f.ctrlNameFmt)
-		assert.Equal(t, wantNameRegexp, f.nameRegexp.String())
-	}
-
-	assertFormatValid := func(pattern string) {
-		err := f.setChunkNameFormat(pattern)
-		assert.NoError(t, err)
-	}
-
-	assertFormatInvalid := func(pattern string) {
-		err := f.setChunkNameFormat(pattern)
-		assert.Error(t, err)
-	}
-
-	assertMakeName := func(wantChunkName, mainName string, chunkNo int, ctrlType, xactID string) {
-		gotChunkName := ""
-		assert.NotPanics(t, func() {
-			gotChunkName = f.makeChunkName(mainName, chunkNo, ctrlType, xactID)
-		}, "makeChunkName(%q,%d,%q,%q) must not panic", mainName, chunkNo, ctrlType, xactID)
-		if gotChunkName != "" {
-			assert.Equal(t, wantChunkName, gotChunkName)
-		}
-	}
-
-	assertMakeNamePanics := func(mainName string, chunkNo int, ctrlType, xactID string) {
-		assert.Panics(t, func() {
-			_ = f.makeChunkName(mainName, chunkNo, ctrlType, xactID)
-		}, "makeChunkName(%q,%d,%q,%q) should panic", mainName, chunkNo, ctrlType, xactID)
-	}
-
-	assertParseName := func(fileName, wantMainName string, wantChunkNo int, wantCtrlType, wantXactID string) {
-		gotMainName, gotChunkNo, gotCtrlType, gotXactID := f.parseChunkName(fileName)
-		assert.Equal(t, wantMainName, gotMainName)
-		assert.Equal(t, wantChunkNo, gotChunkNo)
-		assert.Equal(t, wantCtrlType, gotCtrlType)
-		assert.Equal(t, wantXactID, gotXactID)
-	}
-
-	const newFormatSupported = false // support for patterns not starting with base name (*)
-
-	// valid formats
-	assertFormat(`*.rclone_chunk.###`, `%s.rclone_chunk.%03d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	assertFormat(`*.rclone_chunk.#`, `%s.rclone_chunk.%d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	assertFormat(`*_chunk_#####`, `%s_chunk_%05d`, `%s_chunk__%s`, `^(.+?)_chunk_(?:([0-9]{5,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	assertFormat(`*-chunk-#`, `%s-chunk-%d`, `%s-chunk-_%s`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	assertFormat(`*-chunk-#-%^$()[]{}.+-!?:\`, `%s-chunk-%d-%%^$()[]{}.+-!?:\`, `%s-chunk-_%s-%%^$()[]{}.+-!?:\`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))-%\^\$\(\)\[\]\{\}\.\+-!\?:\\(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	if newFormatSupported {
-		assertFormat(`_*-chunk-##,`, `_%s-chunk-%02d,`, `_%s-chunk-_%s,`, `^_(.+?)-chunk-(?:([0-9]{2,})|_([a-z][a-z0-9]{2,6})),(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	}
-
-	// invalid formats
-	assertFormatInvalid(`chunk-#`)
-	assertFormatInvalid(`*-chunk`)
-	assertFormatInvalid(`*-*-chunk-#`)
-	assertFormatInvalid(`*-chunk-#-#`)
-	assertFormatInvalid(`#-chunk-*`)
-	assertFormatInvalid(`*/#`)
-
-	assertFormatValid(`*#`)
-	assertFormatInvalid(`**#`)
-	assertFormatInvalid(`#*`)
-	assertFormatInvalid(``)
-	assertFormatInvalid(`-`)
-
-	// quick tests
-	if newFormatSupported {
-		assertFormat(`part_*_#`, `part_%s_%d`, `part_%s__%s`, `^part_(.+?)_(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9][0-9a-z]{3,8})\.\.tmp_([0-9]{10,13}))?$`)
-		f.opt.StartFrom = 1
-
-		assertMakeName(`part_fish_1`, "fish", 0, "", "")
-		assertParseName(`part_fish_43`, "fish", 42, "", "")
-		assertMakeName(`part_fish__locks`, "fish", -2, "locks", "")
-		assertParseName(`part_fish__locks`, "fish", -1, "locks", "")
-		assertMakeName(`part_fish__x2y`, "fish", -2, "x2y", "")
-		assertParseName(`part_fish__x2y`, "fish", -1, "x2y", "")
-		assertMakeName(`part_fish_3_0004`, "fish", 2, "", "4")
-		assertParseName(`part_fish_4_0005`, "fish", 3, "", "0005")
-		assertMakeName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -3, "blkinfo", "jj5fvo3wr")
-		assertParseName(`part_fish__blkinfo_zz9fvo3wr`, "fish", -1, "blkinfo", "zz9fvo3wr")
-
-		// old-style temporary suffix (parse only)
-		assertParseName(`part_fish_4..tmp_0000000011`, "fish", 3, "", "000b")
-		assertParseName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -1, "blkinfo", "jj5fvo3wr")
-	}
-
-	// prepare format for long tests
-	assertFormat(`*.chunk.###`, `%s.chunk.%03d`, `%s.chunk._%s`, `^(.+?)\.chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
-	f.opt.StartFrom = 2
-
-	// valid data chunks
-	assertMakeName(`fish.chunk.003`, "fish", 1, "", "")
-	assertParseName(`fish.chunk.003`, "fish", 1, "", "")
-	assertMakeName(`fish.chunk.021`, "fish", 19, "", "")
-	assertParseName(`fish.chunk.021`, "fish", 19, "", "")
-
-	// valid temporary data chunks
-	assertMakeName(`fish.chunk.011_4321`, "fish", 9, "", "4321")
-	assertParseName(`fish.chunk.011_4321`, "fish", 9, "", "4321")
-	assertMakeName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc")
-	assertParseName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc")
-	assertMakeName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr")
-	assertParseName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr")
-	assertMakeName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr")
-	assertParseName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr")
-
-	// valid temporary data chunks (old temporary suffix, only parse)
-	assertParseName(`fish.chunk.004..tmp_0000000047`, "fish", 2, "", "001b")
-	assertParseName(`fish.chunk.323..tmp_9994567890123`, "fish", 321, "", "3jjfvo3wr")
-
-	// parsing invalid data chunk names
-	assertParseName(`fish.chunk.3`, "", -1, "", "")
-	assertParseName(`fish.chunk.001`, "", -1, "", "")
-	assertParseName(`fish.chunk.21`, "", -1, "", "")
-	assertParseName(`fish.chunk.-21`, "", -1, "", "")
-
-	assertParseName(`fish.chunk.004abcd`, "", -1, "", "")        // missing underscore delimiter
-	assertParseName(`fish.chunk.004__1234`, "", -1, "", "")      // extra underscore delimiter
-	assertParseName(`fish.chunk.004_123`, "", -1, "", "")        // too short temporary suffix
-	assertParseName(`fish.chunk.004_1234567890`, "", -1, "", "") // too long temporary suffix
-	assertParseName(`fish.chunk.004_-1234`, "", -1, "", "")      // temporary suffix must be positive
-	assertParseName(`fish.chunk.004_123E`, "", -1, "", "")       // uppercase not allowed
-	assertParseName(`fish.chunk.004_12.3`, "", -1, "", "")       // punctuation not allowed
-
-	// parsing invalid data chunk names (old temporary suffix)
-	assertParseName(`fish.chunk.004.tmp_0000000021`, "", -1, "", "")
-	assertParseName(`fish.chunk.003..tmp_123456789`, "", -1, "", "")
-	assertParseName(`fish.chunk.003..tmp_012345678901234567890123456789`, "", -1, "", "")
-	assertParseName(`fish.chunk.323..tmp_12345678901234`, "", -1, "", "")
-	assertParseName(`fish.chunk.003..tmp_-1`, "", -1, "", "")
-
-	// valid control chunks
-	assertMakeName(`fish.chunk._info`, "fish", -1, "info", "")
-	assertMakeName(`fish.chunk._locks`, "fish", -2, "locks", "")
-	assertMakeName(`fish.chunk._blkinfo`, "fish", -3, "blkinfo", "")
-	assertMakeName(`fish.chunk._x2y`, "fish", -4, "x2y", "")
-
-	assertParseName(`fish.chunk._info`, "fish", -1, "info", "")
-	assertParseName(`fish.chunk._locks`, "fish", -1, "locks", "")
-	assertParseName(`fish.chunk._blkinfo`, "fish", -1, "blkinfo", "")
-	assertParseName(`fish.chunk._x2y`, "fish", -1, "x2y", "")
-
-	// valid temporary control chunks
-	assertMakeName(`fish.chunk._info_0001`, "fish", -1, "info", "1")
-	assertMakeName(`fish.chunk._locks_4321`, "fish", -2, "locks", "4321")
-	assertMakeName(`fish.chunk._uploads_abcd`, "fish", -3, "uploads", "abcd")
-	assertMakeName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -4, "blkinfo", "xyzabcdef")
-	assertMakeName(`fish.chunk._x2y_1aaa`, "fish", -5, "x2y", "1aaa")
-
-	assertParseName(`fish.chunk._info_0001`, "fish", -1, "info", "0001")
-	assertParseName(`fish.chunk._locks_4321`, "fish", -1, "locks", "4321")
-	assertParseName(`fish.chunk._uploads_9abc`, "fish", -1, "uploads", "9abc")
-	assertParseName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -1, "blkinfo", "xyzabcdef")
-	assertParseName(`fish.chunk._x2y_1aaa`, "fish", -1, "x2y", "1aaa")
-
-	// valid temporary control chunks (old temporary suffix, parse only)
-	assertParseName(`fish.chunk._info..tmp_0000000047`, "fish", -1, "info", "001b")
-	assertParseName(`fish.chunk._locks..tmp_0000054321`, "fish", -1, "locks", "15wx")
-	assertParseName(`fish.chunk._uploads..tmp_0000000000`, "fish", -1, "uploads", "0000")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123`, "fish", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk._x2y..tmp_0000000000`, "fish", -1, "x2y", "0000")
-
-	// parsing invalid control chunk names
-	assertParseName(`fish.chunk.metadata`, "", -1, "", "") // must be prepended by underscore
-	assertParseName(`fish.chunk.info`, "", -1, "", "")
-	assertParseName(`fish.chunk.locks`, "", -1, "", "")
-	assertParseName(`fish.chunk.uploads`, "", -1, "", "")
-
-	assertParseName(`fish.chunk._os`, "", -1, "", "")        // too short
-	assertParseName(`fish.chunk._metadata`, "", -1, "", "")  // too long
-	assertParseName(`fish.chunk._blockinfo`, "", -1, "", "") // way too long
-	assertParseName(`fish.chunk._4me`, "", -1, "", "")       // cannot start with digit
-	assertParseName(`fish.chunk._567`, "", -1, "", "")       // cannot be all digits
-	assertParseName(`fish.chunk._me_ta`, "", -1, "", "")     // punctuation not allowed
-	assertParseName(`fish.chunk._in-fo`, "", -1, "", "")
-	assertParseName(`fish.chunk._.bin`, "", -1, "", "")
-	assertParseName(`fish.chunk._.2xy`, "", -1, "", "")
-
-	// parsing invalid temporary control chunks
-	assertParseName(`fish.chunk._blkinfo1234`, "", -1, "", "")     // missing underscore delimiter
-	assertParseName(`fish.chunk._info__1234`, "", -1, "", "")      // extra underscore delimiter
-	assertParseName(`fish.chunk._info_123`, "", -1, "", "")        // too short temporary suffix
-	assertParseName(`fish.chunk._info_1234567890`, "", -1, "", "") // too long temporary suffix
-	assertParseName(`fish.chunk._info_-1234`, "", -1, "", "")      // temporary suffix must be positive
-	assertParseName(`fish.chunk._info_123E`, "", -1, "", "")       // uppercase not allowed
-	assertParseName(`fish.chunk._info_12.3`, "", -1, "", "")       // punctuation not allowed
-
-	assertParseName(`fish.chunk._locks..tmp_123456789`, "", -1, "", "")
-	assertParseName(`fish.chunk._meta..tmp_-1`, "", -1, "", "")
-	assertParseName(`fish.chunk._blockinfo..tmp_012345678901234567890123456789`, "", -1, "", "")
-
-	// short control chunk names: 3 letters ok, 1-2 letters not allowed
-	assertMakeName(`fish.chunk._ext`, "fish", -1, "ext", "")
-	assertParseName(`fish.chunk._int`, "fish", -1, "int", "")
-
-	assertMakeNamePanics("fish", -1, "in", "")
-	assertMakeNamePanics("fish", -1, "up", "4")
-	assertMakeNamePanics("fish", -1, "x", "")
-	assertMakeNamePanics("fish", -1, "c", "1z")
-
-	assertMakeName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0")
-	assertMakeName(`fish.chunk._ext_0026`, "fish", -1, "ext", "26")
-	assertMakeName(`fish.chunk._int_0abc`, "fish", -1, "int", "abc")
-	assertMakeName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz")
-	assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
-	assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
-
-	assertParseName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0000")
-	assertParseName(`fish.chunk._ext_0026`, "fish", -1, "ext", "0026")
-	assertParseName(`fish.chunk._int_0abc`, "fish", -1, "int", "0abc")
-	assertParseName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz")
-	assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
-	assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
-
-	// base file name can sometimes look like a valid chunk name
-	assertParseName(`fish.chunk.003.chunk.004`, "fish.chunk.003", 2, "", "")
-	assertParseName(`fish.chunk.003.chunk._info`, "fish.chunk.003", -1, "info", "")
-	assertParseName(`fish.chunk.003.chunk._Meta`, "", -1, "", "")
-
-	assertParseName(`fish.chunk._info.chunk.004`, "fish.chunk._info", 2, "", "")
-	assertParseName(`fish.chunk._info.chunk._info`, "fish.chunk._info", -1, "info", "")
-	assertParseName(`fish.chunk._info.chunk._info.chunk._Meta`, "", -1, "", "")
-
-	// base file name looking like a valid chunk name (old temporary suffix)
-	assertParseName(`fish.chunk.003.chunk.005..tmp_0000000022`, "fish.chunk.003", 3, "", "000m")
-	assertParseName(`fish.chunk.003.chunk._x..tmp_0000054321`, "", -1, "", "")
-	assertParseName(`fish.chunk._info.chunk.005..tmp_0000000023`, "fish.chunk._info", 3, "", "000n")
-	assertParseName(`fish.chunk._info.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
-
-	assertParseName(`fish.chunk.003.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.003", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk._info.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._info", -1, "blkinfo", "3jjfvo3wr")
-
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk.004`, "fish.chunk.004..tmp_0000000021", 2, "", "")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk.005..tmp_0000000025`, "fish.chunk.004..tmp_0000000021", 3, "", "000p")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._info`, "fish.chunk.004..tmp_0000000021", -1, "info", "")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.004..tmp_0000000021", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._Meta`, "", -1, "", "")
-	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._x..tmp_0000054321`, "", -1, "", "")
-
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.004`, "fish.chunk._blkinfo..tmp_9994567890123", 2, "", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.005..tmp_0000000026`, "fish.chunk._blkinfo..tmp_9994567890123", 3, "", "000q")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "info", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._Meta`, "", -1, "", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
-
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.004`, "fish.chunk._blkinfo..tmp_1234567890123456789", 2, "", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.005..tmp_0000000022`, "fish.chunk._blkinfo..tmp_1234567890123456789", 3, "", "000m")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "info", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "blkinfo", "3jjfvo3wr")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._Meta`, "", -1, "", "")
-	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
-
-	// attempts to make invalid chunk names
-	assertMakeNamePanics("fish", -1, "", "")          // neither data nor control
-	assertMakeNamePanics("fish", 0, "info", "")       // both data and control
-	assertMakeNamePanics("fish", -1, "metadata", "")  // control type too long
-	assertMakeNamePanics("fish", -1, "blockinfo", "") // control type way too long
-	assertMakeNamePanics("fish", -1, "2xy", "")       // first digit not allowed
-	assertMakeNamePanics("fish", -1, "123", "")       // all digits not allowed
-	assertMakeNamePanics("fish", -1, "Meta", "")      // only lower case letters allowed
-	assertMakeNamePanics("fish", -1, "in-fo", "")     // punctuation not allowed
-	assertMakeNamePanics("fish", -1, "_info", "")
-	assertMakeNamePanics("fish", -1, "info_", "")
-	assertMakeNamePanics("fish", -2, ".bind", "")
-	assertMakeNamePanics("fish", -2, "bind.", "")
-
-	assertMakeNamePanics("fish", -1, "", "1")          // neither data nor control
-	assertMakeNamePanics("fish", 0, "info", "23")      // both data and control
-	assertMakeNamePanics("fish", -1, "metadata", "45") // control type too long
-	assertMakeNamePanics("fish", -1, "blockinfo", "7") // control type way too long
-	assertMakeNamePanics("fish", -1, "2xy", "abc")     // first digit not allowed
-	assertMakeNamePanics("fish", -1, "123", "def")     // all digits not allowed
-	assertMakeNamePanics("fish", -1, "Meta", "mnk")    // only lower case letters allowed
-	assertMakeNamePanics("fish", -1, "in-fo", "xyz")   // punctuation not allowed
-	assertMakeNamePanics("fish", -1, "_info", "5678")
-	assertMakeNamePanics("fish", -1, "info_", "999")
-	assertMakeNamePanics("fish", -2, ".bind", "0")
-	assertMakeNamePanics("fish", -2, "bind.", "0")
-
-	assertMakeNamePanics("fish", 0, "", "1234567890") // temporary suffix too long
-	assertMakeNamePanics("fish", 0, "", "123F4")      // uppercase not allowed
-	assertMakeNamePanics("fish", 0, "", "123.")       // punctuation not allowed
-	assertMakeNamePanics("fish", 0, "", "_123")
-}
-
-func testSmallFileInternals(t *testing.T, f *Fs) {
-	const dir = "small"
-	ctx := context.Background()
-	saveOpt := f.opt
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-	f.opt.FailHard = false
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-
-	checkSmallFileInternals := func(obj fs.Object) {
-		assert.NotNil(t, obj)
-		o, ok := obj.(*Object)
-		assert.True(t, ok)
-		assert.NotNil(t, o)
-		if o == nil {
-			return
-		}
-		switch {
-		case !f.useMeta:
-			// If meta format is "none", non-chunked file (even empty)
-			// internally is a single chunk without meta object.
-			assert.Nil(t, o.main)
-			assert.True(t, o.isComposite()) // sorry, sometimes a name is misleading
-			assert.Equal(t, 1, len(o.chunks))
-		case f.hashAll:
-			// Consistent hashing forces meta object on small files too
-			assert.NotNil(t, o.main)
-			assert.True(t, o.isComposite())
-			assert.Equal(t, 1, len(o.chunks))
-		default:
-			// normally non-chunked file is kept in the Object's main field
-			assert.NotNil(t, o.main)
-			assert.False(t, o.isComposite())
-			assert.Equal(t, 0, len(o.chunks))
-		}
-	}
-
-	checkContents := func(obj fs.Object, contents string) {
-		assert.NotNil(t, obj)
-		assert.Equal(t, int64(len(contents)), obj.Size())
-
-		r, err := obj.Open(ctx)
-		assert.NoError(t, err)
-		assert.NotNil(t, r)
-		if r == nil {
-			return
-		}
-		data, err := io.ReadAll(r)
-		assert.NoError(t, err)
-		assert.Equal(t, contents, string(data))
-		_ = r.Close()
-	}
-
-	checkHashsum := func(obj fs.Object) {
-		var ht hash.Type
-		switch {
-		case !f.hashAll:
-			return
-		case f.useMD5:
-			ht = hash.MD5
-		case f.useSHA1:
-			ht = hash.SHA1
-		default:
-			return
-		}
-		// even empty files must have hashsum in consistent mode
-		sum, err := obj.Hash(ctx, ht)
-		assert.NoError(t, err)
-		assert.NotEqual(t, sum, "")
-	}
-
-	checkSmallFile := func(name, contents string) {
-		filename := path.Join(dir, name)
-		item := fstest.Item{Path: filename, ModTime: modTime}
-		put := fstests.PutTestContents(ctx, t, f, &item, contents, false)
-		assert.NotNil(t, put)
-		checkSmallFileInternals(put)
-		checkContents(put, contents)
-		checkHashsum(put)
-
-		// objects returned by Put and NewObject must have similar structure
-		obj, err := f.NewObject(ctx, filename)
-		assert.NoError(t, err)
-		assert.NotNil(t, obj)
-		checkSmallFileInternals(obj)
-		checkContents(obj, contents)
-		checkHashsum(obj)
-
-		_ = obj.Remove(ctx)
-		_ = put.Remove(ctx) // for good
-	}
-
-	checkSmallFile("emptyfile", "")
-	checkSmallFile("smallfile", "Ok")
-}
-
-func testPreventCorruption(t *testing.T, f *Fs) {
-	if f.opt.ChunkSize > 50 {
-		t.Skip("this test requires small chunks")
-	}
-	const dir = "corrupted"
-	ctx := context.Background()
-	saveOpt := f.opt
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-	f.opt.FailHard = true
-
-	contents := random.String(250)
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	const overlapMessage = "chunk overlap"
-
-	assertOverlapError := func(err error) {
-		assert.Error(t, err)
-		if err != nil {
-			assert.Contains(t, err.Error(), overlapMessage)
-		}
-	}
-
-	newFile := func(name string) fs.Object {
-		item := fstest.Item{Path: path.Join(dir, name), ModTime: modTime}
-		obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
-		require.NotNil(t, obj)
-		return obj
-	}
-	billyObj := newFile("billy")
-	billyTxn := billyObj.(*Object).xactID
-	if f.useNoRename {
-		require.True(t, billyTxn != "")
-	} else {
-		require.True(t, billyTxn == "")
-	}
-
-	billyChunkName := func(chunkNo int) string {
-		return f.makeChunkName(billyObj.Remote(), chunkNo, "", billyTxn)
-	}
-
-	err := f.Mkdir(ctx, billyChunkName(1))
-	assertOverlapError(err)
-
-	_, err = f.Move(ctx, newFile("silly1"), billyChunkName(2))
-	assert.Error(t, err)
-	assert.True(t, err == fs.ErrorCantMove || (err != nil && strings.Contains(err.Error(), overlapMessage)))
-
-	_, err = f.Copy(ctx, newFile("silly2"), billyChunkName(3))
-	assert.Error(t, err)
-	assert.True(t, err == fs.ErrorCantCopy || (err != nil && strings.Contains(err.Error(), overlapMessage)))
-
-	// accessing chunks in strict mode is prohibited
-	f.opt.FailHard = true
-	billyChunk4Name := billyChunkName(4)
-	_, err = f.base.NewObject(ctx, billyChunk4Name)
-	require.NoError(t, err)
-	_, err = f.NewObject(ctx, billyChunk4Name)
-	assertOverlapError(err)
-
-	f.opt.FailHard = false
-	billyChunk4, err := f.NewObject(ctx, billyChunk4Name)
-	assert.NoError(t, err)
-	require.NotNil(t, billyChunk4)
-
-	f.opt.FailHard = true
-	_, err = f.Put(ctx, bytes.NewBufferString(contents), billyChunk4)
-	assertOverlapError(err)
-
-	// you can freely read chunks (if you have an object)
-	r, err := billyChunk4.Open(ctx)
-	assert.NoError(t, err)
-	var chunkContents []byte
-	assert.NotPanics(t, func() {
-		chunkContents, err = io.ReadAll(r)
-		_ = r.Close()
-	})
-	assert.NoError(t, err)
-	assert.NotEqual(t, contents, string(chunkContents))
-
-	// but you can't change them
-	err = billyChunk4.Update(ctx, bytes.NewBufferString(contents), newFile("silly3"))
-	assertOverlapError(err)
-
-	// Remove isn't special, you can't corrupt files even if you have an object
-	err = billyChunk4.Remove(ctx)
-	assertOverlapError(err)
-
-	// recreate billy in case it was anyhow corrupted
-	willyObj := newFile("willy")
-	willyTxn := willyObj.(*Object).xactID
-	willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", willyTxn)
-	f.opt.FailHard = false
-	willyChunk, err := f.NewObject(ctx, willyChunkName)
-	f.opt.FailHard = true
-	assert.NoError(t, err)
-	require.NotNil(t, willyChunk)
-
-	_, err = operations.Copy(ctx, f, willyChunk, willyChunkName, newFile("silly4"))
-	assertOverlapError(err)
-
-	// operations.Move will return error when chunker's Move refused
-	// to corrupt target file, but reverts to copy/delete method
-	// still trying to delete target chunk. Chunker must come to rescue.
-	_, err = operations.Move(ctx, f, willyChunk, willyChunkName, newFile("silly5"))
-	assertOverlapError(err)
-	r, err = willyChunk.Open(ctx)
-	assert.NoError(t, err)
-	assert.NotPanics(t, func() {
-		_, err = io.ReadAll(r)
-		_ = r.Close()
-	})
-	assert.NoError(t, err)
-}
-
-func testChunkNumberOverflow(t *testing.T, f *Fs) {
-	if f.opt.ChunkSize > 50 {
-		t.Skip("this test requires small chunks")
-	}
-	const dir = "wreaked"
-	const wreakNumber = 10200300
-	ctx := context.Background()
-	saveOpt := f.opt
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	contents := random.String(100)
-
-	newFile := func(f fs.Fs, name string) (obj fs.Object, filename string, txnID string) {
-		filename = path.Join(dir, name)
-		item := fstest.Item{Path: filename, ModTime: modTime}
-		obj = fstests.PutTestContents(ctx, t, f, &item, contents, true)
-		require.NotNil(t, obj)
-		if chunkObj, isChunkObj := obj.(*Object); isChunkObj {
-			txnID = chunkObj.xactID
-		}
-		return
-	}
-
-	f.opt.FailHard = false
-	file, fileName, fileTxn := newFile(f, "wreaker")
-	wreak, _, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", fileTxn))
-
-	f.opt.FailHard = false
-	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
-	_, err := f.NewObject(ctx, fileName)
-	assert.Error(t, err)
-
-	f.opt.FailHard = true
-	_, err = f.List(ctx, dir)
-	assert.Error(t, err)
-	_, err = f.NewObject(ctx, fileName)
-	assert.Error(t, err)
-
-	f.opt.FailHard = false
-	_ = wreak.Remove(ctx)
-	_ = file.Remove(ctx)
-}
-
-func testMetadataInput(t *testing.T, f *Fs) {
-	const minChunkForTest = 50
-	if f.opt.ChunkSize < minChunkForTest {
-		t.Skip("this test requires chunks that fit metadata")
-	}
-
-	const dir = "usermeta"
-	ctx := context.Background()
-	saveOpt := f.opt
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-	f.opt.FailHard = false
-
-	runSubtest := func(contents, name string) {
-		description := fmt.Sprintf("file with %s metadata", name)
-		filename := path.Join(dir, name)
-		require.True(t, len(contents) > 2 && len(contents) < minChunkForTest, description+" test data is correct")
-
-		part := testPutFile(ctx, t, f.base, f.makeChunkName(filename, 0, "", ""), "oops", "", true)
-		_ = testPutFile(ctx, t, f, filename, contents, "upload "+description, false)
-
-		obj, err := f.NewObject(ctx, filename)
-		assert.NoError(t, err, "access "+description)
-		assert.NotNil(t, obj)
-		assert.Equal(t, int64(len(contents)), obj.Size(), "size "+description)
-
-		o, ok := obj.(*Object)
-		assert.NotNil(t, ok)
-		if o != nil {
-			assert.True(t, o.isComposite() && len(o.chunks) == 1, description+" is forced composite")
-			o = nil
-		}
-
-		defer func() {
-			_ = obj.Remove(ctx)
-			_ = part.Remove(ctx)
-		}()
-
-		r, err := obj.Open(ctx)
-		assert.NoError(t, err, "open "+description)
-		assert.NotNil(t, r, "open stream of "+description)
-		if err == nil && r != nil {
-			data, err := io.ReadAll(r)
-			assert.NoError(t, err, "read all of "+description)
-			assert.Equal(t, contents, string(data), description+" contents is ok")
-			_ = r.Close()
-		}
-	}
-
-	metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "", "")
-	require.NoError(t, err)
-	todaysMeta := string(metaData)
-	runSubtest(todaysMeta, "today")
-
-	pastMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":1`)
-	pastMeta = regexp.MustCompile(`"size":[0-9]+`).ReplaceAllLiteralString(pastMeta, `"size":0`)
-	runSubtest(pastMeta, "past")
-
-	futureMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":999`)
-	futureMeta = regexp.MustCompile(`"nchunks":[0-9]+`).ReplaceAllLiteralString(futureMeta, `"nchunks":0,"x":"y"`)
-	runSubtest(futureMeta, "future")
-}
-
-// Test that chunker refuses to change on objects with future/unknown metadata
-func testFutureProof(t *testing.T, f *Fs) {
-	if !f.useMeta {
-		t.Skip("this test requires metadata support")
-	}
-
-	saveOpt := f.opt
-	ctx := context.Background()
-	f.opt.FailHard = true
-	const dir = "future"
-	const file = dir + "/test"
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-	}()
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	putPart := func(name string, part int, data, msg string) {
-		if part > 0 {
-			name = f.makeChunkName(name, part-1, "", "")
-		}
-		item := fstest.Item{Path: name, ModTime: modTime}
-		obj := fstests.PutTestContents(ctx, t, f.base, &item, data, true)
-		assert.NotNil(t, obj, msg)
-	}
-
-	// simulate chunked object from future
-	meta := `{"ver":999,"nchunks":3,"size":9,"garbage":"litter","sha1":"0707f2970043f9f7c22029482db27733deaec029"}`
-	putPart(file, 0, meta, "metaobject")
-	putPart(file, 1, "abc", "chunk1")
-	putPart(file, 2, "def", "chunk2")
-	putPart(file, 3, "ghi", "chunk3")
-
-	// List should succeed
-	ls, err := f.List(ctx, dir)
-	assert.NoError(t, err)
-	assert.Equal(t, 1, len(ls))
-	assert.Equal(t, int64(9), ls[0].Size())
-
-	// NewObject should succeed
-	obj, err := f.NewObject(ctx, file)
-	assert.NoError(t, err)
-	assert.Equal(t, file, obj.Remote())
-	assert.Equal(t, int64(9), obj.Size())
-
-	// Hash must fail
-	_, err = obj.Hash(ctx, hash.SHA1)
-	assert.Equal(t, ErrMetaUnknown, err)
-
-	// Move must fail
-	mobj, err := operations.Move(ctx, f, nil, file+"2", obj)
-	assert.Nil(t, mobj)
-	assert.Error(t, err)
-	if err != nil {
-		assert.Contains(t, err.Error(), "please upgrade rclone")
-	}
-
-	// Put must fail
-	oi := object.NewStaticObjectInfo(file, modTime, 3, true, nil, nil)
-	buf := bytes.NewBufferString("abc")
-	_, err = f.Put(ctx, buf, oi)
-	assert.Error(t, err)
-
-	// Rcat must fail
-	in := io.NopCloser(bytes.NewBufferString("abc"))
-	robj, err := operations.Rcat(ctx, f, file, in, modTime, nil)
-	assert.Nil(t, robj)
-	assert.NotNil(t, err)
-	if err != nil {
-		assert.Contains(t, err.Error(), "please upgrade rclone")
-	}
-}
-
-// The newer method of doing transactions without renaming should still be able to correctly process chunks that were created with renaming
-// If you attempt to do the inverse, however, the data chunks will be ignored causing commands to perform incorrectly
-func testBackwardsCompatibility(t *testing.T, f *Fs) {
-	if !f.useMeta {
-		t.Skip("Can't do norename transactions without metadata")
-	}
-	const dir = "backcomp"
-	ctx := context.Background()
-	saveOpt := f.opt
-	saveUseNoRename := f.useNoRename
-	defer func() {
-		f.opt.FailHard = false
-		_ = operations.Purge(ctx, f.base, dir)
-		f.opt = saveOpt
-		f.useNoRename = saveUseNoRename
-	}()
-	f.opt.ChunkSize = fs.SizeSuffix(10)
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	contents := random.String(250)
-	newFile := func(f fs.Fs, name string) (fs.Object, string) {
-		filename := path.Join(dir, name)
-		item := fstest.Item{Path: filename, ModTime: modTime}
-		obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
-		require.NotNil(t, obj)
-		return obj, filename
-	}
-
-	f.opt.FailHard = false
-	f.useNoRename = false
-	file, fileName := newFile(f, "renamefile")
-
-	f.opt.FailHard = false
-	item := fstest.NewItem(fileName, contents, modTime)
-
-	var items []fstest.Item
-	items = append(items, item)
-
-	f.useNoRename = true
-	fstest.CheckListingWithRoot(t, f, dir, items, nil, f.Precision())
-	_, err := f.NewObject(ctx, fileName)
-	assert.NoError(t, err)
-
-	f.opt.FailHard = true
-	_, err = f.List(ctx, dir)
-	assert.NoError(t, err)
-
-	f.opt.FailHard = false
-	_ = file.Remove(ctx)
-}
-
-func testChunkerServerSideMove(t *testing.T, f *Fs) {
-	if !f.useMeta {
-		t.Skip("Can't test norename transactions without metadata")
-	}
-
-	ctx := context.Background()
-	const dir = "servermovetest"
-	subRemote := fmt.Sprintf("%s:%s/%s", f.Name(), f.Root(), dir)
-
-	subFs1, err := fs.NewFs(ctx, subRemote+"/subdir1")
-	assert.NoError(t, err)
-	fs1, isChunkerFs := subFs1.(*Fs)
-	assert.True(t, isChunkerFs)
-	fs1.useNoRename = false
-	fs1.opt.ChunkSize = fs.SizeSuffix(3)
-
-	subFs2, err := fs.NewFs(ctx, subRemote+"/subdir2")
-	assert.NoError(t, err)
-	fs2, isChunkerFs := subFs2.(*Fs)
-	assert.True(t, isChunkerFs)
-	fs2.useNoRename = true
-	fs2.opt.ChunkSize = fs.SizeSuffix(3)
-
-	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
-	item := fstest.Item{Path: "movefile", ModTime: modTime}
-	contents := "abcdef"
-	file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true)
-
-	dstOverwritten, _ := fs2.NewObject(ctx, "movefile")
-	dstFile, err := operations.Move(ctx, fs2, dstOverwritten, "movefile", file)
-	assert.NoError(t, err)
-	assert.Equal(t, int64(len(contents)), dstFile.Size())
-
-	r, err := dstFile.Open(ctx)
-	assert.NoError(t, err)
-	assert.NotNil(t, r)
-	data, err := io.ReadAll(r)
-	assert.NoError(t, err)
-	assert.Equal(t, contents, string(data))
-	_ = r.Close()
-	_ = operations.Purge(ctx, f.base, dir)
-}
-
-// Test that md5all creates metadata even for small files
-func testMD5AllSlow(t *testing.T, f *Fs) {
-	ctx := context.Background()
-	fsResult := deriveFs(ctx, t, f, "md5all", settings{
-		"chunk_size":   "1P",
-		"name_format":  "*.#",
-		"hash_type":    "md5all",
-		"transactions": "rename",
-		"meta_format":  "simplejson",
-	})
-	chunkFs, ok := fsResult.(*Fs)
-	require.True(t, ok, "fs must be a chunker remote")
-	baseFs := chunkFs.base
-	if !baseFs.Features().SlowHash {
-		t.Skipf("this test needs a base fs with slow hash, e.g. local")
-	}
-
-	assert.True(t, chunkFs.useMD5, "must use md5")
-	assert.True(t, chunkFs.hashAll, "must hash all files")
-
-	_ = testPutFile(ctx, t, chunkFs, "file", "-", "error", true)
-	obj, err := chunkFs.NewObject(ctx, "file")
-	require.NoError(t, err)
-	sum, err := obj.Hash(ctx, hash.MD5)
-	assert.NoError(t, err)
-	assert.Equal(t, "336d5ebc5436534e61d16e63ddfca327", sum)
-
-	list, err := baseFs.List(ctx, "")
-	require.NoError(t, err)
-	assert.Equal(t, 2, len(list))
-	_, err = baseFs.NewObject(ctx, "file")
-	assert.NoError(t, err, "metadata must be created")
-	_, err = baseFs.NewObject(ctx, "file.1")
-	assert.NoError(t, err, "first chunk must be created")
-
-	require.NoError(t, operations.Purge(ctx, baseFs, ""))
-}
-
-// InternalTest dispatches all internal tests
-func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("PutLarge", func(t *testing.T) {
-		if *UploadKilobytes <= 0 {
-			t.Skip("-upload-kilobytes is not set")
-		}
-		testPutLarge(t, f, *UploadKilobytes)
-	})
-	t.Run("ChunkNameFormat", func(t *testing.T) {
-		testChunkNameFormat(t, f)
-	})
-	t.Run("SmallFileInternals", func(t *testing.T) {
-		testSmallFileInternals(t, f)
-	})
-	t.Run("PreventCorruption", func(t *testing.T) {
-		testPreventCorruption(t, f)
-	})
-	t.Run("ChunkNumberOverflow", func(t *testing.T) {
-		testChunkNumberOverflow(t, f)
-	})
-	t.Run("MetadataInput", func(t *testing.T) {
-		testMetadataInput(t, f)
-	})
-	t.Run("FutureProof", func(t *testing.T) {
-		testFutureProof(t, f)
-	})
-	t.Run("BackwardsCompatibility", func(t *testing.T) {
-		testBackwardsCompatibility(t, f)
-	})
-	t.Run("ChunkerServerSideMove", func(t *testing.T) {
-		testChunkerServerSideMove(t, f)
-	})
-	t.Run("MD5AllSlow", func(t *testing.T) {
-		testMD5AllSlow(t, f)
-	})
-}
-
-var _ fstests.InternalTester = (*Fs)(nil)

backend/chunker/chunker_test.go

@@ -1,62 +0,0 @@
-// Test the Chunker filesystem interface
-package chunker_test
-
-import (
-	"flag"
-	"os"
-	"path/filepath"
-	"testing"
-
-	_ "github.com/rclone/rclone/backend/all" // for integration tests
-	"github.com/rclone/rclone/backend/chunker"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// Command line flags
-var (
-	// Invalid characters are not supported by some remotes, e.g. Mailru.
-	// We enable testing with invalid characters when -remote is not set, so
-	// chunker overlays a local directory, but invalid characters are disabled
-	// by default when -remote is set, e.g. when test_all runs backend tests.
-	// You can still test with invalid characters using the below flag.
-	UseBadChars = flag.Bool("bad-chars", false, "Set to test bad characters in file names when -remote is set")
-)
-
-// TestIntegration runs integration tests against a concrete remote
-// set by the -remote flag. If the flag is not set, it creates a
-// dynamic chunker overlay wrapping a local temporary directory.
-func TestIntegration(t *testing.T) {
-	opt := fstests.Opt{
-		RemoteName:               *fstest.RemoteName,
-		NilObject:                (*chunker.Object)(nil),
-		SkipBadWindowsCharacters: !*UseBadChars,
-		UnimplementableObjectMethods: []string{
-			"MimeType",
-			"GetTier",
-			"SetTier",
-			"Metadata",
-			"SetMetadata",
-		},
-		UnimplementableFsMethods: []string{
-			"PublicLink",
-			"OpenWriterAt",
-			"OpenChunkWriter",
-			"MergeDirs",
-			"DirCacheFlush",
-			"UserInfo",
-			"Disconnect",
-		},
-	}
-	if *fstest.RemoteName == "" {
-		name := "TestChunker"
-		opt.RemoteName = name + ":"
-		tempDir := filepath.Join(os.TempDir(), "rclone-chunker-test-standard")
-		opt.ExtraConfig = []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "chunker"},
-			{Name: name, Key: "remote", Value: tempDir},
-		}
-		opt.QuickTestOK = true
-	}
-	fstests.Run(t, &opt)
-}

backend/combine/combine_internal_test.go

@@ -1,94 +0,0 @@
-package combine
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestAdjustmentDo(t *testing.T) {
-	for _, test := range []struct {
-		root       string
-		mountpoint string
-		in         string
-		want       string
-		wantErr    error
-	}{
-		{
-			root:       "",
-			mountpoint: "mountpoint",
-			in:         "path/to/file.txt",
-			want:       "mountpoint/path/to/file.txt",
-		},
-		{
-			root:       "mountpoint",
-			mountpoint: "mountpoint",
-			in:         "path/to/file.txt",
-			want:       "path/to/file.txt",
-		},
-		{
-			root:       "mountpoint/path",
-			mountpoint: "mountpoint",
-			in:         "path/to/file.txt",
-			want:       "to/file.txt",
-		},
-		{
-			root:       "mountpoint/path",
-			mountpoint: "mountpoint",
-			in:         "wrongpath/to/file.txt",
-			want:       "",
-			wantErr:    errNotUnderRoot,
-		},
-	} {
-		what := fmt.Sprintf("%+v", test)
-		a := newAdjustment(test.root, test.mountpoint)
-		got, gotErr := a.do(test.in)
-		assert.Equal(t, test.wantErr, gotErr)
-		assert.Equal(t, test.want, got, what)
-	}
-
-}
-
-func TestAdjustmentUndo(t *testing.T) {
-	for _, test := range []struct {
-		root       string
-		mountpoint string
-		in         string
-		want       string
-		wantErr    error
-	}{
-		{
-			root:       "",
-			mountpoint: "mountpoint",
-			in:         "mountpoint/path/to/file.txt",
-			want:       "path/to/file.txt",
-		},
-		{
-			root:       "mountpoint",
-			mountpoint: "mountpoint",
-			in:         "path/to/file.txt",
-			want:       "path/to/file.txt",
-		},
-		{
-			root:       "mountpoint/path",
-			mountpoint: "mountpoint",
-			in:         "to/file.txt",
-			want:       "path/to/file.txt",
-		},
-		{
-			root:       "wrongmountpoint/path",
-			mountpoint: "mountpoint",
-			in:         "to/file.txt",
-			want:       "",
-			wantErr:    errNotUnderRoot,
-		},
-	} {
-		what := fmt.Sprintf("%+v", test)
-		a := newAdjustment(test.root, test.mountpoint)
-		got, gotErr := a.undo(test.in)
-		assert.Equal(t, test.wantErr, gotErr)
-		assert.Equal(t, test.want, got, what)
-	}
-
-}

backend/combine/combine_test.go

@@ -1,92 +0,0 @@
-// Test Combine filesystem interface
-package combine_test
-
-import (
-	"testing"
-
-	_ "github.com/rclone/rclone/backend/local"
-	_ "github.com/rclone/rclone/backend/memory"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-var (
-	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "OpenChunkWriter"}
-	unimplementableObjectMethods = []string{}
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	if *fstest.RemoteName == "" {
-		t.Skip("Skipping as -remote not set")
-	}
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   *fstest.RemoteName,
-		UnimplementableFsMethods:     unimplementableFsMethods,
-		UnimplementableObjectMethods: unimplementableObjectMethods,
-	})
-}
-
-func TestLocal(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	dirs := MakeTestDirs(t, 3)
-	upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=" + dirs[2]
-	name := "TestCombineLocal"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":dir1",
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "combine"},
-			{Name: name, Key: "upstreams", Value: upstreams},
-		},
-		QuickTestOK:                  true,
-		UnimplementableFsMethods:     unimplementableFsMethods,
-		UnimplementableObjectMethods: unimplementableObjectMethods,
-	})
-}
-
-func TestMemory(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	upstreams := "dir1=:memory:dir1 dir2=:memory:dir2 dir3=:memory:dir3"
-	name := "TestCombineMemory"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":dir1",
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "combine"},
-			{Name: name, Key: "upstreams", Value: upstreams},
-		},
-		QuickTestOK:                  true,
-		UnimplementableFsMethods:     unimplementableFsMethods,
-		UnimplementableObjectMethods: unimplementableObjectMethods,
-	})
-}
-
-func TestMixed(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	dirs := MakeTestDirs(t, 2)
-	upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=:memory:dir3"
-	name := "TestCombineMixed"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":dir1",
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "combine"},
-			{Name: name, Key: "upstreams", Value: upstreams},
-		},
-		UnimplementableFsMethods:     unimplementableFsMethods,
-		UnimplementableObjectMethods: unimplementableObjectMethods,
-	})
-}
-
-// MakeTestDirs makes directories in /tmp for testing
-func MakeTestDirs(t *testing.T, n int) (dirs []string) {
-	for i := 1; i <= n; i++ {
-		dir := t.TempDir()
-		dirs = append(dirs, dir)
-	}
-	return dirs
-}

backend/compress/.gitignore

@@ -1 +0,0 @@
-test

backend/compress/compress_test.go

@@ -1,55 +0,0 @@
-// Test Crypt filesystem interface
-package compress
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	_ "github.com/rclone/rclone/backend/drive"
-	_ "github.com/rclone/rclone/backend/local"
-	_ "github.com/rclone/rclone/backend/s3"
-	_ "github.com/rclone/rclone/backend/swift"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-var defaultOpt = fstests.Opt{
-	RemoteName: "TestCompress:",
-	NilObject:  (*Object)(nil),
-	UnimplementableFsMethods: []string{
-		"OpenWriterAt",
-		"OpenChunkWriter",
-		"MergeDirs",
-		"DirCacheFlush",
-		"PutUnchecked",
-		"PutStream",
-		"UserInfo",
-		"Disconnect",
-	},
-	TiersToTest:                  []string{"STANDARD", "STANDARD_IA"},
-	UnimplementableObjectMethods: []string{},
-}
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &defaultOpt)
-}
-
-// TestRemoteGzip tests GZIP compression
-func TestRemoteGzip(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	tempdir := filepath.Join(os.TempDir(), "rclone-compress-test-gzip")
-	name := "TestCompressGzip"
-	opt := defaultOpt
-	opt.RemoteName = name + ":"
-	opt.ExtraConfig = []fstests.ExtraConfigItem{
-		{Name: name, Key: "type", Value: "compress"},
-		{Name: name, Key: "remote", Value: tempdir},
-		{Name: name, Key: "compression_mode", Value: "gzip"},
-	}
-	opt.QuickTestOK = true
-	fstests.Run(t, &opt)
-}

backend/crypt/cipher.go

@@ -2,27 +2,21 @@ package crypt
 
 import (
 	"bytes"
-	"context"
 	"crypto/aes"
 	gocipher "crypto/cipher"
 	"crypto/rand"
 	"encoding/base32"
-	"encoding/base64"
-	"errors"
 	"fmt"
 	"io"
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 	"unicode/utf8"
 
-	"github.com/Max-Sum/base32768"
-	"github.com/rclone/rclone/backend/crypt/pkcs7"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/lib/readers"
-	"github.com/rclone/rclone/lib/version"
+	"github.com/ncw/rclone/backend/crypt/pkcs7"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/accounting"
+	"github.com/pkg/errors"
 	"github.com/rfjakob/eme"
 	"golang.org/x/crypto/nacl/secretbox"
 	"golang.org/x/crypto/scrypt"
@@ -38,6 +32,7 @@ const (
 	blockHeaderSize     = secretbox.Overhead
 	blockDataSize       = 64 * 1024
 	blockSize           = blockHeaderSize + blockDataSize
+	encryptedSuffix     = ".bin" // when file name encryption is off we add this suffix to make sure the cloud provider doesn't process the file
 )
 
 // Errors returned by cipher
@@ -53,9 +48,8 @@ var (
 	ErrorEncryptedBadBlock       = errors.New("failed to authenticate decrypted block - bad password?")
 	ErrorBadBase32Encoding       = errors.New("bad base32 filename encoding")
 	ErrorFileClosed              = errors.New("file already closed")
-	ErrorNotAnEncryptedFile      = errors.New("not an encrypted file - does not match suffix")
+	ErrorNotAnEncryptedFile      = errors.New("not an encrypted file - no \"" + encryptedSuffix + "\" suffix")
 	ErrorBadSeek                 = errors.New("Seek beyond end of file")
-	ErrorSuffixMissingDot        = errors.New("suffix config setting should include a '.'")
 	defaultSalt                  = []byte{0xA8, 0x0D, 0xF4, 0x3A, 0x8F, 0xBD, 0x03, 0x08, 0xA7, 0xCA, 0xB8, 0x3E, 0x58, 0x1F, 0x86, 0xB1}
 	obfuscQuoteRune              = '!'
 )
@@ -74,7 +68,31 @@ type ReadSeekCloser interface {
 }
 
 // OpenRangeSeek opens the file handle at the offset with the limit given
-type OpenRangeSeek func(ctx context.Context, offset, limit int64) (io.ReadCloser, error)
+type OpenRangeSeek func(offset, limit int64) (io.ReadCloser, error)
+
+// Cipher is used to swap out the encryption implementations
+type Cipher interface {
+	// EncryptFileName encrypts a file path
+	EncryptFileName(string) string
+	// DecryptFileName decrypts a file path, returns error if decrypt was invalid
+	DecryptFileName(string) (string, error)
+	// EncryptDirName encrypts a directory path
+	EncryptDirName(string) string
+	// DecryptDirName decrypts a directory path, returns error if decrypt was invalid
+	DecryptDirName(string) (string, error)
+	// EncryptData
+	EncryptData(io.Reader) (io.Reader, error)
+	// DecryptData
+	DecryptData(io.ReadCloser) (io.ReadCloser, error)
+	// DecryptDataSeek decrypt at a given position
+	DecryptDataSeek(open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error)
+	// EncryptedSize calculates the size of the data when encrypted
+	EncryptedSize(int64) int64
+	// DecryptedSize calculates the size of the data when decrypted
+	DecryptedSize(int64) (int64, error)
+	// NameEncryptionMode returns the used mode for name handling
+	NameEncryptionMode() NameEncryptionMode
+}
 
 // NameEncryptionMode is the type of file name encryption in use
 type NameEncryptionMode int
@@ -97,12 +115,12 @@ func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) {
 	case "obfuscate":
 		mode = NameEncryptionObfuscated
 	default:
-		err = fmt.Errorf("unknown file name encryption mode %q", s)
+		err = errors.Errorf("Unknown file name encryption mode %q", s)
 	}
 	return mode, err
 }
 
-// String turns mode into a human-readable string
+// String turns mode into a human readable string
 func (mode NameEncryptionMode) String() (out string) {
 	switch mode {
 	case NameEncryptionOff:
@@ -117,83 +135,26 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
-// fileNameEncoding are the encoding methods dealing with encrypted file names
-type fileNameEncoding interface {
-	EncodeToString(src []byte) string
-	DecodeString(s string) ([]byte, error)
-}
-
-// caseInsensitiveBase32Encoding defines a file name encoding
-// using a modified version of standard base32 as described in
-// RFC4648
-//
-// The standard encoding is modified in two ways
-//   - it becomes lower case (no-one likes upper case filenames!)
-//   - we strip the padding character `=`
-type caseInsensitiveBase32Encoding struct{}
-
-// EncodeToString encodes a string using the modified version of
-// base32 encoding.
-func (caseInsensitiveBase32Encoding) EncodeToString(src []byte) string {
-	encoded := base32.HexEncoding.EncodeToString(src)
-	encoded = strings.TrimRight(encoded, "=")
-	return strings.ToLower(encoded)
-}
-
-// DecodeString decodes a string as encoded by EncodeToString
-func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
-	if strings.HasSuffix(s, "=") {
-		return nil, ErrorBadBase32Encoding
-	}
-	// First figure out how many padding characters to add
-	roundUpToMultipleOf8 := (len(s) + 7) &^ 7
-	equals := roundUpToMultipleOf8 - len(s)
-	s = strings.ToUpper(s) + "========"[:equals]
-	return base32.HexEncoding.DecodeString(s)
-}
-
-// NewNameEncoding creates a NameEncoding from a string
-func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
-	s = strings.ToLower(s)
-	switch s {
-	case "base32":
-		enc = caseInsensitiveBase32Encoding{}
-	case "base64":
-		enc = base64.RawURLEncoding
-	case "base32768":
-		enc = base32768.SafeEncoding
-	default:
-		err = fmt.Errorf("unknown file name encoding mode %q", s)
-	}
-	return enc, err
-}
-
-// Cipher defines an encoding and decoding cipher for the crypt backend
-type Cipher struct {
+type cipher struct {
 	dataKey        [32]byte                  // Key for secretbox
 	nameKey        [32]byte                  // 16,24 or 32 bytes
 	nameTweak      [nameCipherBlockSize]byte // used to tweak the name crypto
 	block          gocipher.Block
 	mode           NameEncryptionMode
-	fileNameEnc     fileNameEncoding
 	buffers        sync.Pool // encrypt/decrypt buffers
 	cryptoRand     io.Reader // read crypto random numbers from here
 	dirNameEncrypt bool
-	passBadBlocks   bool // if set passed bad blocks as zeroed blocks
-	encryptedSuffix string
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
-	c := &Cipher{
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*cipher, error) {
+	c := &cipher{
 		mode:           mode,
-		fileNameEnc:     enc,
 		cryptoRand:     rand.Reader,
 		dirNameEncrypt: dirNameEncrypt,
-		encryptedSuffix: ".bin",
 	}
 	c.buffers.New = func() interface{} {
-		return new([blockSize]byte)
+		return make([]byte, blockSize)
 	}
 	err := c.Key(password, salt)
 	if err != nil {
@@ -202,33 +163,15 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 	return c, nil
 }
 
-// setEncryptedSuffix set suffix, or an empty string
-func (c *Cipher) setEncryptedSuffix(suffix string) {
-	if strings.EqualFold(suffix, "none") {
-		c.encryptedSuffix = ""
-		return
-	}
-	if !strings.HasPrefix(suffix, ".") {
-		fs.Errorf(nil, "crypt: bad suffix: %v", ErrorSuffixMissingDot)
-		suffix = "." + suffix
-	}
-	c.encryptedSuffix = suffix
-}
-
-// Call to set bad block pass through
-func (c *Cipher) setPassBadBlocks(passBadBlocks bool) {
-	c.passBadBlocks = passBadBlocks
-}
-
 // Key creates all the internal keys from the password passed in using
 // scrypt.
 //
 // If salt is "" we use a fixed salt just to make attackers lives
-// slightly harder than using no salt.
+// slighty harder than using no salt.
 //
-// Note that empty password makes all 0x00 keys which is used in the
+// Note that empty passsword makes all 0x00 keys which is used in the
 // tests.
-func (c *Cipher) Key(password, salt string) (err error) {
+func (c *cipher) Key(password, salt string) (err error) {
 	const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak)
 	var saltBytes = defaultSalt
 	if salt != "" {
@@ -252,44 +195,86 @@ func (c *Cipher) Key(password, salt string) (err error) {
 }
 
 // getBlock gets a block from the pool of size blockSize
-func (c *Cipher) getBlock() *[blockSize]byte {
-	return c.buffers.Get().(*[blockSize]byte)
+func (c *cipher) getBlock() []byte {
+	return c.buffers.Get().([]byte)
 }
 
 // putBlock returns a block to the pool of size blockSize
-func (c *Cipher) putBlock(buf *[blockSize]byte) {
+func (c *cipher) putBlock(buf []byte) {
+	if len(buf) != blockSize {
+		panic("bad blocksize returned to pool")
+	}
 	c.buffers.Put(buf)
 }
 
+// check to see if the byte string is valid with no control characters
+// from 0x00 to 0x1F and is a valid UTF-8 string
+func checkValidString(buf []byte) error {
+	for i := range buf {
+		c := buf[i]
+		if c >= 0x00 && c < 0x20 || c == 0x7F {
+			return ErrorBadDecryptControlChar
+		}
+	}
+	if !utf8.Valid(buf) {
+		return ErrorBadDecryptUTF8
+	}
+	return nil
+}
+
+// encodeFileName encodes a filename using a modified version of
+// standard base32 as described in RFC4648
+//
+// The standard encoding is modified in two ways
+//  * it becomes lower case (no-one likes upper case filenames!)
+//  * we strip the padding character `=`
+func encodeFileName(in []byte) string {
+	encoded := base32.HexEncoding.EncodeToString(in)
+	encoded = strings.TrimRight(encoded, "=")
+	return strings.ToLower(encoded)
+}
+
+// decodeFileName decodes a filename as encoded by encodeFileName
+func decodeFileName(in string) ([]byte, error) {
+	if strings.HasSuffix(in, "=") {
+		return nil, ErrorBadBase32Encoding
+	}
+	// First figure out how many padding characters to add
+	roundUpToMultipleOf8 := (len(in) + 7) &^ 7
+	equals := roundUpToMultipleOf8 - len(in)
+	in = strings.ToUpper(in) + "========"[:equals]
+	return base32.HexEncoding.DecodeString(in)
+}
+
 // encryptSegment encrypts a path segment
 //
-// This uses EME with AES.
+// This uses EME with AES
 //
 // EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the
 // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
 // Rogaway.
 //
-// This makes for deterministic encryption which is what we want - the
+// This makes for determinstic encryption which is what we want - the
 // same filename must encrypt to the same thing.
 //
 // This means that
-//   - filenames with the same name will encrypt the same
-//   - filenames which start the same won't have a common prefix
-func (c *Cipher) encryptSegment(plaintext string) string {
+//  * filenames with the same name will encrypt the same
+//  * filenames which start the same won't have a common prefix
+func (c *cipher) encryptSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
 	}
 	paddedPlaintext := pkcs7.Pad(nameCipherBlockSize, []byte(plaintext))
 	ciphertext := eme.Transform(c.block, c.nameTweak[:], paddedPlaintext, eme.DirectionEncrypt)
-	return c.fileNameEnc.EncodeToString(ciphertext)
+	return encodeFileName(ciphertext)
 }
 
 // decryptSegment decrypts a path segment
-func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
+func (c *cipher) decryptSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
-	rawCiphertext, err := c.fileNameEnc.DecodeString(ciphertext)
+	rawCiphertext, err := decodeFileName(ciphertext)
 	if err != nil {
 		return "", err
 	}
@@ -308,11 +293,15 @@ func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	err = checkValidString(plaintext)
+	if err != nil {
+		return "", err
+	}
 	return string(plaintext), err
 }
 
 // Simple obfuscation routines
-func (c *Cipher) obfuscateSegment(plaintext string) string {
+func (c *cipher) obfuscateSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
 	}
@@ -329,7 +318,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
 	for _, runeValue := range plaintext {
 		dir += int(runeValue)
 	}
-	dir %= 256
+	dir = dir % 256
 
 	// We'll use this number to store in the result filename...
 	var result bytes.Buffer
@@ -399,7 +388,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
 	return result.String()
 }
 
-func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
+func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
@@ -450,7 +439,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if pos >= 26 {
 				pos -= 6
 			}
-			pos -= thisdir
+			pos = pos - thisdir
 			if pos < 0 {
 				pos += 52
 			}
@@ -486,7 +475,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 }
 
 // encryptFileName encrypts a file path
-func (c *Cipher) encryptFileName(in string) string {
+func (c *cipher) encryptFileName(in string) string {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		// Skip directory name encryption if the user chose to
@@ -494,46 +483,25 @@ func (c *Cipher) encryptFileName(in string) string {
 		if !c.dirNameEncrypt && i != (len(segments)-1) {
 			continue
 		}
-
-		// Strip version string so that only the non-versioned part
-		// of the file name gets encrypted/obfuscated
-		hasVersion := false
-		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
-			var s string
-			t, s = version.Remove(segments[i])
-			// version.Remove can fail, in which case it returns segments[i]
-			if s != segments[i] {
-				segments[i] = s
-				hasVersion = true
-			}
-		}
-
 		if c.mode == NameEncryptionStandard {
 			segments[i] = c.encryptSegment(segments[i])
 		} else {
 			segments[i] = c.obfuscateSegment(segments[i])
 		}
-
-		// Add back a version to the encrypted/obfuscated
-		// file name, if we stripped it off earlier
-		if hasVersion {
-			segments[i] = version.Add(segments[i], t)
-		}
 	}
 	return strings.Join(segments, "/")
 }
 
 // EncryptFileName encrypts a file path
-func (c *Cipher) EncryptFileName(in string) string {
+func (c *cipher) EncryptFileName(in string) string {
 	if c.mode == NameEncryptionOff {
-		return in + c.encryptedSuffix
+		return in + encryptedSuffix
 	}
 	return c.encryptFileName(in)
 }
 
 // EncryptDirName encrypts a directory path
-func (c *Cipher) EncryptDirName(in string) string {
+func (c *cipher) EncryptDirName(in string) string {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in
 	}
@@ -541,7 +509,7 @@ func (c *Cipher) EncryptDirName(in string) string {
 }
 
 // decryptFileName decrypts a file path
-func (c *Cipher) decryptFileName(in string) (string, error) {
+func (c *cipher) decryptFileName(in string) (string, error) {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		var err error
@@ -550,21 +518,6 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		if !c.dirNameEncrypt && i != (len(segments)-1) {
 			continue
 		}
-
-		// Strip version string so that only the non-versioned part
-		// of the file name gets decrypted/deobfuscated
-		hasVersion := false
-		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
-			var s string
-			t, s = version.Remove(segments[i])
-			// version.Remove can fail, in which case it returns segments[i]
-			if s != segments[i] {
-				segments[i] = s
-				hasVersion = true
-			}
-		}
-
 		if c.mode == NameEncryptionStandard {
 			segments[i], err = c.decryptSegment(segments[i])
 		} else {
@@ -574,46 +527,31 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		if err != nil {
 			return "", err
 		}
-
-		// Add back a version to the decrypted/deobfuscated
-		// file name, if we stripped it off earlier
-		if hasVersion {
-			segments[i] = version.Add(segments[i], t)
-		}
 	}
 	return strings.Join(segments, "/"), nil
 }
 
 // DecryptFileName decrypts a file path
-func (c *Cipher) DecryptFileName(in string) (string, error) {
+func (c *cipher) DecryptFileName(in string) (string, error) {
 	if c.mode == NameEncryptionOff {
-		remainingLength := len(in) - len(c.encryptedSuffix)
-		if remainingLength == 0 || !strings.HasSuffix(in, c.encryptedSuffix) {
+		remainingLength := len(in) - len(encryptedSuffix)
+		if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
+			return in[:remainingLength], nil
+		}
 		return "", ErrorNotAnEncryptedFile
 	}
-		decrypted := in[:remainingLength]
-		if version.Match(decrypted) {
-			_, unversioned := version.Remove(decrypted)
-			if unversioned == "" {
-				return "", ErrorNotAnEncryptedFile
-			}
-		}
-		// Leave the version string on, if it was there
-		return decrypted, nil
-	}
 	return c.decryptFileName(in)
 }
 
 // DecryptDirName decrypts a directory path
-func (c *Cipher) DecryptDirName(in string) (string, error) {
+func (c *cipher) DecryptDirName(in string) (string, error) {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in, nil
 	}
 	return c.decryptFileName(in)
 }
 
-// NameEncryptionMode returns the encryption mode in use for names
-func (c *Cipher) NameEncryptionMode() NameEncryptionMode {
+func (c *cipher) NameEncryptionMode() NameEncryptionMode {
 	return c.mode
 }
 
@@ -628,9 +566,9 @@ func (n *nonce) pointer() *[fileNonceSize]byte {
 // fromReader fills the nonce from an io.Reader - normally the OSes
 // crypto random number generator
 func (n *nonce) fromReader(in io.Reader) error {
-	read, err := readers.ReadFill(in, (*n)[:])
+	read, err := io.ReadFull(in, (*n)[:])
 	if read != fileNonceSize {
-		return fmt.Errorf("short read of nonce: %w", err)
+		return errors.Wrap(err, "short read of nonce")
 	}
 	return nil
 }
@@ -661,7 +599,7 @@ func (n *nonce) increment() {
 	n.carry(0)
 }
 
-// add a uint64 to the nonce
+// add an uint64 to the nonce
 func (n *nonce) add(x uint64) {
 	carry := uint16(0)
 	for i := 0; i < 8; i++ {
@@ -681,17 +619,17 @@ func (n *nonce) add(x uint64) {
 type encrypter struct {
 	mu       sync.Mutex
 	in       io.Reader
-	c        *Cipher
+	c        *cipher
 	nonce    nonce
-	buf      *[blockSize]byte
-	readBuf  *[blockSize]byte
+	buf      []byte
+	readBuf  []byte
 	bufIndex int
 	bufSize  int
 	err      error
 }
 
 // newEncrypter creates a new file handle encrypting on the fly
-func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
+func (c *cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 	fh := &encrypter{
 		in:      in,
 		c:       c,
@@ -709,9 +647,9 @@ func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 		}
 	}
 	// Copy magic into buffer
-	copy((*fh.buf)[:], fileMagicBytes)
+	copy(fh.buf, fileMagicBytes)
 	// Copy nonce into buffer
-	copy((*fh.buf)[fileMagicSize:], fh.nonce[:])
+	copy(fh.buf[fileMagicSize:], fh.nonce[:])
 	return fh, nil
 }
 
@@ -726,20 +664,25 @@ func (fh *encrypter) Read(p []byte) (n int, err error) {
 	if fh.bufIndex >= fh.bufSize {
 		// Read data
 		// FIXME should overlap the reads with a go-routine and 2 buffers?
-		readBuf := (*fh.readBuf)[:blockDataSize]
-		n, err = readers.ReadFill(fh.in, readBuf)
+		readBuf := fh.readBuf[:blockDataSize]
+		n, err = io.ReadFull(fh.in, readBuf)
 		if n == 0 {
+			// err can't be nil since:
+			// n == len(buf) if and only if err == nil.
 			return fh.finish(err)
 		}
 		// possibly err != nil here, but we will process the
-		// data and the next call to ReadFill will return 0, err
+		// data and the next call to ReadFull will return 0, err
+		// Write nonce to start of block
+		copy(fh.buf, fh.nonce[:])
 		// Encrypt the block using the nonce
-		secretbox.Seal((*fh.buf)[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+		block := fh.buf
+		secretbox.Seal(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 		fh.bufIndex = 0
 		fh.bufSize = blockHeaderSize + n
 		fh.nonce.increment()
 	}
-	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufSize])
+	n = copy(p, fh.buf[fh.bufIndex:fh.bufSize])
 	fh.bufIndex += n
 	return n, nil
 }
@@ -758,19 +701,13 @@ func (fh *encrypter) finish(err error) (int, error) {
 }
 
 // Encrypt data encrypts the data stream
-func (c *Cipher) encryptData(in io.Reader) (io.Reader, *encrypter, error) {
+func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) {
 	in, wrap := accounting.UnWrap(in) // unwrap the accounting off the Reader
 	out, err := c.newEncrypter(in, nil)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	return wrap(out), out, nil // and wrap the accounting back on
-}
-
-// EncryptData encrypts the data stream
-func (c *Cipher) EncryptData(in io.Reader) (io.Reader, error) {
-	out, _, err := c.encryptData(in)
-	return out, err
+	return wrap(out), nil // and wrap the accounting back on
 }
 
 // decrypter decrypts an io.ReaderCloser on the fly
@@ -779,9 +716,9 @@ type decrypter struct {
 	rc           io.ReadCloser
 	nonce        nonce
 	initialNonce nonce
-	c            *Cipher
-	buf          *[blockSize]byte
-	readBuf      *[blockSize]byte
+	c            *cipher
+	buf          []byte
+	readBuf      []byte
 	bufIndex     int
 	bufSize      int
 	err          error
@@ -790,7 +727,7 @@ type decrypter struct {
 }
 
 // newDecrypter creates a new file handle decrypting on the fly
-func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
+func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	fh := &decrypter{
 		rc:      rc,
 		c:       c,
@@ -799,12 +736,12 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 		limit:   -1,
 	}
 	// Read file header (magic + nonce)
-	readBuf := (*fh.readBuf)[:fileHeaderSize]
-	n, err := readers.ReadFill(fh.rc, readBuf)
-	if n < fileHeaderSize && err == io.EOF {
+	readBuf := fh.readBuf[:fileHeaderSize]
+	_, err := io.ReadFull(fh.rc, readBuf)
+	if err == io.EOF || err == io.ErrUnexpectedEOF {
 		// This read from 0..fileHeaderSize-1 bytes
 		return nil, fh.finishAndClose(ErrorEncryptedFileTooShort)
-	} else if err != io.EOF && err != nil {
+	} else if err != nil {
 		return nil, fh.finishAndClose(err)
 	}
 	// check the magic
@@ -818,22 +755,22 @@ func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 }
 
 // newDecrypterSeek creates a new file handle decrypting on the fly
-func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
+func (c *cipher) newDecrypterSeek(open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
 	var rc io.ReadCloser
 	doRangeSeek := false
 	setLimit := false
 	// Open initially with no seek
 	if offset == 0 && limit < 0 {
 		// If no offset or limit then open whole file
-		rc, err = open(ctx, 0, -1)
+		rc, err = open(0, -1)
 	} else if offset == 0 {
 		// If no offset open the header + limit worth of the file
 		_, underlyingLimit, _, _ := calculateUnderlying(offset, limit)
-		rc, err = open(ctx, 0, int64(fileHeaderSize)+underlyingLimit)
+		rc, err = open(0, int64(fileHeaderSize)+underlyingLimit)
 		setLimit = true
 	} else {
 		// Otherwise just read the header to start with
-		rc, err = open(ctx, 0, int64(fileHeaderSize))
+		rc, err = open(0, int64(fileHeaderSize))
 		doRangeSeek = true
 	}
 	if err != nil {
@@ -846,7 +783,7 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
 	}
 	fh.open = open // will be called by fh.RangeSeek
 	if doRangeSeek {
-		_, err = fh.RangeSeek(ctx, offset, io.SeekStart, limit)
+		_, err = fh.RangeSeek(offset, io.SeekStart, limit)
 		if err != nil {
 			_ = fh.Close()
 			return nil, err
@@ -862,8 +799,10 @@ func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offse
 func (fh *decrypter) fillBuffer() (err error) {
 	// FIXME should overlap the reads with a go-routine and 2 buffers?
 	readBuf := fh.readBuf
-	n, err := readers.ReadFill(fh.rc, (*readBuf)[:])
+	n, err := io.ReadFull(fh.rc, readBuf)
 	if n == 0 {
+		// err can't be nil since:
+		// n == len(buf) if and only if err == nil.
 		return err
 	}
 	// possibly err != nil here, but we will process the data and
@@ -871,26 +810,20 @@ func (fh *decrypter) fillBuffer() (err error) {
 
 	// Check header + 1 byte exists
 	if n <= blockHeaderSize {
-		if err != nil && err != io.EOF {
+		if err != nil {
 			return err // return pending error as it is likely more accurate
 		}
 		return ErrorEncryptedFileBadHeader
 	}
 	// Decrypt the block using the nonce
-	_, ok := secretbox.Open((*fh.buf)[:0], (*readBuf)[:n], fh.nonce.pointer(), &fh.c.dataKey)
+	block := fh.buf
+	_, ok := secretbox.Open(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 	if !ok {
-		if err != nil && err != io.EOF {
+		if err != nil {
 			return err // return pending error as it is likely more accurate
 		}
-		if !fh.c.passBadBlocks {
 		return ErrorEncryptedBadBlock
 	}
-		fs.Errorf(nil, "crypt: ignoring: %v", ErrorEncryptedBadBlock)
-		// Zero out the bad block and continue
-		for i := range (*fh.buf)[:n] {
-			fh.buf[i] = 0
-		}
-	}
 	fh.bufIndex = 0
 	fh.bufSize = n - blockHeaderSize
 	fh.nonce.increment()
@@ -915,7 +848,7 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	if fh.limit >= 0 && fh.limit < int64(toCopy) {
 		toCopy = int(fh.limit)
 	}
-	n = copy(p, (*fh.buf)[fh.bufIndex:fh.bufIndex+toCopy])
+	n = copy(p, fh.buf[fh.bufIndex:fh.bufIndex+toCopy])
 	fh.bufIndex += n
 	if fh.limit >= 0 {
 		fh.limit -= int64(n)
@@ -926,8 +859,9 @@ func (fh *decrypter) Read(p []byte) (n int, err error) {
 	return n, nil
 }
 
-// calculateUnderlying converts an (offset, limit) in an encrypted file
-// into an (underlyingOffset, underlyingLimit) for the underlying file.
+// calculateUnderlying converts an (offset, limit) in a crypted file
+// into an (underlyingOffset, underlyingLimit) for the underlying
+// file.
 //
 // It also returns number of bytes to discard after reading the first
 // block and number of blocks this is from the start so the nonce can
@@ -969,7 +903,7 @@ func calculateUnderlying(offset, limit int64) (underlyingOffset, underlyingLimit
 // limiting the total length to limit.
 //
 // RangeSeek with a limit of < 0 is equivalent to a regular Seek.
-func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, limit int64) (int64, error) {
+func (fh *decrypter) RangeSeek(offset int64, whence int, limit int64) (int64, error) {
 	fh.mu.Lock()
 	defer fh.mu.Unlock()
 
@@ -996,7 +930,7 @@ func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, li
 	// Can we seek underlying stream directly?
 	if do, ok := fh.rc.(fs.RangeSeeker); ok {
 		// Seek underlying stream directly
-		_, err := do.RangeSeek(ctx, underlyingOffset, 0, underlyingLimit)
+		_, err := do.RangeSeek(underlyingOffset, 0, underlyingLimit)
 		if err != nil {
 			return 0, fh.finish(err)
 		}
@@ -1006,9 +940,9 @@ func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, li
 		fh.rc = nil
 
 		// Re-open the underlying object with the offset given
-		rc, err := fh.open(ctx, underlyingOffset, underlyingLimit)
+		rc, err := fh.open(underlyingOffset, underlyingLimit)
 		if err != nil {
-			return 0, fh.finish(fmt.Errorf("couldn't reopen file with offset and limit: %w", err))
+			return 0, fh.finish(errors.Wrap(err, "couldn't reopen file with offset and limit"))
 		}
 
 		// Set the file handle
@@ -1035,7 +969,7 @@ func (fh *decrypter) RangeSeek(ctx context.Context, offset int64, whence int, li
 
 // Seek implements the io.Seeker interface
 func (fh *decrypter) Seek(offset int64, whence int) (int64, error) {
-	return fh.RangeSeek(context.TODO(), offset, whence, -1)
+	return fh.RangeSeek(offset, whence, -1)
 }
 
 // finish sets the final error and tidies up
@@ -1096,7 +1030,7 @@ func (fh *decrypter) finishAndClose(err error) error {
 }
 
 // DecryptData decrypts the data stream
-func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
+func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 	out, err := c.newDecrypter(rc)
 	if err != nil {
 		return nil, err
@@ -1106,11 +1040,11 @@ func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 
 // DecryptDataSeek decrypts the data stream from offset
 //
-// The open function must return a ReadCloser opened to the offset supplied.
+// The open function must return a ReadCloser opened to the offset supplied
 //
 // You must use this form of DecryptData if you might want to Seek the file handle
-func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
-	out, err := c.newDecrypterSeek(ctx, open, offset, limit)
+func (c *cipher) DecryptDataSeek(open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
+	out, err := c.newDecrypterSeek(open, offset, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -1118,7 +1052,7 @@ func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset
 }
 
 // EncryptedSize calculates the size of the data when encrypted
-func (c *Cipher) EncryptedSize(size int64) int64 {
+func (c *cipher) EncryptedSize(size int64) int64 {
 	blocks, residue := size/blockDataSize, size%blockDataSize
 	encryptedSize := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
 	if residue != 0 {
@@ -1128,7 +1062,7 @@ func (c *Cipher) EncryptedSize(size int64) int64 {
 }
 
 // DecryptedSize calculates the size of the data when decrypted
-func (c *Cipher) DecryptedSize(size int64) (int64, error) {
+func (c *cipher) DecryptedSize(size int64) (int64, error) {
 	size -= int64(fileHeaderSize)
 	if size < 0 {
 		return 0, ErrorEncryptedFileTooShort
@@ -1147,6 +1081,7 @@ func (c *Cipher) DecryptedSize(size int64) (int64, error) {
 
 // check interfaces
 var (
+	_ Cipher         = (*cipher)(nil)
 	_ io.ReadCloser  = (*decrypter)(nil)
 	_ io.Seeker      = (*decrypter)(nil)
 	_ fs.RangeSeeker = (*decrypter)(nil)

backend/crypt/crypt_internal_test.go

@@ -1,127 +0,0 @@
-package crypt
-
-import (
-	"bytes"
-	"context"
-	"crypto/md5"
-	"fmt"
-	"io"
-	"testing"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/object"
-	"github.com/rclone/rclone/lib/random"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// Create a temporary local fs to upload things from
-
-func makeTempLocalFs(t *testing.T) (localFs fs.Fs) {
-	localFs, err := fs.TemporaryLocalFs(context.Background())
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		require.NoError(t, localFs.Rmdir(context.Background(), ""))
-	})
-	return localFs
-}
-
-// Upload a file to a remote
-func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object) {
-	inBuf := bytes.NewBufferString(contents)
-	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
-	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
-	obj, err := f.Put(context.Background(), inBuf, upSrc)
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		require.NoError(t, obj.Remove(context.Background()))
-	})
-	return obj
-}
-
-// Test the ObjectInfo
-func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
-	var (
-		contents = random.String(100)
-		path     = "hash_test_object"
-		ctx      = context.Background()
-	)
-	if wrap {
-		path = "_wrap"
-	}
-
-	localFs := makeTempLocalFs(t)
-
-	obj := uploadFile(t, localFs, path, contents)
-
-	// encrypt the data
-	inBuf := bytes.NewBufferString(contents)
-	var outBuf bytes.Buffer
-	enc, err := f.cipher.newEncrypter(inBuf, nil)
-	require.NoError(t, err)
-	nonce := enc.nonce // read the nonce at the start
-	_, err = io.Copy(&outBuf, enc)
-	require.NoError(t, err)
-
-	var oi fs.ObjectInfo = obj
-	if wrap {
-		// wrap the object in an fs.ObjectUnwrapper if required
-		oi = fs.NewOverrideRemote(oi, "new_remote")
-	}
-
-	// wrap the object in a crypt for upload using the nonce we
-	// saved from the encrypter
-	src := f.newObjectInfo(oi, nonce)
-
-	// Test ObjectInfo methods
-	if !f.opt.NoDataEncryption {
-		assert.Equal(t, int64(outBuf.Len()), src.Size())
-	}
-	assert.Equal(t, f, src.Fs())
-	assert.NotEqual(t, path, src.Remote())
-
-	// Test ObjectInfo.Hash
-	wantHash := md5.Sum(outBuf.Bytes())
-	gotHash, err := src.Hash(ctx, hash.MD5)
-	require.NoError(t, err)
-	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
-}
-
-func testComputeHash(t *testing.T, f *Fs) {
-	var (
-		contents = random.String(100)
-		path     = "compute_hash_test"
-		ctx      = context.Background()
-		hashType = f.Fs.Hashes().GetOne()
-	)
-
-	if hashType == hash.None {
-		t.Skipf("%v: does not support hashes", f.Fs)
-	}
-
-	localFs := makeTempLocalFs(t)
-
-	// Upload a file to localFs as a test object
-	localObj := uploadFile(t, localFs, path, contents)
-
-	// Upload the same data to the remote Fs also
-	remoteObj := uploadFile(t, f, path, contents)
-
-	// Calculate the expected Hash of the remote object
-	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)
-	require.NoError(t, err)
-
-	// Test computed hash matches remote object hash
-	remoteObjHash, err := remoteObj.(*Object).Object.Hash(ctx, hashType)
-	require.NoError(t, err)
-	assert.Equal(t, remoteObjHash, computedHash)
-}
-
-// InternalTest is called by fstests.Run to extra tests
-func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("ObjectInfo", func(t *testing.T) { testObjectInfo(t, f, false) })
-	t.Run("ObjectInfoWrap", func(t *testing.T) { testObjectInfo(t, f, true) })
-	t.Run("ComputeHash", func(t *testing.T) { testComputeHash(t, f) })
-}

backend/crypt/crypt_test.go

@@ -4,16 +4,15 @@ package crypt_test
 import (
 	"os"
 	"path/filepath"
-	"runtime"
 	"testing"
 
-	"github.com/rclone/rclone/backend/crypt"
-	_ "github.com/rclone/rclone/backend/drive" // for integration tests
-	_ "github.com/rclone/rclone/backend/local"
-	_ "github.com/rclone/rclone/backend/swift" // for integration tests
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/backend/crypt"
+	_ "github.com/ncw/rclone/backend/drive" // for integration tests
+	_ "github.com/ncw/rclone/backend/local"
+	_ "github.com/ncw/rclone/backend/swift" // for integration tests
+	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fstest"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
@@ -24,13 +23,11 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: *fstest.RemoteName,
 		NilObject:  (*crypt.Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
-		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
 // TestStandard runs integration tests against the remote
-func TestStandardBase32(t *testing.T) {
+func TestStandard(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
@@ -45,53 +42,6 @@ func TestStandardBase32(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
-	})
-}
-
-func TestStandardBase64(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
-	name := "TestCrypt"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		NilObject:  (*crypt.Object)(nil),
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "crypt"},
-			{Name: name, Key: "remote", Value: tempdir},
-			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
-			{Name: name, Key: "filename_encryption", Value: "standard"},
-			{Name: name, Key: "filename_encoding", Value: "base64"},
-		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
-	})
-}
-
-func TestStandardBase32768(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
-	name := "TestCrypt"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		NilObject:  (*crypt.Object)(nil),
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "crypt"},
-			{Name: name, Key: "remote", Value: tempdir},
-			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
-			{Name: name, Key: "filename_encryption", Value: "standard"},
-			{Name: name, Key: "filename_encoding", Value: "base32768"},
-		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -111,9 +61,6 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }
 
@@ -122,9 +69,6 @@ func TestObfuscate(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
-	if runtime.GOOS == "darwin" {
-		t.Skip("Skipping on macOS as obfuscating control characters makes filenames macOS can't cope with")
-	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
 	name := "TestCrypt3"
 	fstests.Run(t, &fstests.Opt{
@@ -137,35 +81,5 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
 		SkipBadWindowsCharacters: true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
-	})
-}
-
-// TestNoDataObfuscate runs integration tests against the remote
-func TestNoDataObfuscate(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	if runtime.GOOS == "darwin" {
-		t.Skip("Skipping on macOS as obfuscating control characters makes filenames macOS can't cope with")
-	}
-	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
-	name := "TestCrypt4"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		NilObject:  (*crypt.Object)(nil),
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "crypt"},
-			{Name: name, Key: "remote", Value: tempdir},
-			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
-			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
-			{Name: name, Key: "no_data_encryption", Value: "true"},
-		},
-		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-		QuickTestOK:                  true,
 	})
 }

backend/crypt/pkcs7/pkcs7.go

@@ -4,15 +4,15 @@
 // buffers which are a multiple of an underlying crypto block size.
 package pkcs7
 
-import "errors"
+import "github.com/pkg/errors"
 
 // Errors Unpad can return
 var (
-	ErrorPaddingNotFound      = errors.New("bad PKCS#7 padding - not padded")
-	ErrorPaddingNotAMultiple  = errors.New("bad PKCS#7 padding - not a multiple of blocksize")
-	ErrorPaddingTooLong       = errors.New("bad PKCS#7 padding - too long")
-	ErrorPaddingTooShort      = errors.New("bad PKCS#7 padding - too short")
-	ErrorPaddingNotAllTheSame = errors.New("bad PKCS#7 padding - not all the same")
+	ErrorPaddingNotFound      = errors.New("Bad PKCS#7 padding - not padded")
+	ErrorPaddingNotAMultiple  = errors.New("Bad PKCS#7 padding - not a multiple of blocksize")
+	ErrorPaddingTooLong       = errors.New("Bad PKCS#7 padding - too long")
+	ErrorPaddingTooShort      = errors.New("Bad PKCS#7 padding - too short")
+	ErrorPaddingNotAllTheSame = errors.New("Bad PKCS#7 padding - not all the same")
 )
 
 // Pad buf using PKCS#7 to a multiple of n.

backend/drive/drive_internal_test.go

@@ -1,61 +1,27 @@
+// +build go1.9
+
 package drive
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
-	"errors"
-	"fmt"
 	"io"
+	"io/ioutil"
 	"mime"
-	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"testing"
-	"time"
 
-	_ "github.com/rclone/rclone/backend/local"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/filter"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fs/sync"
-	"github.com/rclone/rclone/fstest"
-	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/random"
+	_ "github.com/ncw/rclone/backend/local"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/operations"
+	"github.com/ncw/rclone/fstest/fstests"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/api/drive/v3"
-	"google.golang.org/api/googleapi"
 )
 
-func TestDriveScopes(t *testing.T) {
-	for _, test := range []struct {
-		in       string
-		want     []string
-		wantFlag bool
-	}{
-		{"", []string{
-			"https://www.googleapis.com/auth/drive",
-		}, false},
-		{" drive.file , drive.readonly", []string{
-			"https://www.googleapis.com/auth/drive.file",
-			"https://www.googleapis.com/auth/drive.readonly",
-		}, false},
-		{" drive.file , drive.appfolder", []string{
-			"https://www.googleapis.com/auth/drive.file",
-			"https://www.googleapis.com/auth/drive.appfolder",
-		}, true},
-	} {
-		got := driveScopes(test.in)
-		assert.Equal(t, test.want, got, test.in)
-		gotFlag := driveScopesContainsAppFolder(got)
-		assert.Equal(t, test.wantFlag, gotFlag, test.in)
-	}
-}
-
 /*
 var additionalMimeTypes = map[string]string{
 	"application/vnd.ms-excel.sheet.macroenabled.12":                          ".xlsm",
@@ -77,7 +43,7 @@ var additionalMimeTypes = map[string]string{
 // Load the example export formats into exportFormats for testing
 func TestInternalLoadExampleFormats(t *testing.T) {
 	fetchFormatsOnce.Do(func() {})
-	buf, err := os.ReadFile(filepath.FromSlash("test/about.json"))
+	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
 	var about struct {
 		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
 		ImportFormats map[string][]string `json:"importFormats,omitempty"`
@@ -115,7 +81,6 @@ func TestInternalParseExtensions(t *testing.T) {
 }
 
 func TestInternalFindExportFormat(t *testing.T) {
-	ctx := context.Background()
 	item := &drive.File{
 		Name:     "file",
 		MimeType: "application/vnd.google-apps.document",
@@ -133,7 +98,7 @@ func TestInternalFindExportFormat(t *testing.T) {
 	} {
 		f := new(Fs)
 		f.exportExtensions = test.extensions
-		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(ctx, item)
+		gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item)
 		assert.Equal(t, test.wantExtension, gotExtension)
 		if test.wantExtension != "" {
 			assert.Equal(t, item.Name+gotExtension, gotFilename)
@@ -191,69 +156,6 @@ func TestExtensionsForImportFormats(t *testing.T) {
 	}
 }
 
-func (f *Fs) InternalTestShouldRetry(t *testing.T) {
-	ctx := context.Background()
-	gatewayTimeout := googleapi.Error{
-		Code: 503,
-	}
-	timeoutRetry, timeoutError := f.shouldRetry(ctx, &gatewayTimeout)
-	assert.True(t, timeoutRetry)
-	assert.Equal(t, &gatewayTimeout, timeoutError)
-	generic403 := googleapi.Error{
-		Code: 403,
-	}
-	rLEItem := googleapi.ErrorItem{
-		Reason:  "rateLimitExceeded",
-		Message: "User rate limit exceeded.",
-	}
-	generic403.Errors = append(generic403.Errors, rLEItem)
-	oldStopUpload := f.opt.StopOnUploadLimit
-	oldStopDownload := f.opt.StopOnDownloadLimit
-	f.opt.StopOnUploadLimit = true
-	f.opt.StopOnDownloadLimit = true
-	defer func() {
-		f.opt.StopOnUploadLimit = oldStopUpload
-		f.opt.StopOnDownloadLimit = oldStopDownload
-	}()
-	expectedRLError := fserrors.FatalError(&generic403)
-	rateLimitRetry, rateLimitErr := f.shouldRetry(ctx, &generic403)
-	assert.False(t, rateLimitRetry)
-	assert.Equal(t, rateLimitErr, expectedRLError)
-	dQEItem := googleapi.ErrorItem{
-		Reason: "downloadQuotaExceeded",
-	}
-	generic403.Errors[0] = dQEItem
-	expectedDQError := fserrors.FatalError(&generic403)
-	downloadQuotaRetry, downloadQuotaError := f.shouldRetry(ctx, &generic403)
-	assert.False(t, downloadQuotaRetry)
-	assert.Equal(t, downloadQuotaError, expectedDQError)
-	tDFLEItem := googleapi.ErrorItem{
-		Reason: "teamDriveFileLimitExceeded",
-	}
-	generic403.Errors[0] = tDFLEItem
-	expectedTDFLError := fserrors.FatalError(&generic403)
-	teamDriveFileLimitRetry, teamDriveFileLimitError := f.shouldRetry(ctx, &generic403)
-	assert.False(t, teamDriveFileLimitRetry)
-	assert.Equal(t, teamDriveFileLimitError, expectedTDFLError)
-	qEItem := googleapi.ErrorItem{
-		Reason: "quotaExceeded",
-	}
-	generic403.Errors[0] = qEItem
-	expectedQuotaError := fserrors.FatalError(&generic403)
-	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
-	assert.False(t, quotaExceededRetry)
-	assert.Equal(t, quotaExceededError, expectedQuotaError)
-
-	sqEItem := googleapi.ErrorItem{
-		Reason: "storageQuotaExceeded",
-	}
-	generic403.Errors[0] = sqEItem
-	expectedStorageQuotaError := fserrors.FatalError(&generic403)
-	storageQuotaExceededRetry, storageQuotaExceededError := f.shouldRetry(ctx, &generic403)
-	assert.False(t, storageQuotaExceededRetry)
-	assert.Equal(t, storageQuotaExceededError, expectedStorageQuotaError)
-}
-
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {
 	oldAllow := f.opt.AllowImportNameChange
 	f.opt.AllowImportNameChange = true
@@ -264,13 +166,13 @@ func (f *Fs) InternalTestDocumentImport(t *testing.T) {
 	testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
 	require.NoError(t, err)
 
-	testFilesFs, err := fs.NewFs(context.Background(), testFilesPath)
+	testFilesFs, err := fs.NewFs(testFilesPath)
 	require.NoError(t, err)
 
 	_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
 	require.NoError(t, err)
 
-	err = operations.CopyFile(context.Background(), f, testFilesFs, "example2.doc", "example2.doc")
+	err = operations.CopyFile(f, testFilesFs, "example2.doc", "example2.doc")
 	require.NoError(t, err)
 }
 
@@ -278,13 +180,13 @@ func (f *Fs) InternalTestDocumentUpdate(t *testing.T) {
 	testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
 	require.NoError(t, err)
 
-	testFilesFs, err := fs.NewFs(context.Background(), testFilesPath)
+	testFilesFs, err := fs.NewFs(testFilesPath)
 	require.NoError(t, err)
 
 	_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
 	require.NoError(t, err)
 
-	err = operations.CopyFile(context.Background(), f, testFilesFs, "example2.xlsx", "example1.ods")
+	err = operations.CopyFile(f, testFilesFs, "example2.xlsx", "example1.ods")
 	require.NoError(t, err)
 }
 
@@ -295,10 +197,10 @@ func (f *Fs) InternalTestDocumentExport(t *testing.T) {
 	f.exportExtensions, _, err = parseExtensions("txt")
 	require.NoError(t, err)
 
-	obj, err := f.NewObject(context.Background(), "example2.txt")
+	obj, err := f.NewObject("example2.txt")
 	require.NoError(t, err)
 
-	rc, err := obj.Open(context.Background())
+	rc, err := obj.Open()
 	require.NoError(t, err)
 	defer func() { require.NoError(t, rc.Close()) }()
 
@@ -321,10 +223,10 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
 	f.exportExtensions, _, err = parseExtensions("link.html")
 	require.NoError(t, err)
 
-	obj, err := f.NewObject(context.Background(), "example2.link.html")
+	obj, err := f.NewObject("example2.link.html")
 	require.NoError(t, err)
 
-	rc, err := obj.Open(context.Background())
+	rc, err := obj.Open()
 	require.NoError(t, err)
 	defer func() { require.NoError(t, rc.Close()) }()
 
@@ -342,295 +244,6 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
 	}
 }
 
-const (
-	// from fstest/fstests/fstests.go
-	existingDir    = "hello? sausage"
-	existingFile   = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
-	existingSubDir = "êé"
-)
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/Shortcuts
-func (f *Fs) InternalTestShortcuts(t *testing.T) {
-	ctx := context.Background()
-	srcObj, err := f.NewObject(ctx, existingFile)
-	require.NoError(t, err)
-	srcHash, err := srcObj.Hash(ctx, hash.MD5)
-	require.NoError(t, err)
-	assert.NotEqual(t, "", srcHash)
-	t.Run("Errors", func(t *testing.T) {
-		_, err := f.makeShortcut(ctx, "", f, "")
-		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "can't be root")
-
-		_, err = f.makeShortcut(ctx, "notfound", f, "dst")
-		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "can't find source")
-
-		_, err = f.makeShortcut(ctx, existingFile, f, existingFile)
-		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "not overwriting")
-		assert.Contains(t, err.Error(), "existing file")
-
-		_, err = f.makeShortcut(ctx, existingFile, f, existingDir)
-		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "not overwriting")
-		assert.Contains(t, err.Error(), "existing directory")
-	})
-	t.Run("File", func(t *testing.T) {
-		dstObj, err := f.makeShortcut(ctx, existingFile, f, "shortcut.txt")
-		require.NoError(t, err)
-		require.NotNil(t, dstObj)
-		assert.Equal(t, "shortcut.txt", dstObj.Remote())
-		dstHash, err := dstObj.Hash(ctx, hash.MD5)
-		require.NoError(t, err)
-		assert.Equal(t, srcHash, dstHash)
-		require.NoError(t, dstObj.Remove(ctx))
-	})
-	t.Run("Dir", func(t *testing.T) {
-		dstObj, err := f.makeShortcut(ctx, existingDir, f, "shortcutdir")
-		require.NoError(t, err)
-		require.Nil(t, dstObj)
-		entries, err := f.List(ctx, "shortcutdir")
-		require.NoError(t, err)
-		require.Equal(t, 1, len(entries))
-		require.Equal(t, "shortcutdir/"+existingSubDir, entries[0].Remote())
-		require.NoError(t, f.Rmdir(ctx, "shortcutdir"))
-	})
-	t.Run("Command", func(t *testing.T) {
-		_, err := f.Command(ctx, "shortcut", []string{"one"}, nil)
-		require.Error(t, err)
-		require.Contains(t, err.Error(), "need exactly 2 arguments")
-
-		_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
-			"target": "doesnotexistremote:",
-		})
-		require.Error(t, err)
-		require.Contains(t, err.Error(), "couldn't find target")
-
-		_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
-			"target": ".",
-		})
-		require.Error(t, err)
-		require.Contains(t, err.Error(), "target is not a drive backend")
-
-		dstObjI, err := f.Command(ctx, "shortcut", []string{existingFile, "shortcut2.txt"}, map[string]string{
-			"target": fs.ConfigString(f),
-		})
-		require.NoError(t, err)
-		dstObj := dstObjI.(*Object)
-		assert.Equal(t, "shortcut2.txt", dstObj.Remote())
-		dstHash, err := dstObj.Hash(ctx, hash.MD5)
-		require.NoError(t, err)
-		assert.Equal(t, srcHash, dstHash)
-		require.NoError(t, dstObj.Remove(ctx))
-
-		dstObjI, err = f.Command(ctx, "shortcut", []string{existingFile, "shortcut3.txt"}, nil)
-		require.NoError(t, err)
-		dstObj = dstObjI.(*Object)
-		assert.Equal(t, "shortcut3.txt", dstObj.Remote())
-		dstHash, err = dstObj.Hash(ctx, hash.MD5)
-		require.NoError(t, err)
-		assert.Equal(t, srcHash, dstHash)
-		require.NoError(t, dstObj.Remove(ctx))
-	})
-}
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/UnTrash
-func (f *Fs) InternalTestUnTrash(t *testing.T) {
-	ctx := context.Background()
-
-	// Make some objects, one in a subdir
-	contents := random.String(100)
-	file1 := fstest.NewItem("trashDir/toBeTrashed", contents, time.Now())
-	obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false)
-	file2 := fstest.NewItem("trashDir/subdir/toBeTrashed", contents, time.Now())
-	_ = fstests.PutTestContents(ctx, t, f, &file2, contents, false)
-
-	// Check objects
-	checkObjects := func() {
-		fstest.CheckListingWithRoot(t, f, "trashDir", []fstest.Item{
-			file1,
-			file2,
-		}, []string{
-			"trashDir/subdir",
-		}, f.Precision())
-	}
-	checkObjects()
-
-	// Make sure we are using the trash
-	require.Equal(t, true, f.opt.UseTrash)
-
-	// Remove the object and the dir
-	require.NoError(t, obj1.Remove(ctx))
-	require.NoError(t, f.Purge(ctx, "trashDir/subdir"))
-
-	// Check objects gone
-	fstest.CheckListingWithRoot(t, f, "trashDir", []fstest.Item{}, []string{}, f.Precision())
-
-	// Restore the object and directory
-	r, err := f.unTrashDir(ctx, "trashDir", true)
-	require.NoError(t, err)
-	assert.Equal(t, unTrashResult{Errors: 0, Untrashed: 2}, r)
-
-	// Check objects restored
-	checkObjects()
-
-	// Remove the test dir
-	require.NoError(t, f.Purge(ctx, "trashDir"))
-}
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyID
-func (f *Fs) InternalTestCopyID(t *testing.T) {
-	ctx := context.Background()
-	obj, err := f.NewObject(ctx, existingFile)
-	require.NoError(t, err)
-	o := obj.(*Object)
-
-	dir := t.TempDir()
-
-	checkFile := func(name string) {
-		filePath := filepath.Join(dir, name)
-		fi, err := os.Stat(filePath)
-		require.NoError(t, err)
-		assert.Equal(t, int64(100), fi.Size())
-		err = os.Remove(filePath)
-		require.NoError(t, err)
-	}
-
-	t.Run("BadID", func(t *testing.T) {
-		err = f.copyID(ctx, "ID-NOT-FOUND", dir+"/")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "couldn't find id")
-	})
-
-	t.Run("Directory", func(t *testing.T) {
-		rootID, err := f.dirCache.RootID(ctx, false)
-		require.NoError(t, err)
-		err = f.copyID(ctx, rootID, dir+"/")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "can't copy directory")
-	})
-
-	t.Run("WithoutDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/")
-		require.NoError(t, err)
-		checkFile(path.Base(existingFile))
-	})
-
-	t.Run("WithDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/potato.txt")
-		require.NoError(t, err)
-		checkFile("potato.txt")
-	})
-}
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/Query
-func (f *Fs) InternalTestQuery(t *testing.T) {
-	ctx := context.Background()
-	var err error
-	t.Run("BadQuery", func(t *testing.T) {
-		_, err = f.query(ctx, "this is a bad query")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "failed to execute query")
-	})
-
-	t.Run("NoMatch", func(t *testing.T) {
-		results, err := f.query(ctx, fmt.Sprintf("name='%s' and name!='%s'", existingSubDir, existingSubDir))
-		require.NoError(t, err)
-		assert.Len(t, results, 0)
-	})
-
-	t.Run("GoodQuery", func(t *testing.T) {
-		pathSegments := strings.Split(existingFile, "/")
-		var parent string
-		for _, item := range pathSegments {
-			// the file name contains ' characters which must be escaped
-			escapedItem := f.opt.Enc.FromStandardName(item)
-			escapedItem = strings.ReplaceAll(escapedItem, `\`, `\\`)
-			escapedItem = strings.ReplaceAll(escapedItem, `'`, `\'`)
-
-			results, err := f.query(ctx, fmt.Sprintf("%strashed=false and name='%s'", parent, escapedItem))
-			require.NoError(t, err)
-			require.True(t, len(results) > 0)
-			for _, result := range results {
-				assert.True(t, len(result.Id) > 0)
-				assert.Equal(t, result.Name, item)
-			}
-			parent = fmt.Sprintf("'%s' in parents and ", results[0].Id)
-		}
-	})
-}
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
-func (f *Fs) InternalTestAgeQuery(t *testing.T) {
-	// Check set up for filtering
-	assert.True(t, f.Features().FilterAware)
-
-	opt := &filter.Options{}
-	err := opt.MaxAge.Set("1h")
-	assert.NoError(t, err)
-	flt, err := filter.NewFilter(opt)
-	assert.NoError(t, err)
-
-	defCtx := context.Background()
-	fltCtx := filter.ReplaceConfig(defCtx, flt)
-
-	testCtx1 := fltCtx
-	testCtx2 := filter.SetUseFilter(testCtx1, true)
-	testCtx3, testCancel := context.WithCancel(testCtx2)
-	testCtx4 := filter.SetUseFilter(testCtx3, false)
-	testCancel()
-	assert.False(t, filter.GetUseFilter(testCtx1))
-	assert.True(t, filter.GetUseFilter(testCtx2))
-	assert.True(t, filter.GetUseFilter(testCtx3))
-	assert.False(t, filter.GetUseFilter(testCtx4))
-
-	subRemote := fmt.Sprintf("%s:%s/%s", f.Name(), f.Root(), "agequery-testdir")
-	subFsResult, err := fs.NewFs(defCtx, subRemote)
-	require.NoError(t, err)
-	subFs, isDriveFs := subFsResult.(*Fs)
-	require.True(t, isDriveFs)
-
-	tempDir1 := t.TempDir()
-	tempFs1, err := fs.NewFs(defCtx, tempDir1)
-	require.NoError(t, err)
-
-	tempDir2 := t.TempDir()
-	tempFs2, err := fs.NewFs(defCtx, tempDir2)
-	require.NoError(t, err)
-
-	file1 := fstest.Item{ModTime: time.Now(), Path: "agequery.txt"}
-	_ = fstests.PutTestContents(defCtx, t, tempFs1, &file1, "abcxyz", true)
-
-	// validate sync/copy
-	const timeQuery = "(modifiedTime >= '"
-
-	assert.NoError(t, sync.CopyDir(defCtx, subFs, tempFs1, false))
-	assert.NotContains(t, subFs.lastQuery, timeQuery)
-
-	assert.NoError(t, sync.CopyDir(fltCtx, subFs, tempFs1, false))
-	assert.Contains(t, subFs.lastQuery, timeQuery)
-
-	assert.NoError(t, sync.CopyDir(fltCtx, tempFs2, subFs, false))
-	assert.Contains(t, subFs.lastQuery, timeQuery)
-
-	assert.NoError(t, sync.CopyDir(defCtx, tempFs2, subFs, false))
-	assert.NotContains(t, subFs.lastQuery, timeQuery)
-
-	// validate list/walk
-	devNull, errOpen := os.OpenFile(os.DevNull, os.O_WRONLY, 0)
-	require.NoError(t, errOpen)
-	defer func() {
-		_ = devNull.Close()
-	}()
-
-	assert.NoError(t, operations.List(defCtx, subFs, devNull))
-	assert.NotContains(t, subFs.lastQuery, timeQuery)
-
-	assert.NoError(t, operations.List(fltCtx, subFs, devNull))
-	assert.Contains(t, subFs.lastQuery, timeQuery)
-}
-
 func (f *Fs) InternalTest(t *testing.T) {
 	// These tests all depend on each other so run them as nested tests
 	t.Run("DocumentImport", func(t *testing.T) {
@@ -645,12 +258,6 @@ func (f *Fs) InternalTest(t *testing.T) {
 			})
 		})
 	})
-	t.Run("Shortcuts", f.InternalTestShortcuts)
-	t.Run("UnTrash", f.InternalTestUnTrash)
-	t.Run("CopyID", f.InternalTestCopyID)
-	t.Run("Query", f.InternalTestQuery)
-	t.Run("AgeQuery", f.InternalTestAgeQuery)
-	t.Run("ShouldRetry", f.InternalTestShouldRetry)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/drive/drive_test.go

@@ -1,12 +1,14 @@
 // Test Drive filesystem interface
 
+// +build go1.9
+
 package drive
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote

backend/drive/drive_unsupported.go

@@ -0,0 +1,6 @@
+// Build for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+// +build !go1.9
+
+package drive

backend/drive/metadata.go

@@ -1,638 +0,0 @@
-package drive
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"strconv"
-	"strings"
-	"sync"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/errcount"
-	"golang.org/x/sync/errgroup"
-	drive "google.golang.org/api/drive/v3"
-	"google.golang.org/api/googleapi"
-)
-
-// system metadata keys which this backend owns
-var systemMetadataInfo = map[string]fs.MetadataHelp{
-	"content-type": {
-		Help:    "The MIME type of the file.",
-		Type:    "string",
-		Example: "text/plain",
-	},
-	"mtime": {
-		Help:    "Time of last modification with mS accuracy.",
-		Type:    "RFC 3339",
-		Example: "2006-01-02T15:04:05.999Z07:00",
-	},
-	"btime": {
-		Help:    "Time of file birth (creation) with mS accuracy. Note that this is only writable on fresh uploads - it can't be written for updates.",
-		Type:    "RFC 3339",
-		Example: "2006-01-02T15:04:05.999Z07:00",
-	},
-	"copy-requires-writer-permission": {
-		Help:    "Whether the options to copy, print, or download this file, should be disabled for readers and commenters.",
-		Type:    "boolean",
-		Example: "true",
-	},
-	"writers-can-share": {
-		Help:    "Whether users with only writer permission can modify the file's permissions. Not populated and ignored when setting for items in shared drives.",
-		Type:    "boolean",
-		Example: "false",
-	},
-	"viewed-by-me": {
-		Help:     "Whether the file has been viewed by this user.",
-		Type:     "boolean",
-		Example:  "true",
-		ReadOnly: true,
-	},
-	"owner": {
-		Help:    "The owner of the file. Usually an email address. Enable with --drive-metadata-owner.",
-		Type:    "string",
-		Example: "user@example.com",
-	},
-	"permissions": {
-		Help:    "Permissions in a JSON dump of Google drive format. On shared drives these will only be present if they aren't inherited. Enable with --drive-metadata-permissions.",
-		Type:    "JSON",
-		Example: "{}",
-	},
-	"folder-color-rgb": {
-		Help:    "The color for a folder or a shortcut to a folder as an RGB hex string.",
-		Type:    "string",
-		Example: "881133",
-	},
-	"description": {
-		Help:    "A short description of the file.",
-		Type:    "string",
-		Example: "Contract for signing",
-	},
-	"starred": {
-		Help:    "Whether the user has starred the file.",
-		Type:    "boolean",
-		Example: "false",
-	},
-	"labels": {
-		Help:    "Labels attached to this file in a JSON dump of Googled drive format. Enable with --drive-metadata-labels.",
-		Type:    "JSON",
-		Example: "[]",
-	},
-}
-
-// Extra fields we need to fetch to implement the system metadata above
-var metadataFields = googleapi.Field(strings.Join([]string{
-	"copyRequiresWriterPermission",
-	"description",
-	"folderColorRgb",
-	"hasAugmentedPermissions",
-	"owners",
-	"permissionIds",
-	"permissions",
-	"properties",
-	"starred",
-	"viewedByMe",
-	"viewedByMeTime",
-	"writersCanShare",
-}, ","))
-
-// Fields we need to read from permissions
-var permissionsFields = googleapi.Field(strings.Join([]string{
-	"*",
-	"permissionDetails/*",
-}, ","))
-
-// getPermission returns permissions for the fileID and permissionID passed in
-func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
-	f.permissionsMu.Lock()
-	defer f.permissionsMu.Unlock()
-	if useCache {
-		perm = f.permissions[permissionID]
-		if perm != nil {
-			return perm, false, nil
-		}
-	}
-	fs.Debugf(f, "Fetching permission %q", permissionID)
-	err = f.pacer.Call(func() (bool, error) {
-		perm, err = f.svc.Permissions.Get(fileID, permissionID).
-			Fields(permissionsFields).
-			SupportsAllDrives(true).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, false, err
-	}
-
-	inherited = len(perm.PermissionDetails) > 0 && perm.PermissionDetails[0].Inherited
-
-	cleanPermission(perm)
-
-	// cache the permission
-	f.permissions[permissionID] = perm
-
-	return perm, inherited, err
-}
-
-// Set the permissions on the info
-func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) {
-	errs := errcount.New()
-	for _, perm := range permissions {
-		if perm.Role == "owner" {
-			// ignore owner permissions - these are set with owner
-			continue
-		}
-		cleanPermissionForWrite(perm)
-		err := f.pacer.Call(func() (bool, error) {
-			_, err := f.svc.Permissions.Create(info.Id, perm).
-				SupportsAllDrives(true).
-				SendNotificationEmail(false).
-				Context(ctx).Do()
-			return f.shouldRetry(ctx, err)
-		})
-		if err != nil {
-			fs.Errorf(f, "Failed to set permission %s for %q: %v", perm.Role, perm.EmailAddress, err)
-			errs.Add(err)
-		}
-	}
-	err = errs.Err("failed to set permission")
-	if err != nil {
-		err = fserrors.NoRetryError(err)
-	}
-	return err
-}
-
-// Clean attributes from permissions which we can't write
-func cleanPermissionForWrite(perm *drive.Permission) {
-	perm.Deleted = false
-	perm.DisplayName = ""
-	perm.Id = ""
-	perm.Kind = ""
-	perm.PermissionDetails = nil
-	perm.TeamDrivePermissionDetails = nil
-}
-
-// Clean and cache the permission if not already cached
-func (f *Fs) cleanAndCachePermission(perm *drive.Permission) {
-	f.permissionsMu.Lock()
-	defer f.permissionsMu.Unlock()
-	cleanPermission(perm)
-	if _, found := f.permissions[perm.Id]; !found {
-		f.permissions[perm.Id] = perm
-	}
-}
-
-// Clean fields we don't need to keep from the permission
-func cleanPermission(perm *drive.Permission) {
-	// DisplayName: Output only. The "pretty" name of the value of the
-	// permission. The following is a list of examples for each type of
-	// permission: * `user` - User's full name, as defined for their Google
-	// account, such as "Joe Smith." * `group` - Name of the Google Group,
-	// such as "The Company Administrators." * `domain` - String domain
-	// name, such as "thecompany.com." * `anyone` - No `displayName` is
-	// present.
-	perm.DisplayName = ""
-
-	// Kind: Output only. Identifies what kind of resource this is. Value:
-	// the fixed string "drive#permission".
-	perm.Kind = ""
-
-	// PermissionDetails: Output only. Details of whether the permissions on
-	// this shared drive item are inherited or directly on this item. This
-	// is an output-only field which is present only for shared drive items.
-	perm.PermissionDetails = nil
-
-	// PhotoLink: Output only. A link to the user's profile photo, if
-	// available.
-	perm.PhotoLink = ""
-
-	// TeamDrivePermissionDetails: Output only. Deprecated: Output only. Use
-	// `permissionDetails` instead.
-	perm.TeamDrivePermissionDetails = nil
-}
-
-// Fields we need to read from labels
-var labelsFields = googleapi.Field(strings.Join([]string{
-	"*",
-}, ","))
-
-// getLabels returns labels for the fileID passed in
-func (f *Fs) getLabels(ctx context.Context, fileID string) (labels []*drive.Label, err error) {
-	fs.Debugf(f, "Fetching labels for %q", fileID)
-	listLabels := f.svc.Files.ListLabels(fileID).
-		Fields(labelsFields).
-		Context(ctx)
-	for {
-		var info *drive.LabelList
-		err = f.pacer.Call(func() (bool, error) {
-			info, err = listLabels.Do()
-			return f.shouldRetry(ctx, err)
-		})
-		if err != nil {
-			return nil, err
-		}
-		labels = append(labels, info.Labels...)
-		if info.NextPageToken == "" {
-			break
-		}
-		listLabels.PageToken(info.NextPageToken)
-	}
-	for _, label := range labels {
-		cleanLabel(label)
-	}
-	return labels, nil
-}
-
-// Set the labels on the info
-func (f *Fs) setLabels(ctx context.Context, info *drive.File, labels []*drive.Label) (err error) {
-	if len(labels) == 0 {
-		return nil
-	}
-	req := drive.ModifyLabelsRequest{}
-	for _, label := range labels {
-		req.LabelModifications = append(req.LabelModifications, &drive.LabelModification{
-			FieldModifications: labelFieldsToFieldModifications(label.Fields),
-			LabelId:            label.Id,
-		})
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.svc.Files.ModifyLabels(info.Id, &req).
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return fmt.Errorf("failed to set labels: %w", err)
-	}
-	return nil
-}
-
-// Convert label fields into something which can set the fields
-func labelFieldsToFieldModifications(fields map[string]drive.LabelField) (out []*drive.LabelFieldModification) {
-	for id, field := range fields {
-		var emails []string
-		for _, user := range field.User {
-			emails = append(emails, user.EmailAddress)
-		}
-		out = append(out, &drive.LabelFieldModification{
-			// FieldId: The ID of the field to be modified.
-			FieldId: id,
-
-			// SetDateValues: Replaces the value of a dateString Field with these
-			// new values. The string must be in the RFC 3339 full-date format:
-			// YYYY-MM-DD.
-			SetDateValues: field.DateString,
-
-			// SetIntegerValues: Replaces the value of an `integer` field with these
-			// new values.
-			SetIntegerValues: field.Integer,
-
-			// SetSelectionValues: Replaces a `selection` field with these new
-			// values.
-			SetSelectionValues: field.Selection,
-
-			// SetTextValues: Sets the value of a `text` field.
-			SetTextValues: field.Text,
-
-			// SetUserValues: Replaces a `user` field with these new values. The
-			// values must be valid email addresses.
-			SetUserValues: emails,
-		})
-	}
-	return out
-}
-
-// Clean fields we don't need to keep from the label
-func cleanLabel(label *drive.Label) {
-	// Kind: This is always drive#label
-	label.Kind = ""
-
-	for name, field := range label.Fields {
-		// Kind: This is always drive#labelField.
-		field.Kind = ""
-
-		// Note the fields are copies so we need to write them
-		// back to the map
-		label.Fields[name] = field
-	}
-}
-
-// Parse the metadata from drive item
-//
-// It should return nil if there is no Metadata
-func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err error) {
-	metadata := make(fs.Metadata, 16)
-
-	// Dump user metadata first as it overrides system metadata
-	for k, v := range info.Properties {
-		metadata[k] = v
-	}
-
-	// System metadata
-	metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission)
-	metadata["writers-can-share"] = fmt.Sprint(info.WritersCanShare)
-	metadata["viewed-by-me"] = fmt.Sprint(info.ViewedByMe)
-	metadata["content-type"] = info.MimeType
-
-	// Owners: Output only. The owner of this file. Only certain legacy
-	// files may have more than one owner. This field isn't populated for
-	// items in shared drives.
-	if o.fs.opt.MetadataOwner.IsSet(rwRead) && len(info.Owners) > 0 {
-		user := info.Owners[0]
-		if len(info.Owners) > 1 {
-			fs.Logf(o, "Ignoring more than 1 owner")
-		}
-		if user != nil {
-			id := user.EmailAddress
-			if id == "" {
-				id = user.DisplayName
-			}
-			metadata["owner"] = id
-		}
-	}
-
-	if o.fs.opt.MetadataPermissions.IsSet(rwRead) {
-		// We only write permissions out if they are not inherited.
-		//
-		// On My Drives permissions seem to be attached to every item
-		// so they will always be written out.
-		//
-		// On Shared Drives only non-inherited permissions will be
-		// written out.
-
-		// To read the inherited permissions flag will mean we need to
-		// read the permissions for each object and the cache will be
-		// useless. However shared drives don't return permissions
-		// only permissionIds so will need to fetch them for each
-		// object. We use HasAugmentedPermissions to see if there are
-		// special permissions before fetching them to save transactions.
-
-		// HasAugmentedPermissions: Output only. Whether there are permissions
-		// directly on this file. This field is only populated for items in
-		// shared drives.
-		if o.fs.isTeamDrive && !info.HasAugmentedPermissions {
-			// Don't process permissions if there aren't any specifically set
-			fs.Debugf(o, "Ignoring %d permissions and %d permissionIds as is shared drive with hasAugmentedPermissions false", len(info.Permissions), len(info.PermissionIds))
-			info.Permissions = nil
-			info.PermissionIds = nil
-		}
-
-		// PermissionIds: Output only. List of permission IDs for users with
-		// access to this file.
-		//
-		// Only process these if we have no Permissions
-		if len(info.PermissionIds) > 0 && len(info.Permissions) == 0 {
-			info.Permissions = make([]*drive.Permission, 0, len(info.PermissionIds))
-			g, gCtx := errgroup.WithContext(ctx)
-			g.SetLimit(o.fs.ci.Checkers)
-			var mu sync.Mutex // protect the info.Permissions from concurrent writes
-			for _, permissionID := range info.PermissionIds {
-				permissionID := permissionID
-				g.Go(func() error {
-					// must fetch the team drive ones individually to check the inherited flag
-					perm, inherited, err := o.fs.getPermission(gCtx, actualID(info.Id), permissionID, !o.fs.isTeamDrive)
-					if err != nil {
-						return fmt.Errorf("failed to read permission: %w", err)
-					}
-					// Don't write inherited permissions out
-					if inherited {
-						return nil
-					}
-					// Don't write owner role out - these are covered by the owner metadata
-					if perm.Role == "owner" {
-						return nil
-					}
-					mu.Lock()
-					info.Permissions = append(info.Permissions, perm)
-					mu.Unlock()
-					return nil
-				})
-			}
-			err = g.Wait()
-			if err != nil {
-				return err
-			}
-		} else {
-			// Clean the fetched permissions
-			for _, perm := range info.Permissions {
-				o.fs.cleanAndCachePermission(perm)
-			}
-		}
-
-		// Permissions: Output only. The full list of permissions for the file.
-		// This is only available if the requesting user can share the file. Not
-		// populated for items in shared drives.
-		if len(info.Permissions) > 0 {
-			buf, err := json.Marshal(info.Permissions)
-			if err != nil {
-				return fmt.Errorf("failed to marshal permissions: %w", err)
-			}
-			metadata["permissions"] = string(buf)
-		}
-
-		// Permission propagation
-		// https://developers.google.com/drive/api/guides/manage-sharing#permission-propagation
-		// Leads me to believe that in non shared drives, permissions
-		// are added to each item when you set permissions for a
-		// folder whereas in shared drives they are inherited and
-		// placed on the item directly.
-	}
-
-	if info.FolderColorRgb != "" {
-		metadata["folder-color-rgb"] = info.FolderColorRgb
-	}
-	if info.Description != "" {
-		metadata["description"] = info.Description
-	}
-	metadata["starred"] = fmt.Sprint(info.Starred)
-	metadata["btime"] = info.CreatedTime
-	metadata["mtime"] = info.ModifiedTime
-
-	if o.fs.opt.MetadataLabels.IsSet(rwRead) {
-		// FIXME would be really nice if we knew if files had labels
-		// before listing but we need to know all possible label IDs
-		// to get it in the listing.
-
-		labels, err := o.fs.getLabels(ctx, actualID(info.Id))
-		if err != nil {
-			return fmt.Errorf("failed to fetch labels: %w", err)
-		}
-		buf, err := json.Marshal(labels)
-		if err != nil {
-			return fmt.Errorf("failed to marshal labels: %w", err)
-		}
-		metadata["labels"] = string(buf)
-	}
-
-	o.metadata = &metadata
-	return nil
-}
-
-// Set the owner on the info
-func (f *Fs) setOwner(ctx context.Context, info *drive.File, owner string) (err error) {
-	perm := drive.Permission{
-		Role:         "owner",
-		EmailAddress: owner,
-		// Type: The type of the grantee. Valid values are: * `user` * `group` *
-		// `domain` * `anyone` When creating a permission, if `type` is `user`
-		// or `group`, you must provide an `emailAddress` for the user or group.
-		// When `type` is `domain`, you must provide a `domain`. There isn't
-		// extra information required for an `anyone` type.
-		Type: "user",
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.svc.Permissions.Create(info.Id, &perm).
-			SupportsAllDrives(true).
-			TransferOwnership(true).
-			// SendNotificationEmail(false). - required apparently!
-			Context(ctx).Do()
-		return f.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return fmt.Errorf("failed to set owner: %w", err)
-	}
-	return nil
-}
-
-// Call back to set metadata that can't be set on the upload/update
-//
-// The *drive.File passed in holds the current state of the drive.File
-// and this should update it with any modifications.
-type updateMetadataFn func(context.Context, *drive.File) error
-
-// read the metadata from meta and write it into updateInfo
-//
-// update should be true if this is being used to create metadata for
-// an update/PATCH call as the rules on what can be updated are
-// slightly different there.
-//
-// It returns a callback which should be called to finish the updates
-// after the data is uploaded.
-func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update bool) (callback updateMetadataFn, err error) {
-	callbackFns := []updateMetadataFn{}
-	callback = func(ctx context.Context, info *drive.File) error {
-		for _, fn := range callbackFns {
-			err := fn(ctx, info)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-	// merge metadata into request and user metadata
-	for k, v := range meta {
-		k, v := k, v
-		// parse a boolean from v and write into out
-		parseBool := func(out *bool) error {
-			b, err := strconv.ParseBool(v)
-			if err != nil {
-				return fmt.Errorf("can't parse metadata %q = %q: %w", k, v, err)
-			}
-			*out = b
-			return nil
-		}
-		switch k {
-		case "copy-requires-writer-permission":
-			if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil {
-				return nil, err
-			}
-		case "writers-can-share":
-			if !f.isTeamDrive {
-				if err := parseBool(&updateInfo.WritersCanShare); err != nil {
-					return nil, err
-				}
-			} else {
-				fs.Debugf(f, "Ignoring %s=%s as can't set on shared drives", k, v)
-			}
-		case "viewed-by-me":
-			// Can't write this
-		case "content-type":
-			updateInfo.MimeType = v
-		case "owner":
-			if !f.opt.MetadataOwner.IsSet(rwWrite) {
-				continue
-			}
-			// Can't set Owner on upload so need to set afterwards
-			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setOwner(ctx, info, v)
-				if err != nil && f.opt.MetadataOwner.IsSet(rwFailOK) {
-					fs.Errorf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
-			})
-		case "permissions":
-			if !f.opt.MetadataPermissions.IsSet(rwWrite) {
-				continue
-			}
-			var perms []*drive.Permission
-			err := json.Unmarshal([]byte(v), &perms)
-			if err != nil {
-				return nil, fmt.Errorf("failed to unmarshal permissions: %w", err)
-			}
-			// Can't set Permissions on upload so need to set afterwards
-			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setPermissions(ctx, info, perms)
-				if err != nil && f.opt.MetadataPermissions.IsSet(rwFailOK) {
-					// We've already logged the permissions errors individually here
-					fs.Debugf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
-			})
-		case "labels":
-			if !f.opt.MetadataLabels.IsSet(rwWrite) {
-				continue
-			}
-			var labels []*drive.Label
-			err := json.Unmarshal([]byte(v), &labels)
-			if err != nil {
-				return nil, fmt.Errorf("failed to unmarshal labels: %w", err)
-			}
-			// Can't set Labels on upload so need to set afterwards
-			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
-				err := f.setLabels(ctx, info, labels)
-				if err != nil && f.opt.MetadataLabels.IsSet(rwFailOK) {
-					fs.Errorf(f, "Ignoring error as failok is set: %v", err)
-					return nil
-				}
-				return err
-			})
-		case "folder-color-rgb":
-			updateInfo.FolderColorRgb = v
-		case "description":
-			updateInfo.Description = v
-		case "starred":
-			if err := parseBool(&updateInfo.Starred); err != nil {
-				return nil, err
-			}
-		case "btime":
-			if update {
-				fs.Debugf(f, "Skipping btime metadata as can't update it on an existing file: %v", v)
-			} else {
-				updateInfo.CreatedTime = v
-			}
-		case "mtime":
-			updateInfo.ModifiedTime = v
-		default:
-			if updateInfo.Properties == nil {
-				updateInfo.Properties = make(map[string]string, 1)
-			}
-			updateInfo.Properties[k] = v
-		}
-	}
-	return callback, nil
-}
-
-// Fetch metadata and update updateInfo if --metadata is in use
-func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *drive.File, update bool) (callback updateMetadataFn, err error) {
-	meta, err := fs.GetMetadataOptions(ctx, f, src, options)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read metadata from source object: %w", err)
-	}
-	callback, err = f.updateMetadata(ctx, updateInfo, meta, update)
-	if err != nil {
-		return nil, fmt.Errorf("failed to update metadata from source object: %w", err)
-	}
-	return callback, nil
-}

backend/drive/upload.go

@@ -8,21 +8,23 @@
 //
 // This contains code adapted from google.golang.org/api (C) the GO AUTHORS
 
+// +build go1.9
+
 package drive
 
 import (
-	"bytes"
-	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strconv"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/readers"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/rclone/lib/readers"
+	"github.com/pkg/errors"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
 )
@@ -50,13 +52,15 @@ type resumableUpload struct {
 }
 
 // Upload the io.Reader in of size bytes with contentType and info
-func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
+func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
 	params := url.Values{
 		"alt":        {"json"},
 		"uploadType": {"resumable"},
 		"fields":     {partialFields},
 	}
-	params.Set("supportsAllDrives", "true")
+	if f.isTeamDrive {
+		params.Set("supportsTeamDrives", "true")
+	}
 	if f.opt.KeepRevisionForever {
 		params.Set("keepRevisionForever", "true")
 	}
@@ -77,7 +81,7 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 			return false, err
 		}
 		var req *http.Request
-		req, err = http.NewRequestWithContext(ctx, method, urls, body)
+		req, err = http.NewRequest(method, urls, body)
 		if err != nil {
 			return false, err
 		}
@@ -86,15 +90,13 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 		})
 		req.Header.Set("Content-Type", "application/json; charset=UTF-8")
 		req.Header.Set("X-Upload-Content-Type", contentType)
-		if size >= 0 {
 		req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
-		}
 		res, err = f.client.Do(req)
 		if err == nil {
 			defer googleapi.CloseBody(res)
 			err = googleapi.CheckResponse(res)
 		}
-		return f.shouldRetry(ctx, err)
+		return shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -108,30 +110,60 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 		MediaType:     contentType,
 		ContentLength: size,
 	}
-	return rx.Upload(ctx)
+	return rx.Upload()
 }
 
 // Make an http.Request for the range passed in
-func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
-	req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
+func (rx *resumableUpload) makeRequest(start int64, body io.ReadSeeker, reqSize int64) *http.Request {
+	req, _ := http.NewRequest("POST", rx.URI, body)
 	req.ContentLength = reqSize
-	totalSize := "*"
-	if rx.ContentLength >= 0 {
-		totalSize = strconv.FormatInt(rx.ContentLength, 10)
-	}
 	if reqSize != 0 {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, totalSize))
+		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
 	} else {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", totalSize))
+		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength))
 	}
 	req.Header.Set("Content-Type", rx.MediaType)
 	return req
 }
 
+// rangeRE matches the transfer status response from the server. $1 is
+// the last byte index uploaded.
+var rangeRE = regexp.MustCompile(`^0\-(\d+)$`)
+
+// Query drive for the amount transferred so far
+//
+// If error is nil, then start should be valid
+func (rx *resumableUpload) transferStatus() (start int64, err error) {
+	req := rx.makeRequest(0, nil, 0)
+	res, err := rx.f.client.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer googleapi.CloseBody(res)
+	if res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
+		return rx.ContentLength, nil
+	}
+	if res.StatusCode != statusResumeIncomplete {
+		err = googleapi.CheckResponse(res)
+		if err != nil {
+			return 0, err
+		}
+		return 0, errors.Errorf("unexpected http return code %v", res.StatusCode)
+	}
+	Range := res.Header.Get("Range")
+	if m := rangeRE.FindStringSubmatch(Range); len(m) == 2 {
+		start, err = strconv.ParseInt(m[1], 10, 64)
+		if err == nil {
+			return start, nil
+		}
+	}
+	return 0, errors.Errorf("unable to parse range %q", Range)
+}
+
 // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
-func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
+func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
 	_, _ = chunk.Seek(0, io.SeekStart)
-	req := rx.makeRequest(ctx, start, chunk, chunkSize)
+	req := rx.makeRequest(start, chunk, chunkSize)
 	res, err := rx.f.client.Do(req)
 	if err != nil {
 		return 599, err
@@ -164,45 +196,23 @@ func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk
 
 // Upload uploads the chunks from the input
 // It retries each chunk using the pacer and --low-level-retries
-func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
+func (rx *resumableUpload) Upload() (*drive.File, error) {
 	start := int64(0)
 	var StatusCode int
 	var err error
 	buf := make([]byte, int(rx.f.opt.ChunkSize))
-	for finished := false; !finished; {
-		var reqSize int64
-		var chunk io.ReadSeeker
-		if rx.ContentLength >= 0 {
-			// If size known use repeatable reader for smoother bwlimit
-			if start >= rx.ContentLength {
-				break
-			}
-			reqSize = rx.ContentLength - start
+	for start < rx.ContentLength {
+		reqSize := rx.ContentLength - start
 		if reqSize >= int64(rx.f.opt.ChunkSize) {
 			reqSize = int64(rx.f.opt.ChunkSize)
 		}
-			chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
-		} else {
-			// If size unknown read into buffer
-			var n int
-			n, err = readers.ReadFill(rx.Media, buf)
-			if err == io.EOF {
-				// Send the last chunk with the correct ContentLength
-				// otherwise Google doesn't know we've finished
-				rx.ContentLength = start + int64(n)
-				finished = true
-			} else if err != nil {
-				return nil, err
-			}
-			reqSize = int64(n)
-			chunk = bytes.NewReader(buf[:reqSize])
-		}
+		chunk := readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
 
 		// Transfer the chunk
 		err = rx.f.pacer.Call(func() (bool, error) {
 			fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
-			StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
-			again, err := rx.f.shouldRetry(ctx, err)
+			StatusCode, err = rx.transferChunk(start, chunk, reqSize)
+			again, err := shouldRetry(err)
 			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
 				again = false
 				err = nil

backend/dropbox/batcher.go

@@ -1,78 +0,0 @@
-// This file contains the implementation of the sync batcher for uploads
-//
-// Dropbox rules say you can start as many batches as you want, but
-// you may only have one batch being committed and must wait for the
-// batch to be finished before committing another.
-
-package dropbox
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
-	"github.com/rclone/rclone/fs/fserrors"
-)
-
-// finishBatch commits the batch, returning a batch status to poll or maybe complete
-func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
-	var arg = &files.UploadSessionFinishBatchArg{
-		Entries: items,
-	}
-	err = f.pacer.Call(func() (bool, error) {
-		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
-		// If error is insufficient space then don't retry
-		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
-			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
-				err = fserrors.NoRetryError(err)
-				return false, err
-			}
-		}
-		// after the first chunk is uploaded, we retry everything
-		return err != nil, err
-	})
-	if err != nil {
-		return nil, fmt.Errorf("batch commit failed: %w", err)
-	}
-	return complete, nil
-}
-
-// Called by the batcher to commit a batch
-func (f *Fs) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []*files.FileMetadata, errors []error) (err error) {
-	// finalise the batch getting either a result or a job id to poll
-	complete, err := f.finishBatch(ctx, items)
-	if err != nil {
-		return err
-	}
-
-	// Check we got the right number of entries
-	entries := complete.Entries
-	if len(entries) != len(results) {
-		return fmt.Errorf("expecting %d items in batch but got %d", len(results), len(entries))
-	}
-
-	// Format results for return
-	for i := range results {
-		item := entries[i]
-		if item.Tag == "success" {
-			results[i] = item.Success
-		} else {
-			errorTag := item.Tag
-			if item.Failure != nil {
-				errorTag = item.Failure.Tag
-				if item.Failure.LookupFailed != nil {
-					errorTag += "/" + item.Failure.LookupFailed.Tag
-				}
-				if item.Failure.Path != nil {
-					errorTag += "/" + item.Failure.Path.Tag
-				}
-				if item.Failure.PropertiesError != nil {
-					errorTag += "/" + item.Failure.PropertiesError.Tag
-				}
-			}
-			errors[i] = fmt.Errorf("upload failed: %s", errorTag)
-		}
-	}
-
-	return nil
-}

backend/dropbox/dbhash/dbhash_test.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/rclone/rclone/backend/dropbox/dbhash"
+	"github.com/ncw/rclone/backend/dropbox/dbhash"
 	"github.com/stretchr/testify/assert"
 )
 

backend/dropbox/dropbox_internal_test.go

@@ -1,44 +0,0 @@
-package dropbox
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestInternalCheckPathLength(t *testing.T) {
-	rep := func(n int, r rune) (out string) {
-		rs := make([]rune, n)
-		for i := range rs {
-			rs[i] = r
-		}
-		return string(rs)
-	}
-	for _, test := range []struct {
-		in string
-		ok bool
-	}{
-		{in: "", ok: true},
-		{in: rep(maxFileNameLength, 'a'), ok: true},
-		{in: rep(maxFileNameLength+1, 'a'), ok: false},
-		{in: rep(maxFileNameLength, '£'), ok: true},
-		{in: rep(maxFileNameLength+1, '£'), ok: false},
-		{in: rep(maxFileNameLength, '☺'), ok: true},
-		{in: rep(maxFileNameLength+1, '☺'), ok: false},
-		{in: rep(maxFileNameLength, '你'), ok: true},
-		{in: rep(maxFileNameLength+1, '你'), ok: false},
-		{in: "/ok/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength, 'a') + "/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength+1, 'a') + "/ok", ok: false},
-		{in: "/ok/" + rep(maxFileNameLength, '£') + "/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength+1, '£') + "/ok", ok: false},
-		{in: "/ok/" + rep(maxFileNameLength, '☺') + "/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength+1, '☺') + "/ok", ok: false},
-		{in: "/ok/" + rep(maxFileNameLength, '你') + "/ok", ok: true},
-		{in: "/ok/" + rep(maxFileNameLength+1, '你') + "/ok", ok: false},
-	} {
-
-		err := checkPathLength(test.in)
-		assert.Equal(t, test.ok, err == nil, test.in)
-	}
-}

backend/dropbox/dropbox_test.go

@@ -4,8 +4,8 @@ package dropbox
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote

backend/fichier/api.go

@@ -1,587 +0,0 @@
-package fichier
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	429, // Too Many Requests.
-	403, // Forbidden (may happen when request limit is exceeded)
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-var errorRegex = regexp.MustCompile(`#(\d{1,3})`)
-
-func parseFichierError(err error) int {
-	matches := errorRegex.FindStringSubmatch(err.Error())
-	if len(matches) == 0 {
-		return 0
-	}
-	code, err := strconv.Atoi(matches[1])
-	if err != nil {
-		fs.Debugf(nil, "failed parsing fichier error: %v", err)
-		return 0
-	}
-	return code
-}
-
-// shouldRetry returns a boolean as to whether this resp and err
-// deserve to be retried.  It returns the err as a convenience
-func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	// 1Fichier uses HTTP error code 403 (Forbidden) for all kinds of errors with
-	// responses looking like this: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
-	//
-	// We attempt to parse the actual 1Fichier error code from this body and handle it accordingly
-	// Most importantly #374 (Flood detected: IP locked) which the integration tests provoke
-	// The list below is far from complete and should be expanded if we see any more error codes.
-	if err != nil {
-		switch parseFichierError(err) {
-		case 93:
-			return false, err // No such user
-		case 186:
-			return false, err // IP blocked?
-		case 374, 412: // Flood detected seems to be #412 now
-			fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
-			time.Sleep(30 * time.Second)
-		default:
-		}
-	}
-	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-}
-
-var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString
-
-func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, leaf string, directoryID string, err error) {
-	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
-	if err != nil {
-		return
-	}
-	// Temporary Object under construction
-	o = &Object{
-		fs:     f,
-		remote: remote,
-	}
-	return o, leaf, directoryID, nil
-}
-
-func (f *Fs) readFileInfo(ctx context.Context, url string) (*File, error) {
-	request := FileInfoRequest{
-		URL: url,
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/info.cgi",
-	}
-
-	var file File
-	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, &file)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't read file info: %w", err)
-	}
-
-	return &file, err
-}
-
-// maybe do some actual validation later if necessary
-func validToken(token *GetTokenResponse) bool {
-	return token.Status == "OK"
-}
-
-func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenResponse, error) {
-	request := DownloadRequest{
-		URL:    url,
-		Single: 1,
-		Pass:   f.opt.FilePassword,
-	}
-	if f.opt.CDN {
-		request.CDN = 1
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/download/get_token.cgi",
-	}
-
-	var token GetTokenResponse
-	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
-		doretry, err := shouldRetry(ctx, resp, err)
-		return doretry || !validToken(&token), err
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't list files: %w", err)
-	}
-
-	return &token, nil
-}
-
-func fileFromSharedFile(file *SharedFile) File {
-	return File{
-		URL:      file.Link,
-		Filename: file.Filename,
-		Size:     file.Size,
-	}
-}
-
-func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntries, err error) {
-	opts := rest.Opts{
-		Method:      "GET",
-		RootURL:     "https://1fichier.com/dir/",
-		Path:        id,
-		Parameters:  map[string][]string{"json": {"1"}},
-		ContentType: "application/x-www-form-urlencoded",
-	}
-	if f.opt.FolderPassword != "" {
-		opts.Method = "POST"
-		opts.Parameters = nil
-		opts.Body = strings.NewReader("json=1&pass=" + url.QueryEscape(f.opt.FolderPassword))
-	}
-
-	var sharedFiles SharedFolderResponse
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't list files: %w", err)
-	}
-
-	entries = make([]fs.DirEntry, len(sharedFiles))
-
-	for i, sharedFile := range sharedFiles {
-		entries[i] = f.newObjectFromFile(ctx, "", fileFromSharedFile(&sharedFile))
-	}
-
-	return entries, nil
-}
-
-func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesList, err error) {
-	// fs.Debugf(f, "Requesting files for dir `%s`", directoryID)
-	request := ListFilesRequest{
-		FolderID: directoryID,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/ls.cgi",
-	}
-
-	filesList = &FilesList{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't list files: %w", err)
-	}
-	for i := range filesList.Items {
-		item := &filesList.Items[i]
-		item.Filename = f.opt.Enc.ToStandardName(item.Filename)
-	}
-
-	return filesList, nil
-}
-
-func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *FoldersList, err error) {
-	// fs.Debugf(f, "Requesting folders for id `%s`", directoryID)
-
-	request := ListFolderRequest{
-		FolderID: directoryID,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/folder/ls.cgi",
-	}
-
-	foldersList = &FoldersList{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't list folders: %w", err)
-	}
-	foldersList.Name = f.opt.Enc.ToStandardName(foldersList.Name)
-	for i := range foldersList.SubFolders {
-		folder := &foldersList.SubFolders[i]
-		folder.Name = f.opt.Enc.ToStandardName(folder.Name)
-	}
-
-	// fs.Debugf(f, "Got FoldersList for id `%s`", directoryID)
-
-	return foldersList, err
-}
-
-func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return nil, err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return nil, err
-	}
-
-	files, err := f.listFiles(ctx, folderID)
-	if err != nil {
-		return nil, err
-	}
-
-	folders, err := f.listFolders(ctx, folderID)
-	if err != nil {
-		return nil, err
-	}
-
-	entries = make([]fs.DirEntry, len(files.Items)+len(folders.SubFolders))
-
-	for i, item := range files.Items {
-		entries[i] = f.newObjectFromFile(ctx, dir, item)
-	}
-
-	for i, folder := range folders.SubFolders {
-		createDate, err := time.Parse("2006-01-02 15:04:05", folder.CreateDate)
-		if err != nil {
-			return nil, err
-		}
-
-		fullPath := getRemote(dir, folder.Name)
-		folderID := strconv.Itoa(folder.ID)
-
-		entries[len(files.Items)+i] = fs.NewDir(fullPath, createDate).SetID(folderID)
-
-		// fs.Debugf(f, "Put Path `%s` for id `%d` into dircache", fullPath, folder.ID)
-		f.dirCache.Put(fullPath, folderID)
-	}
-
-	return entries, nil
-}
-
-func (f *Fs) newObjectFromFile(ctx context.Context, dir string, item File) *Object {
-	return &Object{
-		fs:     f,
-		remote: getRemote(dir, item.Filename),
-		file:   item,
-	}
-}
-
-func getRemote(dir, fileName string) string {
-	if dir == "" {
-		return fileName
-	}
-
-	return dir + "/" + fileName
-}
-
-func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (response *MakeFolderResponse, err error) {
-	name := f.opt.Enc.FromStandardName(leaf)
-	// fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID)
-
-	request := MakeFolderRequest{
-		FolderID: folderID,
-		Name:     name,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/folder/mkdir.cgi",
-	}
-
-	response = &MakeFolderResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't create folder: %w", err)
-	}
-
-	// fs.Debugf(f, "Created Folder `%s` in id `%s`", name, directoryID)
-
-	return response, err
-}
-
-func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (response *GenericOKResponse, err error) {
-	// fs.Debugf(f, "Removing folder with id `%s`", directoryID)
-
-	request := &RemoveFolderRequest{
-		FolderID: folderID,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/folder/rm.cgi",
-	}
-
-	response = &GenericOKResponse{}
-	var resp *http.Response
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("couldn't remove folder: %w", err)
-	}
-	if response.Status != "OK" {
-		return nil, fmt.Errorf("can't remove folder: %s", response.Message)
-	}
-
-	// fs.Debugf(f, "Removed Folder with id `%s`", directoryID)
-
-	return response, nil
-}
-
-func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKResponse, err error) {
-	request := &RemoveFileRequest{
-		Files: []RmFile{
-			{url},
-		},
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/rm.cgi",
-	}
-
-	response = &GenericOKResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("couldn't remove file: %w", err)
-	}
-
-	// fs.Debugf(f, "Removed file with url `%s`", url)
-
-	return response, nil
-}
-
-func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename string) (response *MoveFileResponse, err error) {
-	request := &MoveFileRequest{
-		URLs:     []string{url},
-		FolderID: folderID,
-		Rename:   rename,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/mv.cgi",
-	}
-
-	response = &MoveFileResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("couldn't copy file: %w", err)
-	}
-
-	return response, nil
-}
-
-func (f *Fs) moveDir(ctx context.Context, folderID int, newLeaf string, destinationFolderID int) (response *MoveDirResponse, err error) {
-	request := &MoveDirRequest{
-		FolderID:            folderID,
-		DestinationFolderID: destinationFolderID,
-		Rename:              newLeaf,
-		// DestinationUser:     destinationUser,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/folder/mv.cgi",
-	}
-
-	response = &MoveDirResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("couldn't move dir: %w", err)
-	}
-
-	return response, nil
-}
-
-func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename string) (response *CopyFileResponse, err error) {
-	request := &CopyFileRequest{
-		URLs:     []string{url},
-		FolderID: folderID,
-		Rename:   rename,
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/cp.cgi",
-	}
-
-	response = &CopyFileResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("couldn't copy file: %w", err)
-	}
-
-	return response, nil
-}
-
-func (f *Fs) renameFile(ctx context.Context, url string, newName string) (response *RenameFileResponse, err error) {
-	request := &RenameFileRequest{
-		URLs: []RenameFileURL{
-			{
-				URL:      url,
-				Filename: newName,
-			},
-		},
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/file/rename.cgi",
-	}
-
-	response = &RenameFileResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("couldn't rename file: %w", err)
-	}
-
-	return response, nil
-}
-
-func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse, err error) {
-	// fs.Debugf(f, "Requesting Upload node")
-
-	opts := rest.Opts{
-		Method:      "GET",
-		ContentType: "application/json", // 1Fichier API is bad
-		Path:        "/upload/get_upload_server.cgi",
-	}
-
-	response = &GetUploadNodeResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("didn't get an upload node: %w", err)
-	}
-
-	// fs.Debugf(f, "Got Upload node")
-
-	return response, err
-}
-
-func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string, options ...fs.OpenOption) (response *http.Response, err error) {
-	// fs.Debugf(f, "Uploading File `%s`", fileName)
-
-	fileName = f.opt.Enc.FromStandardName(fileName)
-
-	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
-		return nil, errors.New("invalid UploadID")
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/upload.cgi",
-		Parameters: map[string][]string{
-			"id": {uploadID},
-		},
-		NoResponse:           true,
-		Body:                 in,
-		ContentLength:        &size,
-		Options:              options,
-		MultipartContentName: "file[]",
-		MultipartFileName:    fileName,
-		MultipartParams: map[string][]string{
-			"did": {folderID},
-		},
-	}
-
-	if node != "" {
-		opts.RootURL = "https://" + node
-	}
-
-	err = f.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("couldn't upload file: %w", err)
-	}
-
-	// fs.Debugf(f, "Uploaded File `%s`", fileName)
-
-	return response, err
-}
-
-func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) {
-	// fs.Debugf(f, "Ending File Upload `%s`", uploadID)
-
-	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
-		return nil, errors.New("invalid UploadID")
-	}
-
-	opts := rest.Opts{
-		Method:  "GET",
-		Path:    "/end.pl",
-		RootURL: "https://" + nodeurl,
-		Parameters: map[string][]string{
-			"xid": {uploadID},
-		},
-		ExtraHeaders: map[string]string{
-			"JSON": "1",
-		},
-	}
-
-	response = &EndFileUploadResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("couldn't finish file upload: %w", err)
-	}
-
-	return response, err
-}

backend/fichier/fichier.go

@@ -1,624 +0,0 @@
-// Package fichier provides an interface to the 1Fichier storage system.
-package fichier
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/dircache"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-const (
-	rootID         = "0"
-	apiBaseURL     = "https://api.1fichier.com/v1"
-	minSleep       = 400 * time.Millisecond // api is extremely rate limited now
-	maxSleep       = 5 * time.Second
-	decayConstant  = 2 // bigger for slower decay, exponential
-	attackConstant = 0 // start with max sleep
-)
-
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "fichier",
-		Description: "1Fichier",
-		NewFs:       NewFs,
-		Options: []fs.Option{{
-			Help:      "Your API Key, get it from https://1fichier.com/console/params.pl.",
-			Name:      "api_key",
-			Sensitive: true,
-		}, {
-			Help:     "If you want to download a shared folder, add this parameter.",
-			Name:     "shared_folder",
-			Advanced: true,
-		}, {
-			Help:       "If you want to download a shared file that is password protected, add this parameter.",
-			Name:       "file_password",
-			Advanced:   true,
-			IsPassword: true,
-		}, {
-			Help:       "If you want to list the files in a shared folder that is password protected, add this parameter.",
-			Name:       "folder_password",
-			Advanced:   true,
-			IsPassword: true,
-		}, {
-			Help:     "Set if you wish to use CDN download links.",
-			Name:     "cdn",
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name:     config.ConfigEncoding,
-			Help:     config.ConfigEncodingHelp,
-			Advanced: true,
-			// Characters that need escaping
-			//
-			// 		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-			// 		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-			// 		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-			// 		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-			// 		'\'': '＇', // FULLWIDTH APOSTROPHE
-			// 		'$':  '＄', // FULLWIDTH DOLLAR SIGN
-			// 		'`':  '｀', // FULLWIDTH GRAVE ACCENT
-			//
-			// Leading space and trailing space
-			Default: (encoder.Display |
-				encoder.EncodeBackSlash |
-				encoder.EncodeSingleQuote |
-				encoder.EncodeBackQuote |
-				encoder.EncodeDoubleQuote |
-				encoder.EncodeLtGt |
-				encoder.EncodeDollar |
-				encoder.EncodeLeftSpace |
-				encoder.EncodeRightSpace |
-				encoder.EncodeInvalidUtf8),
-		}},
-	})
-}
-
-// Options defines the configuration for this backend
-type Options struct {
-	APIKey         string               `config:"api_key"`
-	SharedFolder   string               `config:"shared_folder"`
-	FilePassword   string               `config:"file_password"`
-	FolderPassword string               `config:"folder_password"`
-	CDN            bool                 `config:"cdn"`
-	Enc            encoder.MultiEncoder `config:"encoding"`
-}
-
-// Fs is the interface a cloud storage system must provide
-type Fs struct {
-	root       string
-	name       string
-	features   *fs.Features
-	opt        Options
-	dirCache   *dircache.DirCache
-	baseClient *http.Client
-	pacer      *fs.Pacer
-	rest       *rest.Client
-}
-
-// FindLeaf finds a directory of name leaf in the folder with ID pathID
-func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
-	folderID, err := strconv.Atoi(pathID)
-	if err != nil {
-		return "", false, err
-	}
-	folders, err := f.listFolders(ctx, folderID)
-	if err != nil {
-		return "", false, err
-	}
-
-	for _, folder := range folders.SubFolders {
-		if folder.Name == leaf {
-			pathIDOut := strconv.Itoa(folder.ID)
-			return pathIDOut, true, nil
-		}
-	}
-
-	return "", false, nil
-}
-
-// CreateDir makes a directory with pathID as parent and name leaf
-func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
-	folderID, err := strconv.Atoi(pathID)
-	if err != nil {
-		return "", err
-	}
-	resp, err := f.makeFolder(ctx, leaf, folderID)
-	if err != nil {
-		return "", err
-	}
-	return strconv.Itoa(resp.FolderID), err
-}
-
-// Name of the remote (as passed into NewFs)
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root of the remote (as passed into NewFs)
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String returns a description of the FS
-func (f *Fs) String() string {
-	return fmt.Sprintf("1Fichier root '%s'", f.root)
-}
-
-// Precision of the ModTimes in this Fs
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-// Hashes returns the supported hash types of the filesystem
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.Whirlpool)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// NewFs makes a new Fs object from the path
-//
-// The path is of the form remote:path
-//
-// Remotes are looked up in the config file.  If the remote isn't
-// found then NotFoundInConfigFile will be returned.
-//
-// On Windows avoid single character remote names as they can be mixed
-// up with drive letters.
-func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
-	opt := new(Options)
-	err := configstruct.Set(config, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	// If using a Shared Folder override root
-	if opt.SharedFolder != "" {
-		root = ""
-	}
-
-	//workaround for wonky parser
-	root = strings.Trim(root, "/")
-
-	f := &Fs{
-		name:       name,
-		root:       root,
-		opt:        *opt,
-		pacer:      fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
-		baseClient: &http.Client{},
-	}
-
-	f.features = (&fs.Features{
-		DuplicateFiles:          true,
-		CanHaveEmptyDirectories: true,
-		ReadMimeType:            true,
-	}).Fill(ctx, f)
-
-	client := fshttp.NewClient(ctx)
-
-	f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
-
-	f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
-
-	f.dirCache = dircache.New(root, rootID, f)
-
-	// Find the current root
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		// Assume it is a file
-		newRoot, remote := dircache.SplitPath(root)
-		tempF := *f
-		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
-		tempF.root = newRoot
-		// Make new Fs which is the parent
-		err = tempF.dirCache.FindRoot(ctx, false)
-		if err != nil {
-			// No root so return old f
-			return f, nil
-		}
-		_, err := tempF.NewObject(ctx, remote)
-		if err != nil {
-			if err == fs.ErrorObjectNotFound {
-				// File doesn't exist so return old f
-				return f, nil
-			}
-			return nil, err
-		}
-		f.features.Fill(ctx, &tempF)
-		// XXX: update the old f here instead of returning tempF, since
-		// `features` were already filled with functions having *f as a receiver.
-		// See https://github.com/rclone/rclone/issues/2182
-		f.dirCache = tempF.dirCache
-		f.root = tempF.root
-		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
-	}
-	return f, nil
-}
-
-// List the objects and directories in dir into entries.  The
-// entries can be returned in any order but should be for a
-// complete directory.
-//
-// dir should be "" to list the root, and should not have
-// trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	if f.opt.SharedFolder != "" {
-		return f.listSharedFiles(ctx, f.opt.SharedFolder)
-	}
-
-	dirContent, err := f.listDir(ctx, dir)
-	if err != nil {
-		return nil, err
-	}
-
-	return dirContent, nil
-}
-
-// NewObject finds the Object at remote.  If it can't be found
-// it returns the error ErrorObjectNotFound.
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
-	if err != nil {
-		if err == fs.ErrorDirNotFound {
-			return nil, fs.ErrorObjectNotFound
-		}
-		return nil, err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return nil, err
-	}
-	files, err := f.listFiles(ctx, folderID)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, file := range files.Items {
-		if file.Filename == leaf {
-			path, ok := f.dirCache.GetInv(directoryID)
-
-			if !ok {
-				return nil, errors.New("cannot find dir in dircache")
-			}
-
-			return f.newObjectFromFile(ctx, path, file), nil
-		}
-	}
-
-	return nil, fs.ErrorObjectNotFound
-}
-
-// Put in to the remote path with the modTime given of the given size
-//
-// When called from outside an Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
-// return an error or upload it properly (rather than e.g. calling panic).
-//
-// May create the object even if it returns an error - if so
-// will return the object and the error, otherwise will return
-// nil and the error
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.NewObject(ctx, src.Remote())
-	switch err {
-	case nil:
-		return existingObj, existingObj.Update(ctx, in, src, options...)
-	case fs.ErrorObjectNotFound:
-		// Not found so create it
-		return f.PutUnchecked(ctx, in, src, options...)
-	default:
-		return nil, err
-	}
-}
-
-// putUnchecked uploads the object with the given name and size
-//
-// This will create a duplicate if we upload a new file without
-// checking to see if there is one already - use Put() for that.
-func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
-	if size > int64(300e9) {
-		return nil, errors.New("File too big, can't upload")
-	} else if size == 0 {
-		return nil, fs.ErrorCantUploadEmptyFiles
-	}
-
-	nodeResponse, err := f.getUploadNode(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL, options...)
-	if err != nil {
-		return nil, err
-	}
-
-	fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(fileUploadResponse.Links) == 0 {
-		return nil, errors.New("upload response not found")
-	} else if len(fileUploadResponse.Links) > 1 {
-		fs.Debugf(remote, "Multiple upload responses found, using the first")
-	}
-
-	link := fileUploadResponse.Links[0]
-	fileSize, err := strconv.ParseInt(link.Size, 10, 64)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return &Object{
-		fs:     f,
-		remote: remote,
-		file: File{
-			CDN:         0,
-			Checksum:    link.Whirlpool,
-			ContentType: "",
-			Date:        time.Now().Format("2006-01-02 15:04:05"),
-			Filename:    link.Filename,
-			Pass:        0,
-			Size:        fileSize,
-			URL:         link.Download,
-		},
-	}, nil
-}
-
-// PutUnchecked uploads the object
-//
-// This will create a duplicate if we upload a new file without
-// checking to see if there is one already - use Put() for that.
-func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.putUnchecked(ctx, in, src.Remote(), src.Size(), options...)
-}
-
-// Mkdir makes the directory (container, bucket)
-//
-// Shouldn't return an error if it already exists
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	_, err := f.dirCache.FindDir(ctx, dir, true)
-	return err
-}
-
-// Rmdir removes the directory (container, bucket) if empty
-//
-// Return an error if it doesn't exist or isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		return err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return err
-	}
-
-	_, err = f.removeFolder(ctx, dir, folderID)
-	if err != nil {
-		return err
-	}
-
-	f.dirCache.FlushDir(dir)
-
-	return nil
-}
-
-// Move src to this remote using server side move operations.
-func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't move - not same remote type")
-		return nil, fs.ErrorCantMove
-	}
-	srcFs := srcObj.fs
-
-	// Find current directory ID
-	srcLeaf, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcObj.remote, false)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create temporary object
-	dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote)
-	if err != nil {
-		return nil, err
-	}
-
-	// If it is in the correct directory, just rename it
-	var url string
-	if srcDirectoryID == dstDirectoryID {
-		// No rename needed
-		if srcLeaf == dstLeaf {
-			return src, nil
-		}
-		resp, err := f.renameFile(ctx, srcObj.file.URL, dstLeaf)
-		if err != nil {
-			return nil, fmt.Errorf("couldn't rename file: %w", err)
-		}
-		if resp.Status != "OK" {
-			return nil, fmt.Errorf("couldn't rename file: %s", resp.Message)
-		}
-		url = resp.URLs[0].URL
-	} else {
-		dstFolderID, err := strconv.Atoi(dstDirectoryID)
-		if err != nil {
-			return nil, err
-		}
-		rename := dstLeaf
-		// No rename needed
-		if srcLeaf == dstLeaf {
-			rename = ""
-		}
-		resp, err := f.moveFile(ctx, srcObj.file.URL, dstFolderID, rename)
-		if err != nil {
-			return nil, fmt.Errorf("couldn't move file: %w", err)
-		}
-		if resp.Status != "OK" {
-			return nil, fmt.Errorf("couldn't move file: %s", resp.Message)
-		}
-		url = resp.URLs[0]
-	}
-
-	file, err := f.readFileInfo(ctx, url)
-	if err != nil {
-		return nil, errors.New("couldn't read file data")
-	}
-	dstObj.setMetaData(*file)
-	return dstObj, nil
-}
-
-// DirMove moves src, srcRemote to this remote at dstRemote
-// using server-side move operations.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantDirMove.
-//
-// If destination exists then return fs.ErrorDirExists.
-//
-// This is complicated by the fact that we can't use moveDir to move
-// to a different directory AND rename at the same time as it can
-// overwrite files in the source directory.
-func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
-	srcFs, ok := src.(*Fs)
-	if !ok {
-		fs.Debugf(srcFs, "Can't move directory - not same remote type")
-		return fs.ErrorCantDirMove
-	}
-
-	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
-	if err != nil {
-		return err
-	}
-	srcIDnumeric, err := strconv.Atoi(srcID)
-	if err != nil {
-		return err
-	}
-	dstDirectoryIDnumeric, err := strconv.Atoi(dstDirectoryID)
-	if err != nil {
-		return err
-	}
-
-	var resp *MoveDirResponse
-	resp, err = f.moveDir(ctx, srcIDnumeric, dstLeaf, dstDirectoryIDnumeric)
-	if err != nil {
-		return fmt.Errorf("couldn't rename leaf: %w", err)
-	}
-	if resp.Status != "OK" {
-		return fmt.Errorf("couldn't rename leaf: %s", resp.Message)
-	}
-
-	srcFs.dirCache.FlushDir(srcRemote)
-	return nil
-}
-
-// Copy src to this remote using server side move operations.
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't move - not same remote type")
-		return nil, fs.ErrorCantMove
-	}
-
-	// Create temporary object
-	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
-	if err != nil {
-		return nil, err
-	}
-
-	folderID, err := strconv.Atoi(directoryID)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := f.copyFile(ctx, srcObj.file.URL, folderID, leaf)
-	if err != nil {
-		return nil, fmt.Errorf("couldn't move file: %w", err)
-	}
-	if resp.Status != "OK" {
-		return nil, fmt.Errorf("couldn't move file: %s", resp.Message)
-	}
-
-	file, err := f.readFileInfo(ctx, resp.URLs[0].ToURL)
-	if err != nil {
-		return nil, errors.New("couldn't read file data")
-	}
-	dstObj.setMetaData(*file)
-	return dstObj, nil
-}
-
-// About gets quota information
-func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
-	opts := rest.Opts{
-		Method:      "POST",
-		Path:        "/user/info.cgi",
-		ContentType: "application/json",
-	}
-	var accountInfo AccountInfo
-	var resp *http.Response
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.rest.CallJSON(ctx, &opts, nil, &accountInfo)
-		return shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to read user info: %w", err)
-	}
-
-	// FIXME max upload size would be useful to use in Update
-	usage = &fs.Usage{
-		Used:  fs.NewUsageValue(accountInfo.ColdStorage),                                    // bytes in use
-		Total: fs.NewUsageValue(accountInfo.AvailableColdStorage),                           // bytes total
-		Free:  fs.NewUsageValue(accountInfo.AvailableColdStorage - accountInfo.ColdStorage), // bytes free
-	}
-	return usage, nil
-}
-
-// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
-	o, err := f.NewObject(ctx, remote)
-	if err != nil {
-		return "", err
-	}
-	return o.(*Object).file.URL, nil
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs              = (*Fs)(nil)
-	_ fs.Mover           = (*Fs)(nil)
-	_ fs.DirMover        = (*Fs)(nil)
-	_ fs.Copier          = (*Fs)(nil)
-	_ fs.PublicLinker    = (*Fs)(nil)
-	_ fs.PutUncheckeder  = (*Fs)(nil)
-	_ dircache.DirCacher = (*Fs)(nil)
-)

backend/fichier/fichier_test.go

@@ -1,15 +0,0 @@
-// Test 1Fichier filesystem interface
-package fichier
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFichier:",
-	})
-}

backend/fichier/object.go

@@ -1,163 +0,0 @@
-package fichier
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// Object is a filesystem like object provided by an Fs
-type Object struct {
-	fs     *Fs
-	remote string
-	file   File
-}
-
-// String returns a description of the Object
-func (o *Object) String() string {
-	return o.file.Filename
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// ModTime returns the modification date of the file
-// It should return a best guess if one isn't available
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	modTime, err := time.Parse("2006-01-02 15:04:05", o.file.Date)
-
-	if err != nil {
-		return time.Now()
-	}
-
-	return modTime
-}
-
-// Size returns the size of the file
-func (o *Object) Size() int64 {
-	return o.file.Size
-}
-
-// Fs returns read only access to the Fs that this object is part of
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Hash returns the selected checksum of the file
-// If no checksum is available it returns ""
-func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.Whirlpool {
-		return "", hash.ErrUnsupported
-	}
-
-	return o.file.Checksum, nil
-}
-
-// Storable says whether this object can be stored
-func (o *Object) Storable() bool {
-	return true
-}
-
-// SetModTime sets the metadata on the object to set the modification date
-func (o *Object) SetModTime(context.Context, time.Time) error {
-	return fs.ErrorCantSetModTime
-	//return errors.New("setting modtime is not supported for 1fichier remotes")
-}
-
-func (o *Object) setMetaData(file File) {
-	o.file = file
-}
-
-// Open opens the file for read.  Call Close() on the returned io.ReadCloser
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
-	fs.FixRangeOption(options, o.file.Size)
-	downloadToken, err := o.fs.getDownloadToken(ctx, o.file.URL)
-
-	if err != nil {
-		return nil, err
-	}
-
-	var resp *http.Response
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: downloadToken.URL,
-		Options: options,
-	}
-
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.rest.Call(ctx, &opts)
-		return shouldRetry(ctx, resp, err)
-	})
-
-	if err != nil {
-		return nil, err
-	}
-	return resp.Body, err
-}
-
-// Update in to the object with the modTime given of the given size
-//
-// When called from outside an Fs by rclone, src.Size() will always be >= 0.
-// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
-// return an error or update the object properly (rather than e.g. calling panic).
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	if src.Size() < 0 {
-		return errors.New("refusing to update with unknown size")
-	}
-
-	// upload with new size but old name
-	info, err := o.fs.putUnchecked(ctx, in, o.Remote(), src.Size(), options...)
-	if err != nil {
-		return err
-	}
-
-	// Delete duplicate after successful upload
-	err = o.Remove(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to remove old version: %w", err)
-	}
-
-	// Replace guts of old object with new one
-	*o = *info.(*Object)
-
-	return nil
-}
-
-// Remove removes this object
-func (o *Object) Remove(ctx context.Context) error {
-	// fs.Debugf(f, "Removing file `%s` with url `%s`", o.file.Filename, o.file.URL)
-
-	_, err := o.fs.deleteFile(ctx, o.file.URL)
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// MimeType of an Object if known, "" otherwise
-func (o *Object) MimeType(ctx context.Context) string {
-	return o.file.ContentType
-}
-
-// ID returns the ID of the Object if known, or "" if not
-func (o *Object) ID() string {
-	return o.file.URL
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Object    = (*Object)(nil)
-	_ fs.MimeTyper = (*Object)(nil)
-	_ fs.IDer      = (*Object)(nil)
-)