From 7105919f0cb6101592c5866b099cda6c739160cc Mon Sep 17 00:00:00 2001 From: Grigory Entin Date: Tue, 21 Jun 2022 00:06:55 +0200 Subject: [PATCH] Added support for chunked uploads. (#1208) * Added tests for mid-size and big artifacts, reproducing a problem with chunked uploads. * Added support for chunked uploads. * Enforced overwriting uploaded artifacts on receiving the first chunk. Co-authored-by: Casey Lee --- pkg/artifacts/server.go | 27 +++++- pkg/artifacts/server_test.go | 10 ++ .../upload-and-download/artifacts.yml | 92 ++++++++++++++++++- 3 files changed, 126 insertions(+), 3 deletions(-) diff --git a/pkg/artifacts/server.go b/pkg/artifacts/server.go index c39c921..a470fb4 100644 --- a/pkg/artifacts/server.go +++ b/pkg/artifacts/server.go @@ -49,6 +49,7 @@ type MkdirFS interface { fs.FS MkdirAll(path string, perm fs.FileMode) error Open(name string) (fs.File, error) + OpenAtEnd(name string) (fs.File, error) } type MkdirFsImpl struct { @@ -61,7 +62,22 @@ func (fsys MkdirFsImpl) MkdirAll(path string, perm fs.FileMode) error { } func (fsys MkdirFsImpl) Open(name string) (fs.File, error) { - return os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR, 0644) + return os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) +} + +func (fsys MkdirFsImpl) OpenAtEnd(name string) (fs.File, error) { + file, err := os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR, 0644) + + if err != nil { + return nil, err + } + + _, err = file.Seek(0, os.SEEK_END) + if err != nil { + return nil, err + } + + return file, nil } var gzipExtension = ".gz__" @@ -98,7 +114,14 @@ func uploads(router *httprouter.Router, fsys MkdirFS) { panic(err) } - file, err := fsys.Open(filePath) + file, err := func() (fs.File, error) { + contentRange := req.Header.Get("Content-Range") + if contentRange != "" && !strings.HasPrefix(contentRange, "bytes 0-") { + return fsys.OpenAtEnd(filePath) + } + return fsys.Open(filePath) + }() + if err != nil { panic(err) } diff --git a/pkg/artifacts/server_test.go b/pkg/artifacts/server_test.go index 9244c8b..f1c09a3 100644 --- a/pkg/artifacts/server_test.go +++ b/pkg/artifacts/server_test.go @@ -51,6 +51,16 @@ func (fsys MapFsImpl) Open(path string) (fs.File, error) { return WritableFile{result, fsys.MapFS, path}, err } +func (fsys MapFsImpl) OpenAtEnd(path string) (fs.File, error) { + var file = fstest.MapFile{ + Data: []byte("content2"), + } + fsys.MapFS[path] = &file + + result, err := fsys.MapFS.Open(path) + return WritableFile{result, fsys.MapFS, path}, err +} + func TestNewArtifactUploadPrepare(t *testing.T) { assert := assert.New(t) diff --git a/pkg/artifacts/testdata/upload-and-download/artifacts.yml b/pkg/artifacts/testdata/upload-and-download/artifacts.yml index cfebc70..a24c061 100644 --- a/pkg/artifacts/testdata/upload-and-download/artifacts.yml +++ b/pkg/artifacts/testdata/upload-and-download/artifacts.yml @@ -26,10 +26,17 @@ jobs: run: | mkdir -p path/to/dir-1 mkdir -p path/to/dir-2 - mkdir -p path/to/dir-3 + mkdir -p path/to/dir-3 + mkdir -p path/to/dir-5 + mkdir -p path/to/dir-6 + mkdir -p path/to/dir-7 echo "Lorem ipsum dolor sit amet" > path/to/dir-1/file1.txt echo "Hello world from file #2" > path/to/dir-2/file2.txt echo "This is a going to be a test for a large enough file that should get compressed with GZip. The @actions/artifact package uses GZip to upload files. This text should have a compression ratio greater than 100% so it should get uploaded using GZip" > path/to/dir-3/gzip.txt + dd if=/dev/random of=path/to/dir-5/file5.rnd bs=1024 count=1024 + dd if=/dev/random of=path/to/dir-6/file6.rnd bs=1024 count=$((10*1024)) + dd if=/dev/random of=path/to/dir-7/file7.rnd bs=1024 count=$((10*1024)) + # Upload a single file artifact - name: 'Upload artifact #1' uses: actions/upload-artifact@v2 @@ -59,6 +66,35 @@ jobs: path/to/dir-1/* path/to/dir-[23]/* !path/to/dir-3/*.txt + + # Upload a mid-size file artifact + - name: 'Upload artifact #5' + uses: actions/upload-artifact@v2 + with: + name: 'Mid-Size-Artifact' + path: path/to/dir-5/file5.rnd + + # Upload a big file artifact + - name: 'Upload artifact #6' + uses: actions/upload-artifact@v2 + with: + name: 'Big-Artifact' + path: path/to/dir-6/file6.rnd + + # Upload a big file artifact twice + - name: 'Upload artifact #7 (First)' + uses: actions/upload-artifact@v2 + with: + name: 'Big-Uploaded-Twice' + path: path/to/dir-7/file7.rnd + + # Upload a big file artifact twice + - name: 'Upload artifact #7 (Second)' + uses: actions/upload-artifact@v2 + with: + name: 'Big-Uploaded-Twice' + path: path/to/dir-7/file7.rnd + # Verify artifacts. Switch to download-artifact@v2 once it's out of preview # Download Artifact #1 and verify the correctness of the content @@ -138,3 +174,57 @@ jobs: echo "File contents of downloaded artifacts are incorrect" exit 1 fi + + - name: 'Download artifact #5' + uses: actions/download-artifact@v2 + with: + name: 'Mid-Size-Artifact' + path: mid-size/artifact/path + + - name: 'Verify Artifact #5' + run: | + file="mid-size/artifact/path/file5.rnd" + if [ ! -f $file ] ; then + echo "Expected file does not exist" + exit 1 + fi + if ! diff $file path/to/dir-5/file5.rnd ; then + echo "File contents of downloaded artifact are incorrect" + exit 1 + fi + + - name: 'Download artifact #6' + uses: actions/download-artifact@v2 + with: + name: 'Big-Artifact' + path: big/artifact/path + + - name: 'Verify Artifact #6' + run: | + file="big/artifact/path/file6.rnd" + if [ ! -f $file ] ; then + echo "Expected file does not exist" + exit 1 + fi + if ! diff $file path/to/dir-6/file6.rnd ; then + echo "File contents of downloaded artifact are incorrect" + exit 1 + fi + + - name: 'Download artifact #7' + uses: actions/download-artifact@v2 + with: + name: 'Big-Uploaded-Twice' + path: big-uploaded-twice/artifact/path + + - name: 'Verify Artifact #7' + run: | + file="big-uploaded-twice/artifact/path/file7.rnd" + if [ ! -f $file ] ; then + echo "Expected file does not exist" + exit 1 + fi + if ! diff $file path/to/dir-7/file7.rnd ; then + echo "File contents of downloaded artifact are incorrect" + exit 1 + fi