From da92c34ae934e6ff8dc960e193a63c16e82d5804 Mon Sep 17 00:00:00 2001 From: Milos Gajdos Date: Mon, 23 Oct 2023 14:52:57 +0100 Subject: [PATCH] ci: add cloud storage driver integration tests to CI * include storage integration tests in the build matrix * add a new CI job that runs E2E tests backed by S3 storage driver Signed-off-by: Milos Gajdos --- .github/workflows/build.yml | 5 +- .github/workflows/e2e.yml | 24 +++++++-- Makefile | 12 ++++- tests/conf-e2e-cloud-storage.yml | 48 +++++++++++++++++ tests/docker-compose-e2e-cloud-storage.yml | 61 ++++++++++++++++++++++ 5 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 tests/conf-e2e-cloud-storage.yml create mode 100644 tests/docker-compose-e2e-cloud-storage.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6621d621..596ebd5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,6 +28,9 @@ jobs: go: - 1.20.10 - 1.21.3 + target: + - test-coverage + - test-cloud-storage steps: - name: Checkout @@ -40,7 +43,7 @@ jobs: - name: Test run: | - make coverage + make ${{ matrix.target }} - name: Codecov uses: codecov/codecov-action@v3 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index cd0662c7..c5ce9d32 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,12 +31,26 @@ jobs: - name: Start distribution server run: | - IP=`hostname -I | awk '{print $1}'` - echo "IP=$IP" >> $GITHUB_ENV - echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json - sudo service docker restart docker run --rm -p 5000:5000 -p 5001:5001 -idt "registry:local" - name: Tests run: | - bash ./tests/push.sh $IP + bash ./tests/push.sh 127.0.0.0 + + run-e2e-test-s3-storage: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - + name: Start E2E environment + run: | + make start-e2e-s3-env + + - name: Tests + run: | + bash ./tests/push.sh 127.0.0.0 + make stop-e2e-s3-env diff --git a/Makefile b/Makefile index 2800cdd5..2723a2af 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ BINARIES=$(addprefix bin/,$(COMMANDS)) TESTFLAGS ?= -v $(TESTFLAGS_RACE) TESTFLAGS_PARALLEL ?= 8 -.PHONY: all build binaries clean test test-race test-full integration coverage validate lint validate-git validate-vendor vendor mod-outdated image +.PHONY: all build binaries clean test test-race test-full integration test-coverage validate lint validate-git validate-vendor vendor mod-outdated image .DEFAULT: all .PHONY: FORCE @@ -104,7 +104,7 @@ integration: ## run integration tests @echo "$(WHALE) $@" @go test ${TESTFLAGS} -parallel ${TESTFLAGS_PARALLEL} ${INTEGRATION_PACKAGE} -coverage: ## generate coverprofiles from the unit tests +test-coverage: ## run unit tests and generate test coverprofiles @echo "$(WHALE) $@" @rm -f coverage.txt @go test ${GO_TAGS} -i ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${COVERAGE_PACKAGES}) 2> /dev/null @@ -150,6 +150,14 @@ run-s3-tests: start-cloud-storage ## run S3 storage driver integration tests AWS_S3_FORCE_PATH_STYLE=true \ go test ${TESTFLAGS} -count=1 ./registry/storage/driver/s3-aws/... +.PHONY: start-e2e-s3-env +start-e2e-s3-env: ## starts E2E S3 storage test environment (S3, Redis, registry) + $(COMPOSE) -f tests/docker-compose-e2e-cloud-storage.yml up -d + +.PHONY: stop-e2e-s3-env +stop-e2e-s3-env: ## stops E2E S3 storage test environment (S3, Redis, registry) + $(COMPOSE) -f tests/docker-compose-e2e-cloud-storage.yml down + ##@ Validate lint: ## run all linters diff --git a/tests/conf-e2e-cloud-storage.yml b/tests/conf-e2e-cloud-storage.yml new file mode 100644 index 00000000..63a8778c --- /dev/null +++ b/tests/conf-e2e-cloud-storage.yml @@ -0,0 +1,48 @@ +version: 0.1 +http: + addr: :5000 + debug: + addr: :5001 + prometheus: + enabled: true + path: /metrics + draintimeout: 5s + secret: hmacsecret +log: + accesslog: + disabled: false + fields: + environment: local + service: registry + formatter: text + level: debug +redis: + addr: redis:6379 + db: 0 + dialtimeout: 5s + readtimeout: 10ms + writetimeout: 10ms + pool: + idletimeout: 60s + maxactive: 64 + maxidle: 16 +storage: + redirect: + disable: true + cache: + blobdescriptor: redis + maintenance: + uploadpurging: + enabled: false + s3: + region: us-east-1 + accesskey: distribution + secretkey: password + bucket: images-local + rootdirectory: /registry-v2 + regionendpoint: http://minio:9000 + encrypt: false + secure: false + chunksize: 33554432 + secure: true + v4auth: true diff --git a/tests/docker-compose-e2e-cloud-storage.yml b/tests/docker-compose-e2e-cloud-storage.yml new file mode 100644 index 00000000..887c9db9 --- /dev/null +++ b/tests/docker-compose-e2e-cloud-storage.yml @@ -0,0 +1,61 @@ +services: + minio: + image: docker.io/minio/minio:RELEASE.2023-10-16T04-13-43Z + command: server /data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 5s + retries: 10 + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: distribution + MINIO_ROOT_PASSWORD: password + volumes: + - ./miniodata/distribution:/data:Z + + minio-init: + image: docker.io/minio/mc:RELEASE.2023-10-14T01-57-03Z + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/bash -c " + /usr/bin/mc config host add minio http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD} && ( + /usr/bin/mc stat minio/images-local || /usr/bin/mc mb minio/images-local + ) && /usr/bin/mc anonymous set public minio/images-local" + environment: + MINIO_ROOT_USER: distribution + MINIO_ROOT_PASSWORD: password + + redis: + image: redis:7.2-alpine + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + ports: + - "6379:6379" + + registry: + build: + context: ../. + dockerfile: ./Dockerfile + command: + - "serve" + - "/etc/docker/registry/config-test.yml" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5001/debug/health"] + interval: 5s + timeout: 5s + retries: 10 + depends_on: + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + ports: + - "5000:5000" + - "5001:5001" + volumes: + - ./conf-e2e-cloud-storage.yml:/etc/docker/registry/config-test.yml