ci: add cloud storage driver integration tests to CI (#4121)

.github/workflows/build.yml

@@ -28,6 +28,9 @@ jobs:
         go:
           - 1.20.10
           - 1.21.3
+        target:
+          - test-coverage
+          - test-cloud-storage
     steps:
       -
         name: Checkout
@@ -40,7 +43,7 @@ jobs:
       -
         name: Test
         run: |
-          make coverage
+          make ${{ matrix.target }}
       -
         name: Codecov
         uses: codecov/codecov-action@v3

.github/workflows/e2e.yml

@@ -31,12 +31,26 @@ jobs:
       -
         name: Start distribution server
         run: |
-          IP=`hostname -I | awk '{print $1}'`
-          echo "IP=$IP" >> $GITHUB_ENV
-          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
-          sudo service docker restart
           docker run --rm -p 5000:5000 -p 5001:5001 -idt "registry:local"
       -
         name: Tests
         run: |
-          bash ./tests/push.sh $IP
+          bash ./tests/push.sh 127.0.0.0
+
+  run-e2e-test-s3-storage:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Start E2E environment
+        run: |
+          make start-e2e-s3-env
+
+      - name: Tests
+        run: |
+          bash ./tests/push.sh 127.0.0.0
+          make stop-e2e-s3-env

Makefile

@@ -45,7 +45,7 @@ BINARIES=$(addprefix bin/,$(COMMANDS))
 TESTFLAGS ?= -v $(TESTFLAGS_RACE)
 TESTFLAGS_PARALLEL ?= 8
 
-.PHONY: all build binaries clean test test-race test-full integration coverage validate lint validate-git validate-vendor vendor mod-outdated image
+.PHONY: all build binaries clean test test-race test-full integration test-coverage validate lint validate-git validate-vendor vendor mod-outdated image
 .DEFAULT: all
 
 .PHONY: FORCE
@@ -104,7 +104,7 @@ integration: ## run integration tests
 	@echo "$(WHALE) $@"
 	@go test ${TESTFLAGS} -parallel ${TESTFLAGS_PARALLEL} ${INTEGRATION_PACKAGE}
 
-coverage: ## generate coverprofiles from the unit tests
+test-coverage: ## run unit tests and generate test coverprofiles
 	@echo "$(WHALE) $@"
 	@rm -f coverage.txt
 	@go test ${GO_TAGS} -i ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${COVERAGE_PACKAGES}) 2> /dev/null
@@ -150,6 +150,14 @@ run-s3-tests: start-cloud-storage ## run S3 storage driver integration tests
 	AWS_S3_FORCE_PATH_STYLE=true \
 	go test ${TESTFLAGS} -count=1 ./registry/storage/driver/s3-aws/...
 
+.PHONY: start-e2e-s3-env
+start-e2e-s3-env: ## starts E2E S3 storage test environment (S3, Redis, registry)
+	$(COMPOSE) -f tests/docker-compose-e2e-cloud-storage.yml up -d
+
+.PHONY: stop-e2e-s3-env
+stop-e2e-s3-env: ## stops E2E S3 storage test environment (S3, Redis, registry)
+	$(COMPOSE) -f tests/docker-compose-e2e-cloud-storage.yml down
+
 ##@ Validate
 
 lint: ## run all linters

tests/conf-e2e-cloud-storage.yml

@@ -0,0 +1,48 @@
+version: 0.1
+http:
+    addr: :5000
+    debug:
+      addr: :5001
+      prometheus:
+        enabled: true
+        path: /metrics
+    draintimeout: 5s
+    secret: hmacsecret
+log:
+    accesslog:
+      disabled: false
+    fields:
+      environment: local
+      service: registry
+    formatter: text
+    level: debug
+redis:
+   addr: redis:6379
+   db: 0
+   dialtimeout: 5s
+   readtimeout: 10ms
+   writetimeout: 10ms
+   pool:
+     idletimeout: 60s
+     maxactive: 64
+     maxidle: 16
+storage:
+  redirect:
+    disable: true
+  cache:
+    blobdescriptor: redis
+  maintenance:
+    uploadpurging:
+      enabled: false
+  s3:
+    region: us-east-1
+    accesskey: distribution
+    secretkey: password
+    bucket: images-local
+    rootdirectory: /registry-v2
+    regionendpoint: http://minio:9000
+    encrypt: false
+    secure: false
+    chunksize: 33554432
+    secure: true
+    v4auth: true

tests/docker-compose-e2e-cloud-storage.yml

@@ -0,0 +1,61 @@
+services:
+  minio:
+    image: docker.io/minio/minio:RELEASE.2023-10-16T04-13-43Z
+    command: server /data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: distribution
+      MINIO_ROOT_PASSWORD: password
+    volumes:
+      - ./miniodata/distribution:/data:Z
+
+  minio-init:
+    image: docker.io/minio/mc:RELEASE.2023-10-14T01-57-03Z
+    depends_on:
+      minio:
+        condition: service_healthy
+    entrypoint: >
+      /bin/bash -c "
+        /usr/bin/mc config host add minio http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD} && (
+          /usr/bin/mc stat minio/images-local || /usr/bin/mc mb minio/images-local
+        ) && /usr/bin/mc anonymous set public minio/images-local"
+    environment:
+      MINIO_ROOT_USER: distribution
+      MINIO_ROOT_PASSWORD: password
+
+  redis:
+    image: redis:7.2-alpine
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+    ports:
+      - "6379:6379"
+
+  registry:
+    build:
+      context: ../.
+      dockerfile: ./Dockerfile
+    command:
+      - "serve"
+      - "/etc/docker/registry/config-test.yml"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5001/debug/health"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    depends_on:
+      minio:
+        condition: service_healthy
+      minio-init:
+        condition: service_completed_successfully
+    ports:
+      - "5000:5000"
+      - "5001:5001"
+    volumes:
+      - ./conf-e2e-cloud-storage.yml:/etc/docker/registry/config-test.yml