diff --git a/.gitignore b/.gitignore index 4cf7888e..dcda068a 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ bin/* *.sublime-project *.sublime-workspace .idea/* + +tests/miniodata diff --git a/BUILDING.md b/BUILDING.md index 75487bbc..aa1bcff0 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -11,27 +11,25 @@ Most people should use the [official Registry docker image](https://hub.docker.c People looking for advanced operational use cases might consider rolling their own image with a custom Dockerfile inheriting `FROM registry:2`. -OS X users who want to run natively can do so following [the instructions here](https://github.com/docker/docker.github.io/blob/master/registry/recipes/osx-setup-guide.md). +The latest updates to `main` branch are automatically pushed to [distribution Docker Hub repository](https://hub.docker.com/r/distribution/distribution) and tagged with `edge` tag. ### Gotchas -You are expected to know your way around with go & git. +You are expected to know your way around with `go` & `git`. -If you are a casual user with no development experience, and no preliminary knowledge of go, building from source is probably not a good solution for you. +If you are a casual user with no development experience, and no preliminary knowledge of Go, building from source is probably not a good solution for you. ## Configure the development environment The first prerequisite of properly building distribution targets is to have a Go -development environment setup. Please follow [How to Write Go Code](https://golang.org/doc/code.html) -for proper setup. If done correctly, you should have a GOROOT and GOPATH set in the -environment. +development environment setup. Please follow [How to Write Go Code](https://go.dev/doc/code) for proper setup. Next, fetch the code from the repository using git: git clone https://github.com/distribution/distribution cd distribution -If you are planning to create a pull request with changes, you may want to clone directly from your [fork](https://help.github.com/en/articles/about-forks). +If you are planning to create a pull request with changes, you may want to clone directly from your [fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks). ## Build and run from source @@ -105,3 +103,29 @@ the environment variable `BUILDTAGS`.
include_gcs
Adds support for Google Cloud Storage
+ +### Local cloud storage environment + +You can run an S3 API compatible storage locally with [minio](https://min.io/). + +You must have a [docker compose](https://docs.docker.com/compose/) compatible tool installed on your workstation. + +Start the local cloud environment: +``` +make start-cloud-storage +``` +There is a sample registry configuration file that lets you point the registry to the started storage: +``` +AWS_ACCESS_KEY=distribution \ + AWS_SECRET_KEY=password \ + AWS_REGION=us-east-1 \ + S3_BUCKET=images-local \ + S3_ENCRYPT=false \ + REGION_ENDPOINT=http://127.0.0.1:9000 \ + S3_SECURE=false \ +./bin/registry serve tests/conf-local-cloud.yml +``` +Stop the local storage when done: +``` +make stop-cloud-storage +``` diff --git a/Makefile b/Makefile index d2059afe..2800cdd5 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) VERSION ?= $(shell git describe --match 'v[0-9]*' --dirty='.m' --always) REVISION ?= $(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) +# default compose command +COMPOSE ?= docker compose PKG=github.com/distribution/distribution/v3 @@ -117,6 +119,37 @@ coverage: ## generate coverprofiles from the unit tests fi; \ done ) +.PHONY: test-cloud-storage +test-cloud-storage: start-cloud-storage run-s3-tests stop-cloud-storage ## run cloud storage driver tests + +.PHONY: start-cloud-storage +start-cloud-storage: ## start local cloud storage (minio) + $(COMPOSE) -f tests/docker-compose-storage.yml up minio minio-init -d + +.PHONY: stop-cloud-storage +stop-cloud-storage: ## stop local cloud storage (minio) + $(COMPOSE) -f tests/docker-compose-storage.yml down + +.PHONY: reset-cloud-storage +reset-cloud-storage: ## reset (stop, delete, start) local cloud storage (minio) + $(COMPOSE) -f tests/docker-compose-storage.yml down + @mkdir -p tests/miniodata/distribution + @rm -rf tests/miniodata/distribution/* tests/miniodata/.minio.sys + $(COMPOSE) -f tests/docker-compose-storage.yml up minio minio-init -d + +.PHONY: run-s3-tests +run-s3-tests: start-cloud-storage ## run S3 storage driver integration tests + AWS_ACCESS_KEY=distribution \ + AWS_SECRET_KEY=password \ + AWS_REGION=us-east-1 \ + S3_BUCKET=images-local \ + S3_ENCRYPT=false \ + REGION_ENDPOINT=http://127.0.0.1:9000 \ + S3_SECURE=false \ + S3_ACCELERATE=false \ + AWS_S3_FORCE_PATH_STYLE=true \ + go test ${TESTFLAGS} -count=1 ./registry/storage/driver/s3-aws/... + ##@ Validate lint: ## run all linters @@ -133,7 +166,7 @@ validate-vendor: ## validate vendor .PHONY: help help: - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_\/%-]+:.*?##/ { printf " \033[36m%-27s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_\/%-]+:.*?##/ { printf " \033[36m%-27s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @echo "" @echo "Go binaries: $(BINARIES)" @echo "Docker image: $(IMAGE_NAME)" diff --git a/README.md b/README.md index 9654b797..01c3da29 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,6 @@ [![OCI Conformance](https://github.com/distribution/distribution/workflows/conformance/badge.svg)](https://github.com/distribution/distribution/actions?query=workflow%3Aconformance) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/distribution/distribution/badge)](https://securityscorecards.dev/viewer/?uri=github.com/distribution/distribution) - - The toolset to pack, ship, store, and deliver content. This repository's main product is the Open Source Registry implementation diff --git a/cmd/registry/config-dev.yml b/cmd/registry/config-dev.yml index 8acf92ea..985892b1 100644 --- a/cmd/registry/config-dev.yml +++ b/cmd/registry/config-dev.yml @@ -63,7 +63,7 @@ notifications: timeout: 1s threshold: 10 backoff: 1s - disabled: true + disabled: true health: storagedriver: enabled: true diff --git a/tests/conf-local-cloud.yml b/tests/conf-local-cloud.yml new file mode 100644 index 00000000..5e82f8cf --- /dev/null +++ b/tests/conf-local-cloud.yml @@ -0,0 +1,38 @@ +version: 0.1 +http: + addr: :5000 + debug: + addr: :5001 + prometheus: + enabled: true + path: /metrics + draintimeout: 5s + secret: hmacsecret +log: + accesslog: + disabled: false + fields: + environment: local + service: registry + formatter: text + level: debug +storage: + delete: + enabled: true + cache: + blobdescriptor: inmemory + maintenance: + uploadpurging: + enabled: false + s3: + region: us-east-1 + accesskey: distribution + secretkey: password + bucket: images-local + rootdirectory: /registry-v2 + regionendpoint: http://127.0.0.1:9000 + encrypt: false + secure: false + chunksize: 33554432 + secure: true + v4auth: true diff --git a/tests/docker-compose-storage.yml b/tests/docker-compose-storage.yml new file mode 100644 index 00000000..7377abcd --- /dev/null +++ b/tests/docker-compose-storage.yml @@ -0,0 +1,31 @@ +services: + minio: + image: docker.io/minio/minio:RELEASE.2023-09-20T22-49-55Z + command: server /data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 5s + retries: 10 + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: distribution + MINIO_ROOT_PASSWORD: password + volumes: + - ./miniodata/distribution:/data:Z + + minio-init: + image: docker.io/minio/mc:RELEASE.2023-02-16T19-20-11Z + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/bash -c " + /usr/bin/mc config host add minio http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD} && ( + /usr/bin/mc stat minio/images-local || /usr/bin/mc mb minio/images-local + ) && /usr/bin/mc anonymous set public minio/images-local" + environment: + MINIO_ROOT_USER: distribution + MINIO_ROOT_PASSWORD: password