distribution/registry/registry.go

491 lines
15 KiB
Go
Raw Normal View History

package registry
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
logrus_bugsnag "github.com/Shopify/logrus-bugsnag"
logstash "github.com/bshuster-repo/logrus-logstash-hook"
"github.com/bugsnag/bugsnag-go"
"github.com/docker/go-metrics"
gorhandlers "github.com/gorilla/handlers"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/yvasiyarov/gorelic"
"golang.org/x/crypto/acme"
"golang.org/x/crypto/acme/autocert"
"github.com/distribution/distribution/v3/configuration"
dcontext "github.com/distribution/distribution/v3/context"
"github.com/distribution/distribution/v3/health"
"github.com/distribution/distribution/v3/registry/handlers"
"github.com/distribution/distribution/v3/registry/listener"
"github.com/distribution/distribution/v3/uuid"
"github.com/distribution/distribution/v3/version"
)
// a map of TLS cipher suite names to constants in https://golang.org/pkg/crypto/tls/#pkg-constants
var cipherSuites = map[string]uint16{
// TLS 1.0 - 1.2 cipher suites
"TLS_RSA_WITH_3DES_EDE_CBC_SHA": tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
"TLS_RSA_WITH_AES_128_CBC_SHA": tls.TLS_RSA_WITH_AES_128_CBC_SHA,
"TLS_RSA_WITH_AES_256_CBC_SHA": tls.TLS_RSA_WITH_AES_256_CBC_SHA,
"TLS_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
"TLS_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256": tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
// TLS 1.3 cipher suites
"TLS_AES_128_GCM_SHA256": tls.TLS_AES_128_GCM_SHA256,
"TLS_AES_256_GCM_SHA384": tls.TLS_AES_256_GCM_SHA384,
"TLS_CHACHA20_POLY1305_SHA256": tls.TLS_CHACHA20_POLY1305_SHA256,
}
// a list of default ciphersuites to utilize
var defaultCipherSuites = []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_AES_128_GCM_SHA256,
tls.TLS_CHACHA20_POLY1305_SHA256,
tls.TLS_AES_256_GCM_SHA384,
}
const defaultTLSVersionStr = "tls1.2"
// tlsVersions maps user-specified values to tls version constants.
var tlsVersions = map[string]uint16{
"tls1.2": tls.VersionTLS12,
"tls1.3": tls.VersionTLS13,
}
// defaultLogFormatter is the default formatter to use for logs.
const defaultLogFormatter = "text"
// this channel gets notified when process receives signal. It is global to ease unit testing
var quit = make(chan os.Signal, 1)
// ServeCmd is a cobra command for running the registry.
var ServeCmd = &cobra.Command{
Use: "serve <config>",
Short: "`serve` stores and distributes Docker images",
Long: "`serve` stores and distributes Docker images.",
Run: func(cmd *cobra.Command, args []string) {
// setup context
ctx := dcontext.WithVersion(dcontext.Background(), version.Version)
config, err := resolveConfiguration(args)
if err != nil {
fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
cmd.Usage()
os.Exit(1)
}
if config.HTTP.Debug.Addr != "" {
go func(addr string) {
logrus.Infof("debug server listening %v", addr)
if err := http.ListenAndServe(addr, nil); err != nil {
logrus.Fatalf("error listening on debug interface: %v", err)
}
}(config.HTTP.Debug.Addr)
}
registry, err := NewRegistry(ctx, config)
if err != nil {
logrus.Fatalln(err)
}
if config.HTTP.Debug.Prometheus.Enabled {
path := config.HTTP.Debug.Prometheus.Path
if path == "" {
path = "/metrics"
}
logrus.Info("providing prometheus metrics on ", path)
http.Handle(path, metrics.Handler())
}
if err = registry.ListenAndServe(); err != nil {
logrus.Fatalln(err)
}
},
}
// A Registry represents a complete instance of the registry.
//
// TODO(aaronl): It might make sense for Registry to become an interface.
type Registry struct {
config *configuration.Configuration
app *handlers.App
server *http.Server
}
// NewRegistry creates a new registry from a context and configuration struct.
func NewRegistry(ctx context.Context, config *configuration.Configuration) (*Registry, error) {
var err error
ctx, err = configureLogging(ctx, config)
if err != nil {
return nil, fmt.Errorf("error configuring logger: %v", err)
}
configureBugsnag(config)
// inject a logger into the uuid library. warns us if there is a problem
// with uuid generation under low entropy.
uuid.Loggerf = dcontext.GetLogger(ctx).Warnf
app := handlers.NewApp(ctx, config)
// TODO(aaronl): The global scope of the health checks means NewRegistry
// can only be called once per process.
app.RegisterHealthChecks()
handler := configureReporting(app)
handler = alive("/", handler)
handler = health.Handler(handler)
handler = panicHandler(handler)
if !config.Log.AccessLog.Disabled {
handler = gorhandlers.CombinedLoggingHandler(os.Stdout, handler)
}
server := &http.Server{
Handler: handler,
}
return &Registry{
app: app,
config: config,
server: server,
}, nil
}
// takes a list of cipher suites and converts it to a list of respective tls constants
// if an empty list is provided, then the defaults will be used
func getCipherSuites(names []string) ([]uint16, error) {
if len(names) == 0 {
return defaultCipherSuites, nil
}
cipherSuiteConsts := make([]uint16, len(names))
for i, name := range names {
cipherSuiteConst, ok := cipherSuites[name]
if !ok {
return nil, fmt.Errorf("unknown TLS cipher suite '%s' specified for http.tls.cipherSuites", name)
}
cipherSuiteConsts[i] = cipherSuiteConst
}
return cipherSuiteConsts, nil
}
// takes a list of cipher suite ids and converts it to a list of respective names
func getCipherSuiteNames(ids []uint16) []string {
if len(ids) == 0 {
return nil
}
names := make([]string, len(ids))
for i, id := range ids {
names[i] = tls.CipherSuiteName(id)
}
return names
}
// ListenAndServe runs the registry's HTTP server.
func (registry *Registry) ListenAndServe() error {
config := registry.config
ln, err := listener.NewListener(config.HTTP.Net, config.HTTP.Addr)
if err != nil {
return err
}
if config.HTTP.TLS.Certificate != "" || config.HTTP.TLS.LetsEncrypt.CacheFile != "" {
if config.HTTP.TLS.MinimumTLS == "" {
config.HTTP.TLS.MinimumTLS = defaultTLSVersionStr
}
tlsMinVersion, ok := tlsVersions[config.HTTP.TLS.MinimumTLS]
if !ok {
return fmt.Errorf("unknown minimum TLS level '%s' specified for http.tls.minimumtls", config.HTTP.TLS.MinimumTLS)
}
dcontext.GetLogger(registry.app).Infof("restricting TLS version to %s or higher", config.HTTP.TLS.MinimumTLS)
var tlsCipherSuites []uint16
// configuring cipher suites are no longer supported after the tls1.3.
// (https://go.dev/blog/tls-cipher-suites)
if tlsMinVersion > tls.VersionTLS12 {
dcontext.GetLogger(registry.app).Warnf("restricting TLS cipher suites to empty. Because configuring cipher suites is no longer supported in %s", config.HTTP.TLS.MinimumTLS)
} else {
tlsCipherSuites, err = getCipherSuites(config.HTTP.TLS.CipherSuites)
if err != nil {
return err
}
dcontext.GetLogger(registry.app).Infof("restricting TLS cipher suites to: %s", strings.Join(getCipherSuiteNames(tlsCipherSuites), ","))
}
tlsConf := &tls.Config{
ClientAuth: tls.NoClientCert,
NextProtos: nextProtos(config),
MinVersion: tlsMinVersion,
PreferServerCipherSuites: true,
CipherSuites: tlsCipherSuites,
}
if config.HTTP.TLS.LetsEncrypt.CacheFile != "" {
if config.HTTP.TLS.Certificate != "" {
return fmt.Errorf("cannot specify both certificate and Let's Encrypt")
}
m := &autocert.Manager{
HostPolicy: autocert.HostWhitelist(config.HTTP.TLS.LetsEncrypt.Hosts...),
Cache: autocert.DirCache(config.HTTP.TLS.LetsEncrypt.CacheFile),
Email: config.HTTP.TLS.LetsEncrypt.Email,
Prompt: autocert.AcceptTOS,
}
tlsConf.GetCertificate = m.GetCertificate
tlsConf.NextProtos = append(tlsConf.NextProtos, acme.ALPNProto)
} else {
tlsConf.Certificates = make([]tls.Certificate, 1)
tlsConf.Certificates[0], err = tls.LoadX509KeyPair(config.HTTP.TLS.Certificate, config.HTTP.TLS.Key)
if err != nil {
return err
}
}
if len(config.HTTP.TLS.ClientCAs) != 0 {
pool := x509.NewCertPool()
for _, ca := range config.HTTP.TLS.ClientCAs {
caPem, err := os.ReadFile(ca)
if err != nil {
return err
}
if ok := pool.AppendCertsFromPEM(caPem); !ok {
return fmt.Errorf("could not add CA to pool")
}
}
for _, subj := range pool.Subjects() {
dcontext.GetLogger(registry.app).Debugf("CA Subject: %s", string(subj))
}
tlsConf.ClientAuth = tls.RequireAndVerifyClientCert
tlsConf.ClientCAs = pool
}
ln = tls.NewListener(ln, tlsConf)
dcontext.GetLogger(registry.app).Infof("listening on %v, tls", ln.Addr())
} else {
dcontext.GetLogger(registry.app).Infof("listening on %v", ln.Addr())
}
if config.HTTP.DrainTimeout == 0 {
return registry.server.Serve(ln)
}
// setup channel to get notified on SIGTERM signal
signal.Notify(quit, syscall.SIGTERM)
serveErr := make(chan error)
// Start serving in goroutine and listen for stop signal in main thread
go func() {
serveErr <- registry.server.Serve(ln)
}()
select {
case err := <-serveErr:
return err
case <-quit:
dcontext.GetLogger(registry.app).Info("stopping server gracefully. Draining connections for ", config.HTTP.DrainTimeout)
// shutdown the server with a grace period of configured timeout
c, cancel := context.WithTimeout(context.Background(), config.HTTP.DrainTimeout)
defer cancel()
return registry.server.Shutdown(c)
}
}
func configureReporting(app *handlers.App) http.Handler {
var handler http.Handler = app
if app.Config.Reporting.Bugsnag.APIKey != "" {
handler = bugsnag.Handler(handler)
}
if app.Config.Reporting.NewRelic.LicenseKey != "" {
agent := gorelic.NewAgent()
agent.NewrelicLicense = app.Config.Reporting.NewRelic.LicenseKey
if app.Config.Reporting.NewRelic.Name != "" {
agent.NewrelicName = app.Config.Reporting.NewRelic.Name
}
agent.CollectHTTPStat = true
agent.Verbose = app.Config.Reporting.NewRelic.Verbose
agent.Run()
handler = agent.WrapHTTPHandler(handler)
}
return handler
}
// configureLogging prepares the context with a logger using the
// configuration.
func configureLogging(ctx context.Context, config *configuration.Configuration) (context.Context, error) {
logrus.SetLevel(logLevel(config.Log.Level))
formatter := config.Log.Formatter
if formatter == "" {
formatter = defaultLogFormatter
}
switch formatter {
case "json":
logrus.SetFormatter(&logrus.JSONFormatter{
TimestampFormat: time.RFC3339Nano,
DisableHTMLEscape: true,
})
case "text":
logrus.SetFormatter(&logrus.TextFormatter{
TimestampFormat: time.RFC3339Nano,
})
case "logstash":
logrus.SetFormatter(&logstash.LogstashFormatter{
Formatter: &logrus.JSONFormatter{TimestampFormat: time.RFC3339Nano},
})
default:
return ctx, fmt.Errorf("unsupported logging formatter: %q", formatter)
}
logrus.Debugf("using %q logging formatter", formatter)
if len(config.Log.Fields) > 0 {
// build up the static fields, if present.
var fields []interface{}
for k := range config.Log.Fields {
fields = append(fields, k)
}
ctx = dcontext.WithValues(ctx, config.Log.Fields)
ctx = dcontext.WithLogger(ctx, dcontext.GetLogger(ctx, fields...))
}
dcontext.SetDefaultLogger(dcontext.GetLogger(ctx))
return ctx, nil
}
func logLevel(level configuration.Loglevel) logrus.Level {
l, err := logrus.ParseLevel(string(level))
if err != nil {
l = logrus.InfoLevel
logrus.Warnf("error parsing level %q: %v, using %q ", level, err, l)
}
return l
}
// configureBugsnag configures bugsnag reporting, if enabled
func configureBugsnag(config *configuration.Configuration) {
if config.Reporting.Bugsnag.APIKey == "" {
return
}
bugsnagConfig := bugsnag.Configuration{
APIKey: config.Reporting.Bugsnag.APIKey,
}
if config.Reporting.Bugsnag.ReleaseStage != "" {
bugsnagConfig.ReleaseStage = config.Reporting.Bugsnag.ReleaseStage
}
if config.Reporting.Bugsnag.Endpoint != "" {
bugsnagConfig.Endpoint = config.Reporting.Bugsnag.Endpoint
}
bugsnag.Configure(bugsnagConfig)
// configure logrus bugsnag hook
hook, err := logrus_bugsnag.NewBugsnagHook()
if err != nil {
logrus.Fatalln(err)
}
logrus.AddHook(hook)
}
// panicHandler add an HTTP handler to web app. The handler recover the happening
// panic. logrus.Panic transmits panic message to pre-config log hooks, which is
// defined in config.yml.
func panicHandler(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
logrus.Panic(fmt.Sprintf("%v", err))
}
}()
handler.ServeHTTP(w, r)
})
}
// alive simply wraps the handler with a route that always returns an http 200
// response when the path is matched. If the path is not matched, the request
// is passed to the provided handler. There is no guarantee of anything but
// that the server is up. Wrap with other handlers (such as health.Handler)
// for greater affect.
func alive(path string, handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == path {
w.Header().Set("Cache-Control", "no-cache")
w.WriteHeader(http.StatusOK)
return
}
handler.ServeHTTP(w, r)
})
}
func resolveConfiguration(args []string) (*configuration.Configuration, error) {
var configurationPath string
if len(args) > 0 {
configurationPath = args[0]
} else if os.Getenv("REGISTRY_CONFIGURATION_PATH") != "" {
configurationPath = os.Getenv("REGISTRY_CONFIGURATION_PATH")
}
if configurationPath == "" {
return nil, fmt.Errorf("configuration path unspecified")
}
fp, err := os.Open(configurationPath)
if err != nil {
return nil, err
}
defer fp.Close()
config, err := configuration.Parse(fp)
if err != nil {
return nil, fmt.Errorf("error parsing %s: %v", configurationPath, err)
}
return config, nil
}
func nextProtos(config *configuration.Configuration) []string {
switch config.HTTP.HTTP2.Disabled {
case true:
return []string{"http/1.1"}
default:
return []string{"h2", "http/1.1"}
}
}