From 276fdce3d9a28e0d56700c8e8333b687cc948769 Mon Sep 17 00:00:00 2001 From: Andrew Leung Date: Wed, 27 Jun 2018 08:47:40 -0700 Subject: [PATCH 1/3] Add configurable layers in manifest events Signed-off-by: Andrew Leung --- cmd/registry/config-cache.yml | 2 ++ cmd/registry/config-dev.yml | 2 ++ configuration/configuration.go | 7 +++++++ docs/configuration.md | 11 +++++++++++ manifest/schema2/manifest.go | 2 +- notifications/bridge.go | 29 +++++++++++++++++------------ notifications/bridge_test.go | 26 ++++++++++++++++++++++---- notifications/event.go | 3 +++ registry/handlers/app.go | 2 +- 9 files changed, 66 insertions(+), 18 deletions(-) diff --git a/cmd/registry/config-cache.yml b/cmd/registry/config-cache.yml index 7a274ea59..f7e47b82d 100644 --- a/cmd/registry/config-cache.yml +++ b/cmd/registry/config-cache.yml @@ -29,6 +29,8 @@ redis: readtimeout: 10ms writetimeout: 10ms notifications: + events: + manifestlayers: true endpoints: - name: local-8082 url: http://localhost:5003/callback diff --git a/cmd/registry/config-dev.yml b/cmd/registry/config-dev.yml index e6b090646..250025691 100644 --- a/cmd/registry/config-dev.yml +++ b/cmd/registry/config-dev.yml @@ -47,6 +47,8 @@ redis: readtimeout: 10ms writetimeout: 10ms notifications: + events: + manifestlayers: true endpoints: - name: local-5003 url: http://localhost:5003/callback diff --git a/configuration/configuration.go b/configuration/configuration.go index 621d29aed..598413826 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -544,6 +544,8 @@ func (auth Auth) MarshalYAML() (interface{}, error) { // Notifications configures multiple http endpoints. type Notifications struct { + // EventConfig is the configuration for the event format that is sent to each Endpoint. + EventConfig Events `yaml:"events,omitempty"` // Endpoints is a list of http configurations for endpoints that // respond to webhook notifications. In the future, we may allow other // kinds of endpoints, such as external queues. @@ -564,6 +566,11 @@ type Endpoint struct { Ignore Ignore `yaml:"ignore"` // ignore event types } +// Events configures notification events. +type Events struct { + ManifestLayers bool `yaml:"manifestlayers"` // include layer data in manifest events +} + //Ignore configures mediaTypes and actions of the event, that it won't be propagated type Ignore struct { MediaTypes []string `yaml:"mediatypes"` // target media types to ignore diff --git a/docs/configuration.md b/docs/configuration.md index f2e835133..7a6b9b2c9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -226,6 +226,8 @@ http: http2: disabled: false notifications: + events: + manifestlayers: true endpoints: - name: alistener disabled: false @@ -853,6 +855,8 @@ settings for the registry. ```none notifications: + events: + manifestlayers: true endpoints: - name: alistener disabled: false @@ -896,6 +900,13 @@ accept event notifications. | `mediatypes`|no| A list of target media types to ignore. Events with these target media types are not published to the endpoint. | | `actions` |no| A list of actions to ignore. Events with these actions are not published to the endpoint. | +### `events` + +The `events` structure configures the information provided in event notifications. + +| Parameter | Required | Description | +|-----------|----------|-------------------------------------------------------| +| `manifestlayers` | no | If `true`, include layer information in manifest events. | ## `redis` diff --git a/manifest/schema2/manifest.go b/manifest/schema2/manifest.go index a2708c750..243d58e94 100644 --- a/manifest/schema2/manifest.go +++ b/manifest/schema2/manifest.go @@ -71,7 +71,7 @@ type Manifest struct { Layers []distribution.Descriptor `json:"layers"` } -// References returnes the descriptors of this manifests references. +// References returns the descriptors of this manifests references. func (m Manifest) References() []distribution.Descriptor { references := make([]distribution.Descriptor, 0, 1+len(m.Layers)) references = append(references, m.Config) diff --git a/notifications/bridge.go b/notifications/bridge.go index 8f6386d3c..b336cd89e 100644 --- a/notifications/bridge.go +++ b/notifications/bridge.go @@ -12,11 +12,12 @@ import ( ) type bridge struct { - ub URLBuilder - actor ActorRecord - source SourceRecord - request RequestRecord - sink Sink + ub URLBuilder + manfiestLayers bool + actor ActorRecord + source SourceRecord + request RequestRecord + sink Sink } var _ Listener = &bridge{} @@ -31,13 +32,14 @@ type URLBuilder interface { // using the actor and source. Any urls populated in the events created by // this bridge will be created using the URLBuilder. // TODO(stevvooe): Update this to simply take a context.Context object. -func NewBridge(ub URLBuilder, source SourceRecord, actor ActorRecord, request RequestRecord, sink Sink) Listener { +func NewBridge(ub URLBuilder, source SourceRecord, actor ActorRecord, request RequestRecord, sink Sink, manifestLayers bool) Listener { return &bridge{ - ub: ub, - actor: actor, - source: source, - request: request, - sink: sink, + ub: ub, + manfiestLayers: manifestLayers, + actor: actor, + source: source, + request: request, + sink: sink, } } @@ -135,7 +137,7 @@ func (b *bridge) createManifestEvent(action string, repo reference.Named, sm dis } // Ensure we have the canonical manifest descriptor here - _, desc, err := distribution.UnmarshalManifest(mt, p) + manifest, desc, err := distribution.UnmarshalManifest(mt, p) if err != nil { return nil, err } @@ -144,6 +146,9 @@ func (b *bridge) createManifestEvent(action string, repo reference.Named, sm dis event.Target.Length = desc.Size event.Target.Size = desc.Size event.Target.Digest = desc.Digest + if b.manfiestLayers { + event.Target.Layers = append(event.Target.Layers, manifest.References()...) + } ref, err := reference.WithDigest(repo, event.Target.Digest) if err != nil { diff --git a/notifications/bridge_test.go b/notifications/bridge_test.go index 863509936..cdf500c32 100644 --- a/notifications/bridge_test.go +++ b/notifications/bridge_test.go @@ -26,9 +26,18 @@ var ( Name: "test", } request = RequestRecord{} - m = schema1.Manifest{ - Name: repo, - Tag: "latest", + layers = []schema1.FSLayer{ + { + BlobSum: "asdf", + }, + { + BlobSum: "qwer", + }, + } + m = schema1.Manifest{ + Name: repo, + Tag: "latest", + FSLayers: layers, } sm *schema1.SignedManifest @@ -120,7 +129,7 @@ func createTestEnv(t *testing.T, fn testSinkFn) Listener { payload = sm.Canonical dgst = digest.FromBytes(payload) - return NewBridge(ub, source, actor, request, fn) + return NewBridge(ub, source, actor, request, fn, true) } func checkDeleted(t *testing.T, action string, events ...Event) { @@ -170,6 +179,15 @@ func checkCommonManifest(t *testing.T, action string, events ...Event) { if event.Target.URL != u { t.Fatalf("incorrect url passed: \n%q != \n%q", event.Target.URL, u) } + + if len(event.Target.Layers) != len(layers) { + t.Fatalf("unexpected number of layers %v != %v", len(event.Target.Layers), len(layers)) + } + for i, layer := range event.Target.Layers { + if layer.Digest != layers[i].BlobSum { + t.Fatalf("unexpected layer: %q != %q", layer.Digest, layers[i].BlobSum) + } + } } func checkCommon(t *testing.T, events ...Event) { diff --git a/notifications/event.go b/notifications/event.go index 9651cd1b1..ab74c6752 100644 --- a/notifications/event.go +++ b/notifications/event.go @@ -71,6 +71,9 @@ type Event struct { // Tag provides the tag Tag string `json:"tag,omitempty"` + + // Layers provides the layers descriptors. + Layers []distribution.Descriptor `json:"layers,omitempty"` } `json:"target,omitempty"` // Request covers the request that generated the event. diff --git a/registry/handlers/app.go b/registry/handlers/app.go index a40a4df38..8ccfffe50 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -869,7 +869,7 @@ func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listene } request := notifications.NewRequestRecord(dcontext.GetRequestID(ctx), r) - return notifications.NewBridge(ctx.urlBuilder, app.events.source, actor, request, app.events.sink) + return notifications.NewBridge(ctx.urlBuilder, app.events.source, actor, request, app.events.sink, app.Config.Notifications.EventConfig.ManifestLayers) } // nameRequired returns true if the route requires a name. From 54aef6251e4970010567f7b79779c18e07e5db02 Mon Sep 17 00:00:00 2001 From: Andrew Leung Date: Sun, 29 Jul 2018 07:53:33 -0700 Subject: [PATCH 2/3] Fix typo in manifestLayers variable name Signed-off-by: Andrew Leung --- notifications/bridge.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/notifications/bridge.go b/notifications/bridge.go index b336cd89e..1a58c2bb5 100644 --- a/notifications/bridge.go +++ b/notifications/bridge.go @@ -13,7 +13,7 @@ import ( type bridge struct { ub URLBuilder - manfiestLayers bool + manifestLayers bool actor ActorRecord source SourceRecord request RequestRecord @@ -35,7 +35,7 @@ type URLBuilder interface { func NewBridge(ub URLBuilder, source SourceRecord, actor ActorRecord, request RequestRecord, sink Sink, manifestLayers bool) Listener { return &bridge{ ub: ub, - manfiestLayers: manifestLayers, + manifestLayers: manifestLayers, actor: actor, source: source, request: request, @@ -146,7 +146,7 @@ func (b *bridge) createManifestEvent(action string, repo reference.Named, sm dis event.Target.Length = desc.Size event.Target.Size = desc.Size event.Target.Digest = desc.Digest - if b.manfiestLayers { + if b.manifestLayers { event.Target.Layers = append(event.Target.Layers, manifest.References()...) } From 5e4b81a578256648d897c5bf6ea9b84665763e98 Mon Sep 17 00:00:00 2001 From: Andrew Leung Date: Mon, 20 Aug 2018 10:01:40 -0700 Subject: [PATCH 3/3] Use references terminology instead of layers. Signed-off-by: Andrew Leung --- cmd/registry/config-cache.yml | 2 +- cmd/registry/config-dev.yml | 2 +- configuration/configuration.go | 2 +- docs/configuration.md | 6 +++--- notifications/bridge.go | 30 +++++++++++++++--------------- notifications/bridge_test.go | 10 +++++----- notifications/event.go | 4 ++-- registry/handlers/app.go | 2 +- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cmd/registry/config-cache.yml b/cmd/registry/config-cache.yml index f7e47b82d..d648303d9 100644 --- a/cmd/registry/config-cache.yml +++ b/cmd/registry/config-cache.yml @@ -30,7 +30,7 @@ redis: writetimeout: 10ms notifications: events: - manifestlayers: true + includereferences: true endpoints: - name: local-8082 url: http://localhost:5003/callback diff --git a/cmd/registry/config-dev.yml b/cmd/registry/config-dev.yml index 250025691..9539bae4a 100644 --- a/cmd/registry/config-dev.yml +++ b/cmd/registry/config-dev.yml @@ -48,7 +48,7 @@ redis: writetimeout: 10ms notifications: events: - manifestlayers: true + includereferences: true endpoints: - name: local-5003 url: http://localhost:5003/callback diff --git a/configuration/configuration.go b/configuration/configuration.go index 598413826..0b5b46aae 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -568,7 +568,7 @@ type Endpoint struct { // Events configures notification events. type Events struct { - ManifestLayers bool `yaml:"manifestlayers"` // include layer data in manifest events + IncludeReferences bool `yaml:"includereferences"` // include reference data in manifest events } //Ignore configures mediaTypes and actions of the event, that it won't be propagated diff --git a/docs/configuration.md b/docs/configuration.md index 7a6b9b2c9..3f3464394 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -227,7 +227,7 @@ http: disabled: false notifications: events: - manifestlayers: true + includereferences: true endpoints: - name: alistener disabled: false @@ -856,7 +856,7 @@ settings for the registry. ```none notifications: events: - manifestlayers: true + includereferences: true endpoints: - name: alistener disabled: false @@ -906,7 +906,7 @@ The `events` structure configures the information provided in event notification | Parameter | Required | Description | |-----------|----------|-------------------------------------------------------| -| `manifestlayers` | no | If `true`, include layer information in manifest events. | +| `includereferences` | no | If `true`, include reference information in manifest events. | ## `redis` diff --git a/notifications/bridge.go b/notifications/bridge.go index 1a58c2bb5..efc73f689 100644 --- a/notifications/bridge.go +++ b/notifications/bridge.go @@ -12,12 +12,12 @@ import ( ) type bridge struct { - ub URLBuilder - manifestLayers bool - actor ActorRecord - source SourceRecord - request RequestRecord - sink Sink + ub URLBuilder + includeReferences bool + actor ActorRecord + source SourceRecord + request RequestRecord + sink Sink } var _ Listener = &bridge{} @@ -32,14 +32,14 @@ type URLBuilder interface { // using the actor and source. Any urls populated in the events created by // this bridge will be created using the URLBuilder. // TODO(stevvooe): Update this to simply take a context.Context object. -func NewBridge(ub URLBuilder, source SourceRecord, actor ActorRecord, request RequestRecord, sink Sink, manifestLayers bool) Listener { +func NewBridge(ub URLBuilder, source SourceRecord, actor ActorRecord, request RequestRecord, sink Sink, includeReferences bool) Listener { return &bridge{ - ub: ub, - manifestLayers: manifestLayers, - actor: actor, - source: source, - request: request, - sink: sink, + ub: ub, + includeReferences: includeReferences, + actor: actor, + source: source, + request: request, + sink: sink, } } @@ -146,8 +146,8 @@ func (b *bridge) createManifestEvent(action string, repo reference.Named, sm dis event.Target.Length = desc.Size event.Target.Size = desc.Size event.Target.Digest = desc.Digest - if b.manifestLayers { - event.Target.Layers = append(event.Target.Layers, manifest.References()...) + if b.includeReferences { + event.Target.References = append(event.Target.References, manifest.References()...) } ref, err := reference.WithDigest(repo, event.Target.Digest) diff --git a/notifications/bridge_test.go b/notifications/bridge_test.go index cdf500c32..87c868a5c 100644 --- a/notifications/bridge_test.go +++ b/notifications/bridge_test.go @@ -180,12 +180,12 @@ func checkCommonManifest(t *testing.T, action string, events ...Event) { t.Fatalf("incorrect url passed: \n%q != \n%q", event.Target.URL, u) } - if len(event.Target.Layers) != len(layers) { - t.Fatalf("unexpected number of layers %v != %v", len(event.Target.Layers), len(layers)) + if len(event.Target.References) != len(layers) { + t.Fatalf("unexpected number of references %v != %v", len(event.Target.References), len(layers)) } - for i, layer := range event.Target.Layers { - if layer.Digest != layers[i].BlobSum { - t.Fatalf("unexpected layer: %q != %q", layer.Digest, layers[i].BlobSum) + for i, targetReference := range event.Target.References { + if targetReference.Digest != layers[i].BlobSum { + t.Fatalf("unexpected reference: %q != %q", targetReference.Digest, layers[i].BlobSum) } } } diff --git a/notifications/event.go b/notifications/event.go index ab74c6752..54940a46f 100644 --- a/notifications/event.go +++ b/notifications/event.go @@ -72,8 +72,8 @@ type Event struct { // Tag provides the tag Tag string `json:"tag,omitempty"` - // Layers provides the layers descriptors. - Layers []distribution.Descriptor `json:"layers,omitempty"` + // References provides the references descriptors. + References []distribution.Descriptor `json:"references,omitempty"` } `json:"target,omitempty"` // Request covers the request that generated the event. diff --git a/registry/handlers/app.go b/registry/handlers/app.go index 8ccfffe50..8c1f18e61 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -869,7 +869,7 @@ func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listene } request := notifications.NewRequestRecord(dcontext.GetRequestID(ctx), r) - return notifications.NewBridge(ctx.urlBuilder, app.events.source, actor, request, app.events.sink, app.Config.Notifications.EventConfig.ManifestLayers) + return notifications.NewBridge(ctx.urlBuilder, app.events.source, actor, request, app.events.sink, app.Config.Notifications.EventConfig.IncludeReferences) } // nameRequired returns true if the route requires a name.