dkirillov/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Merge pull request #742 from BrianBland/ng-push-pull

Browse source 
Adds sliding-window parallelization to Push/Pull operations
This commit is contained in:
Stephen Day 2014-11-18 15:19:04 -08:00
commit 1e8f0ce50a
2 changed files with 178 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
client/pull.go
View file

@ -4,9 +4,16 @@ import (
"fmt"
"io"
"github.com/docker/docker-registry"
log "github.com/Sirupsen/logrus"
)
// simultaneousLayerPullWindow is the size of the parallel layer pull window.
// A layer may not be pulled until the layer preceeding it by the length of the
// pull window has been successfully pulled.
const simultaneousLayerPullWindow = 4
// Pull implements a client pull workflow for the image defined by the given
// name and tag pair, using the given ObjectStore for local manifest and layer
// storage
@ -24,7 +31,46 @@ func Pull(c Client, objectStore ObjectStore, name, tag string) error {
return fmt.Errorf("Image has no layers")
}
for _, fsLayer := range manifest.FSLayers {
errChans := make([]chan error, len(manifest.FSLayers))
for i := range manifest.FSLayers {
errChans[i] = make(chan error)
}
// Iterate over each layer in the manifest, simultaneously pulling no more
// than simultaneousLayerPullWindow layers at a time. If an error is
// received from a layer pull, we abort the push.
for i := 0; i < len(manifest.FSLayers)+simultaneousLayerPullWindow; i++ {
dependentLayer := i - simultaneousLayerPullWindow
if dependentLayer >= 0 {
err := <-errChans[dependentLayer]
if err != nil {
log.WithField("error", err).Warn("Pull aborted")
return err
}
}
if i < len(manifest.FSLayers) {
go func(i int) {
errChans[i] <- pullLayer(c, objectStore, name, manifest.FSLayers[i])
}(i)
}
}
err = objectStore.WriteManifest(name, tag, manifest)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"manifest": manifest,
}).Warn("Unable to write image manifest")
return err
}
return nil
}
func pullLayer(c Client, objectStore ObjectStore, name string, fsLayer registry.FSLayer) error {
log.WithField("layer", fsLayer).Info("Pulling layer")
layer, err := objectStore.Layer(fsLayer.BlobSum)
if err != nil {
log.WithFields(log.Fields{
@ -37,12 +83,12 @@ func Pull(c Client, objectStore ObjectStore, name, tag string) error {
writer, err := layer.Writer()
if err == ErrLayerAlreadyExists {
log.WithField("layer", fsLayer).Info("Layer already exists")
continue
return nil
}
if err == ErrLayerLocked {
log.WithField("layer", fsLayer).Info("Layer download in progress, waiting")
layer.Wait()
continue
return nil
}
if err != nil {
log.WithFields(log.Fields{
@ -77,17 +123,10 @@ func Pull(c Client, objectStore ObjectStore, name, tag string) error {
"written": copied,
"layer": fsLayer,
}).Warn("Wrote incorrect number of bytes for layer")
return fmt.Errorf(
"Wrote incorrect number of bytes for layer %v. Expected %d, Wrote %d",
fsLayer, length, copied,
)
}
}
err = objectStore.WriteManifest(name, tag, manifest)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"manifest": manifest,
}).Warn("Unable to write image manifest")
return err
}
return nil
}

60
client/push.go
View file

@ -11,6 +11,13 @@ import (
log "github.com/Sirupsen/logrus"
)
// simultaneousLayerPushWindow is the size of the parallel layer push window.
// A layer may not be pushed until the layer preceeding it by the length of the
// push window has been successfully pushed.
const simultaneousLayerPushWindow = 4
type pushFunction func(fsLayer registry.FSLayer) error
// Push implements a client push workflow for the image defined by the given
// name and tag pair, using the given ObjectStore for local manifest and layer
// storage
@ -25,7 +32,46 @@ func Push(c Client, objectStore ObjectStore, name, tag string) error {
return err
}
for _, fsLayer := range manifest.FSLayers {
errChans := make([]chan error, len(manifest.FSLayers))
for i := range manifest.FSLayers {
errChans[i] = make(chan error)
}
// Iterate over each layer in the manifest, simultaneously pushing no more
// than simultaneousLayerPushWindow layers at a time. If an error is
// received from a layer push, we abort the push.
for i := 0; i < len(manifest.FSLayers)+simultaneousLayerPushWindow; i++ {
dependentLayer := i - simultaneousLayerPushWindow
if dependentLayer >= 0 {
err := <-errChans[dependentLayer]
if err != nil {
log.WithField("error", err).Warn("Push aborted")
return err
}
}
if i < len(manifest.FSLayers) {
go func(i int) {
errChans[i] <- pushLayer(c, objectStore, name, manifest.FSLayers[i])
}(i)
}
}
err = c.PutImageManifest(name, tag, manifest)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"manifest": manifest,
}).Warn("Unable to upload manifest")
return err
}
return nil
}
func pushLayer(c Client, objectStore ObjectStore, name string, fsLayer registry.FSLayer) error {
log.WithField("layer", fsLayer).Info("Pushing layer")
layer, err := objectStore.Layer(fsLayer.BlobSum)
if err != nil {
log.WithFields(log.Fields{
@ -47,7 +93,7 @@ func Push(c Client, objectStore ObjectStore, name, tag string) error {
location, err := c.InitiateLayerUpload(name, fsLayer.BlobSum)
if _, ok := err.(*registry.LayerAlreadyExistsError); ok {
log.WithField("layer", fsLayer).Info("Layer already exists")
continue
return nil
}
if err != nil {
log.WithFields(log.Fields{
@ -80,16 +126,6 @@ func Push(c Client, objectStore ObjectStore, name, tag string) error {
}).Warn("Unable to upload layer")
return err
}
}
err = c.PutImageManifest(name, tag, manifest)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"manifest": manifest,
}).Warn("Unable to upload manifest")
return err
}
return nil
}