Add configuration for upload purging

Signed-off-by: Richard Scothern <richard.scothern@gmail.com>
This commit is contained in:
Richard 2015-04-16 18:34:29 -07:00
parent 84559affdc
commit a72fb20b85
5 changed files with 125 additions and 13 deletions

View file

@ -9,6 +9,9 @@ storage:
layerinfo: inmemory layerinfo: inmemory
filesystem: filesystem:
rootdirectory: /tmp/registry-dev rootdirectory: /tmp/registry-dev
maintenance:
uploadpurging:
enabled: false
http: http:
addr: :5000 addr: :5000
secret: asecretforlocaldevelopment secret: asecretforlocaldevelopment
@ -39,3 +42,4 @@ notifications:
threshold: 10 threshold: 10
backoff: 1s backoff: 1s
disabled: true disabled: true

View file

@ -188,6 +188,8 @@ func (storage Storage) Type() string {
// Return only key in this map // Return only key in this map
for k := range storage { for k := range storage {
switch k { switch k {
case "maintenance":
// allow configuration of maintenance
case "cache": case "cache":
// allow configuration of caching // allow configuration of caching
default: default:
@ -217,6 +219,8 @@ func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error {
types := make([]string, 0, len(storageMap)) types := make([]string, 0, len(storageMap))
for k := range storageMap { for k := range storageMap {
switch k { switch k {
case "maintenance":
// allow for configuration of maintenance
case "cache": case "cache":
// allow configuration of caching // allow configuration of caching
default: default:

View file

@ -43,6 +43,12 @@ storage:
rootdirectory: /s3/object/name/prefix rootdirectory: /s3/object/name/prefix
cache: cache:
layerinfo: inmemory layerinfo: inmemory
maintenance:
uploadpurging:
enabled: true
age: 168h
interval: 24h
dryrun: false
auth: auth:
silly: silly:
realm: silly-realm realm: silly-realm
@ -221,6 +227,12 @@ storage:
rootdirectory: /s3/object/name/prefix rootdirectory: /s3/object/name/prefix
cache: cache:
layerinfo: inmemory layerinfo: inmemory
maintenance:
uploadpurging:
enabled: true
age: 168h
interval: 24h
dryrun: false
``` ```
The storage option is **required** and defines which storage backend is in use. The storage option is **required** and defines which storage backend is in use.
@ -410,6 +422,27 @@ This storage backend uses Amazon's Simple Storage Service (S3).
</tr> </tr>
</table> </table>
### Maintenance
Currently the registry can perform one maintenance function: upload purging. This and future
maintenance functions which are related to storage can be configured under the maintenance section.
### Upload Purging
Upload purging is a background process that periodically removes orphaned files from the upload
directories of the registry. Upload purging is enabled by default. To
configure upload directory purging, the following parameters
must be set.
| Parameter | Required | Description
--------- | -------- | -----------
`enabled` | yes | Set to true to enable upload purging. Default=true. |
`age` | yes | Upload directories which are older than this age will be deleted. Default=168h (1 week)
`interval` | yes | The interval between upload directory purging. Default=24h.
`dryrun` | yes | dryrun can be set to true to obtain a summary of what directories will be deleted. Default=false.
Note: `age` and `interval` are strings containing a number with optional fraction and a unit suffix: e.g. 45m, 2h10m, 168h (1 week).
## auth ## auth
@ -1139,7 +1172,8 @@ Configure the behavior of the Redis connection pool.
</td> </td>
</tr> </tr>
</table> </table>
## Example: Development configuration ## Example: Development configuration
The following is a simple example you can use for local development: The following is a simple example you can use for local development:

View file

@ -205,6 +205,9 @@ storage:
layerinfo: inmemory layerinfo: inmemory
filesystem: filesystem:
rootdirectory: /tmp/registry-dev rootdirectory: /tmp/registry-dev
maintenance:
uploadpurging:
enabled: false
http: http:
addr: :5000 addr: :5000
secret: asecretforlocaldevelopment secret: asecretforlocaldevelopment

View file

@ -81,7 +81,18 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App
panic(err) panic(err)
} }
startUploadPurger(app.driver, ctxu.GetLogger(app)) purgeConfig := uploadPurgeDefaultConfig()
if mc, ok := configuration.Storage["maintenance"]; ok {
for k, v := range mc {
switch k {
case "uploadpurging":
purgeConfig = v.(map[interface{}]interface{})
}
}
}
startUploadPurger(app.driver, ctxu.GetLogger(app), purgeConfig)
app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"]) app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"])
if err != nil { if err != nil {
@ -568,26 +579,82 @@ func applyStorageMiddleware(driver storagedriver.StorageDriver, middlewares []co
return driver, nil return driver, nil
} }
// uploadPurgeDefaultConfig provides a default configuration for upload
// purging to be used in the absence of configuration in the
// confifuration file
func uploadPurgeDefaultConfig() map[interface{}]interface{} {
config := map[interface{}]interface{}{}
config["enabled"] = true
config["age"] = "168h"
config["interval"] = "24h"
config["dryrun"] = false
return config
}
func badPurgeUploadConfig(reason string) {
panic(fmt.Sprintf("Unable to parse upload purge configuration: %s", reason))
}
// startUploadPurger schedules a goroutine which will periodically // startUploadPurger schedules a goroutine which will periodically
// check upload directories for old files and delete them // check upload directories for old files and delete them
func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger) { func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger, config map[interface{}]interface{}) {
rand.Seed(time.Now().Unix()) if config["enabled"] == false {
jitter := time.Duration(rand.Int()%60) * time.Minute return
}
// Start with reasonable defaults var purgeAgeDuration time.Duration
// TODO:(richardscothern) make configurable var err error
purgeAge := time.Duration(7 * 24 * time.Hour) purgeAge, ok := config["age"]
timeBetweenPurges := time.Duration(1 * 24 * time.Hour) if ok {
ageStr, ok := purgeAge.(string)
if !ok {
badPurgeUploadConfig("age is not a string")
}
purgeAgeDuration, err = time.ParseDuration(ageStr)
if err != nil {
badPurgeUploadConfig(fmt.Sprintf("Cannot parse duration: %s", err.Error()))
}
} else {
badPurgeUploadConfig("age missing")
}
var intervalDuration time.Duration
interval, ok := config["interval"]
if ok {
intervalStr, ok := interval.(string)
if !ok {
badPurgeUploadConfig("interval is not a string")
}
intervalDuration, err = time.ParseDuration(intervalStr)
if err != nil {
badPurgeUploadConfig(fmt.Sprintf("Cannot parse interval: %s", err.Error()))
}
} else {
badPurgeUploadConfig("interval missing")
}
var dryRunBool bool
dryRun, ok := config["dryrun"]
if ok {
dryRunBool, ok = dryRun.(bool)
if !ok {
badPurgeUploadConfig("cannot parse dryrun")
}
} else {
badPurgeUploadConfig("dryrun missing")
}
go func() { go func() {
rand.Seed(time.Now().Unix())
jitter := time.Duration(rand.Int()%60) * time.Minute
log.Infof("Starting upload purge in %s", jitter) log.Infof("Starting upload purge in %s", jitter)
time.Sleep(jitter) time.Sleep(jitter)
for { for {
storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAge), true) storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAgeDuration), !dryRunBool)
log.Infof("Starting upload purge in %s", timeBetweenPurges) log.Infof("Starting upload purge in %s", intervalDuration)
time.Sleep(timeBetweenPurges) time.Sleep(intervalDuration)
} }
}() }()
} }