Add configuration for upload purging

Signed-off-by: Richard Scothern <richard.scothern@gmail.com>
This commit is contained in:
Richard 2015-04-16 18:34:29 -07:00
parent 4dde6b9d64
commit 6460ddb2cb
5 changed files with 125 additions and 13 deletions

View file

@ -9,6 +9,9 @@ storage:
layerinfo: inmemory
filesystem:
rootdirectory: /tmp/registry-dev
maintenance:
uploadpurging:
enabled: false
http:
addr: :5000
secret: asecretforlocaldevelopment
@ -39,3 +42,4 @@ notifications:
threshold: 10
backoff: 1s
disabled: true

View file

@ -188,6 +188,8 @@ func (storage Storage) Type() string {
// Return only key in this map
for k := range storage {
switch k {
case "maintenance":
// allow configuration of maintenance
case "cache":
// allow configuration of caching
default:
@ -217,6 +219,8 @@ func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error {
types := make([]string, 0, len(storageMap))
for k := range storageMap {
switch k {
case "maintenance":
// allow for configuration of maintenance
case "cache":
// allow configuration of caching
default:

View file

@ -43,6 +43,12 @@ storage:
rootdirectory: /s3/object/name/prefix
cache:
layerinfo: inmemory
maintenance:
uploadpurging:
enabled: true
age: 168h
interval: 24h
dryrun: false
auth:
silly:
realm: silly-realm
@ -221,6 +227,12 @@ storage:
rootdirectory: /s3/object/name/prefix
cache:
layerinfo: inmemory
maintenance:
uploadpurging:
enabled: true
age: 168h
interval: 24h
dryrun: false
```
The storage option is **required** and defines which storage backend is in use.
@ -410,6 +422,27 @@ This storage backend uses Amazon's Simple Storage Service (S3).
</tr>
</table>
### Maintenance
Currently the registry can perform one maintenance function: upload purging. This and future
maintenance functions which are related to storage can be configured under the maintenance section.
### Upload Purging
Upload purging is a background process that periodically removes orphaned files from the upload
directories of the registry. Upload purging is enabled by default. To
configure upload directory purging, the following parameters
must be set.
| Parameter | Required | Description
--------- | -------- | -----------
`enabled` | yes | Set to true to enable upload purging. Default=true. |
`age` | yes | Upload directories which are older than this age will be deleted. Default=168h (1 week)
`interval` | yes | The interval between upload directory purging. Default=24h.
`dryrun` | yes | dryrun can be set to true to obtain a summary of what directories will be deleted. Default=false.
Note: `age` and `interval` are strings containing a number with optional fraction and a unit suffix: e.g. 45m, 2h10m, 168h (1 week).
## auth
@ -1139,7 +1172,8 @@ Configure the behavior of the Redis connection pool.
</td>
</tr>
</table>
## Example: Development configuration
The following is a simple example you can use for local development:

View file

@ -205,6 +205,9 @@ storage:
layerinfo: inmemory
filesystem:
rootdirectory: /tmp/registry-dev
maintenance:
uploadpurging:
enabled: false
http:
addr: :5000
secret: asecretforlocaldevelopment

View file

@ -81,7 +81,18 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App
panic(err)
}
startUploadPurger(app.driver, ctxu.GetLogger(app))
purgeConfig := uploadPurgeDefaultConfig()
if mc, ok := configuration.Storage["maintenance"]; ok {
for k, v := range mc {
switch k {
case "uploadpurging":
purgeConfig = v.(map[interface{}]interface{})
}
}
}
startUploadPurger(app.driver, ctxu.GetLogger(app), purgeConfig)
app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"])
if err != nil {
@ -568,26 +579,82 @@ func applyStorageMiddleware(driver storagedriver.StorageDriver, middlewares []co
return driver, nil
}
// uploadPurgeDefaultConfig provides a default configuration for upload
// purging to be used in the absence of configuration in the
// confifuration file
func uploadPurgeDefaultConfig() map[interface{}]interface{} {
config := map[interface{}]interface{}{}
config["enabled"] = true
config["age"] = "168h"
config["interval"] = "24h"
config["dryrun"] = false
return config
}
func badPurgeUploadConfig(reason string) {
panic(fmt.Sprintf("Unable to parse upload purge configuration: %s", reason))
}
// startUploadPurger schedules a goroutine which will periodically
// check upload directories for old files and delete them
func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger) {
rand.Seed(time.Now().Unix())
jitter := time.Duration(rand.Int()%60) * time.Minute
func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger, config map[interface{}]interface{}) {
if config["enabled"] == false {
return
}
// Start with reasonable defaults
// TODO:(richardscothern) make configurable
purgeAge := time.Duration(7 * 24 * time.Hour)
timeBetweenPurges := time.Duration(1 * 24 * time.Hour)
var purgeAgeDuration time.Duration
var err error
purgeAge, ok := config["age"]
if ok {
ageStr, ok := purgeAge.(string)
if !ok {
badPurgeUploadConfig("age is not a string")
}
purgeAgeDuration, err = time.ParseDuration(ageStr)
if err != nil {
badPurgeUploadConfig(fmt.Sprintf("Cannot parse duration: %s", err.Error()))
}
} else {
badPurgeUploadConfig("age missing")
}
var intervalDuration time.Duration
interval, ok := config["interval"]
if ok {
intervalStr, ok := interval.(string)
if !ok {
badPurgeUploadConfig("interval is not a string")
}
intervalDuration, err = time.ParseDuration(intervalStr)
if err != nil {
badPurgeUploadConfig(fmt.Sprintf("Cannot parse interval: %s", err.Error()))
}
} else {
badPurgeUploadConfig("interval missing")
}
var dryRunBool bool
dryRun, ok := config["dryrun"]
if ok {
dryRunBool, ok = dryRun.(bool)
if !ok {
badPurgeUploadConfig("cannot parse dryrun")
}
} else {
badPurgeUploadConfig("dryrun missing")
}
go func() {
rand.Seed(time.Now().Unix())
jitter := time.Duration(rand.Int()%60) * time.Minute
log.Infof("Starting upload purge in %s", jitter)
time.Sleep(jitter)
for {
storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAge), true)
log.Infof("Starting upload purge in %s", timeBetweenPurges)
time.Sleep(timeBetweenPurges)
storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAgeDuration), !dryRunBool)
log.Infof("Starting upload purge in %s", intervalDuration)
time.Sleep(intervalDuration)
}
}()
}