registry: Try to continue iteration #131
8 changed files with 22 additions and 9 deletions
|
@ -78,7 +78,7 @@ func rootCmdRun(cmd *cobra.Command, args []string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
objRegistry := registry.NewObjRegistry(cmd.Context(), args[0])
|
objRegistry := registry.NewObjRegistry(cmd.Context(), args[0])
|
||||||
objSelector := registry.NewObjSelector(objRegistry, 0, ®istry.ObjFilter{
|
objSelector := registry.NewObjSelector(objRegistry, 0, false, ®istry.ObjFilter{
|
||||||
Status: status,
|
Status: status,
|
||||||
Age: age,
|
Age: age,
|
||||||
})
|
})
|
||||||
|
|
|
@ -20,6 +20,7 @@ type ObjSelector struct {
|
||||||
boltDB *bbolt.DB
|
boltDB *bbolt.DB
|
||||||
filter *ObjFilter
|
filter *ObjFilter
|
||||||
cacheSize int
|
cacheSize int
|
||||||
|
looped bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// objectSelectCache is the default maximum size of a batch to select from DB.
|
// objectSelectCache is the default maximum size of a batch to select from DB.
|
||||||
|
@ -27,7 +28,7 @@ const objectSelectCache = 1000
|
||||||
|
|
||||||
// NewObjSelector creates a new instance of object selector that can iterate over
|
// NewObjSelector creates a new instance of object selector that can iterate over
|
||||||
// objects in the specified registry.
|
// objects in the specified registry.
|
||||||
func NewObjSelector(registry *ObjRegistry, selectionSize int, filter *ObjFilter) *ObjSelector {
|
func NewObjSelector(registry *ObjRegistry, selectionSize int, looped bool, filter *ObjFilter) *ObjSelector {
|
||||||
if selectionSize <= 0 {
|
if selectionSize <= 0 {
|
||||||
selectionSize = objectSelectCache
|
selectionSize = objectSelectCache
|
||||||
}
|
}
|
||||||
|
@ -40,6 +41,7 @@ func NewObjSelector(registry *ObjRegistry, selectionSize int, filter *ObjFilter)
|
||||||
filter: filter,
|
filter: filter,
|
||||||
objChan: make(chan *ObjectInfo, selectionSize*2),
|
objChan: make(chan *ObjectInfo, selectionSize*2),
|
||||||
cacheSize: selectionSize,
|
cacheSize: selectionSize,
|
||||||
|
looped: looped,
|
||||||
}
|
}
|
||||||
|
|
||||||
go objSelector.selectLoop()
|
go objSelector.selectLoop()
|
||||||
|
@ -160,13 +162,16 @@ func (o *ObjSelector) selectLoop() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(cache) != o.cacheSize {
|
if !o.looped && len(cache) != o.cacheSize {
|
||||||
// no more objects, wait a little; the logic could be improved.
|
// no more objects, wait a little; the logic could be improved.
|
||||||
select {
|
select {
|
||||||
case <-time.After(time.Second):
|
case <-time.After(time.Second):
|
||||||
case <-o.ctx.Done():
|
case <-o.ctx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if o.looped && len(cache) != o.cacheSize {
|
||||||
lastID = 0
|
lastID = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -75,6 +75,14 @@ func (r *Registry) open(dbFilePath string) *ObjRegistry {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Registry) GetSelector(dbFilePath string, name string, cacheSize int, filter map[string]string) *ObjSelector {
|
func (r *Registry) GetSelector(dbFilePath string, name string, cacheSize int, filter map[string]string) *ObjSelector {
|
||||||
|
return r.getSelectorInternal(dbFilePath, name, cacheSize, false, filter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) GetLoopedSelector(dbFilePath string, name string, cacheSize int, filter map[string]string) *ObjSelector {
|
||||||
|
return r.getSelectorInternal(dbFilePath, name, cacheSize, true, filter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) getSelectorInternal(dbFilePath string, name string, cacheSize int, looped bool, filter map[string]string) *ObjSelector {
|
||||||
objFilter, err := parseFilter(filter)
|
objFilter, err := parseFilter(filter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
|
@ -86,7 +94,7 @@ func (r *Registry) GetSelector(dbFilePath string, name string, cacheSize int, fi
|
||||||
selector := r.root.selectors[name]
|
selector := r.root.selectors[name]
|
||||||
if selector == nil {
|
if selector == nil {
|
||||||
registry := r.open(dbFilePath)
|
registry := r.open(dbFilePath)
|
||||||
selector = NewObjSelector(registry, cacheSize, objFilter)
|
selector = NewObjSelector(registry, cacheSize, looped, objFilter)
|
||||||
r.root.selectors[name] = selector
|
r.root.selectors[name] = selector
|
||||||
} else if !reflect.DeepEqual(selector.filter, objFilter) {
|
} else if !reflect.DeepEqual(selector.filter, objFilter) {
|
||||||
panic(fmt.Sprintf("selector %s already has been created with a different filter", name))
|
panic(fmt.Sprintf("selector %s already has been created with a different filter", name))
|
||||||
|
|
|
@ -57,7 +57,7 @@ if (registry_enabled && delete_age) {
|
||||||
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
||||||
let obj_to_read_selector = undefined;
|
let obj_to_read_selector = undefined;
|
||||||
if (registry_enabled) {
|
if (registry_enabled) {
|
||||||
obj_to_read_selector = registry.getSelector(
|
obj_to_read_selector = registry.getLoopedSelector(
|
||||||
__ENV.REGISTRY_FILE,
|
__ENV.REGISTRY_FILE,
|
||||||
"obj_to_read",
|
"obj_to_read",
|
||||||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||||
|
|
|
@ -57,7 +57,7 @@ if (registry_enabled && delete_age) {
|
||||||
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
||||||
let obj_to_read_selector = undefined;
|
let obj_to_read_selector = undefined;
|
||||||
if (registry_enabled) {
|
if (registry_enabled) {
|
||||||
obj_to_read_selector = registry.getSelector(
|
obj_to_read_selector = registry.getLoopedSelector(
|
||||||
__ENV.REGISTRY_FILE,
|
__ENV.REGISTRY_FILE,
|
||||||
"obj_to_read",
|
"obj_to_read",
|
||||||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||||
|
|
|
@ -56,7 +56,7 @@ if (registry_enabled && delete_age) {
|
||||||
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
||||||
let obj_to_read_selector = undefined;
|
let obj_to_read_selector = undefined;
|
||||||
if (registry_enabled) {
|
if (registry_enabled) {
|
||||||
obj_to_read_selector = registry.getSelector(
|
obj_to_read_selector = registry.getLoopedSelector(
|
||||||
__ENV.REGISTRY_FILE,
|
__ENV.REGISTRY_FILE,
|
||||||
"obj_to_read",
|
"obj_to_read",
|
||||||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||||
|
|
|
@ -56,7 +56,7 @@ if (registry_enabled && delete_age) {
|
||||||
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
const read_age = __ENV.READ_AGE ? parseInt(__ENV.READ_AGE) : 10;
|
||||||
let obj_to_read_selector = undefined;
|
let obj_to_read_selector = undefined;
|
||||||
if (registry_enabled) {
|
if (registry_enabled) {
|
||||||
obj_to_read_selector = registry.getSelector(
|
obj_to_read_selector = registry.getLoopedSelector(
|
||||||
__ENV.REGISTRY_FILE,
|
__ENV.REGISTRY_FILE,
|
||||||
"obj_to_read",
|
"obj_to_read",
|
||||||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||||
|
|
|
@ -54,7 +54,7 @@ const obj_registry = registry_enabled ? registry.open(__ENV.REGISTRY_FILE) : und
|
||||||
|
|
||||||
let obj_to_read_selector = undefined;
|
let obj_to_read_selector = undefined;
|
||||||
if (registry_enabled) {
|
if (registry_enabled) {
|
||||||
obj_to_read_selector = registry.getSelector(
|
obj_to_read_selector = registry.getLoopedSelector(
|
||||||
__ENV.REGISTRY_FILE,
|
__ENV.REGISTRY_FILE,
|
||||||
"obj_to_read",
|
"obj_to_read",
|
||||||
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
__ENV.SELECTION_SIZE ? parseInt(__ENV.SELECTION_SIZE) : 0,
|
||||||
|
|
Loading…
Reference in a new issue