TrueCloudLab/rclone
16
0
Fork 4
Code Issues 3 Pull requests 2 Projects Releases 2 Packages Wiki Activity Actions

[#5] Add container zone names support.

Browse source 
Signed-off-by: Aleksey Kravchenko <al.kravchenko@yadro.com>
This commit is contained in:
Aleksey Kravchenko 2025-01-22 19:07:48 +03:00
parent b93e134b5b
commit 2833ac991c
3 changed files with 120 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
backend/frostfs/frostfs.go
View file

@ -127,6 +127,11 @@ func init() {
},
},
},
{
Name: "default_container_zone",
Default: "container",
Help: "The name of the zone in which containers will be created or resolved if the zone name is not explicitly specified with the container name. Can be empty.",
},
{
Name: "container_creation_policy",
Default: "private",
@ -166,6 +171,7 @@ type Options struct {
Address string `config:"address"`
Password string `config:"password"`
PlacementPolicy string `config:"placement_policy"`
DefaultContainerZone string `config:"default_container_zone"`
ContainerCreationPolicy string `config:"container_creation_policy"`
APERules []chain.Rule `config:"-"`
}
@ -718,6 +724,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return nil
}
func (f *Fs) getZoneAndContainerNames(containerStr string) (string, string) {
return getZoneAndContainerNames(containerStr, f.opt.DefaultContainerZone)
}
// Remove an object
func (o *Object) Remove(ctx context.Context) error {
cnrID, _ := o.ContainerID()
@ -805,7 +815,7 @@ func (f *Fs) waitForAPECacheInvalidated(ctx context.Context, expectedCh chain.Ch
}
}
func (f *Fs) createContainer(ctx context.Context, containerName string) (cid.ID, error) {
func (f *Fs) createContainer(ctx context.Context, containerStr string) (cid.ID, error) {
var policy netmap.PlacementPolicy
if err := policy.DecodeString(f.opt.PlacementPolicy); err != nil {
return cid.ID{}, fmt.Errorf("parse placement policy: %w", err)
@ -817,10 +827,12 @@ func (f *Fs) createContainer(ctx context.Context, containerName string) (cid.ID,
cnr.SetOwner(*f.owner)
container.SetCreationTime(&cnr, time.Now())
container.SetName(&cnr, containerName)
container.SetName(&cnr, containerStr)
cnrZone, cnrName := f.getZoneAndContainerNames(containerStr)
var domain container.Domain
domain.SetName(containerName)
domain.SetZone(cnrZone)
domain.SetName(cnrName)
container.WriteDomain(&cnr, domain)
if err := pool.SyncContainerWithNetwork(ctx, &cnr, f.pool); err != nil {
@ -944,8 +956,9 @@ func parseContainerID(containerStr string) (cid.ID, error) {
return cnrID, err
}
func getContainerIDByName(dirEntry fs.DirEntry, containerName string) (ok bool, cnrID cid.ID, err error) {
if dirEntry.Remote() != containerName {
func getContainerIDByZoneAndName(dirEntry fs.DirEntry, cnrZone, cnrName, defaultZone string) (ok bool, cnrID cid.ID, err error) {
actualZone, actualName := getZoneAndContainerNames(dirEntry.Remote(), defaultZone)
if cnrName != actualName || cnrZone != actualZone {
return
}
var idEr fs.IDer
@ -956,41 +969,46 @@ func getContainerIDByName(dirEntry fs.DirEntry, containerName string) (ok bool,
return
}
func resolveContainerIDWithNNS(resolver *resolver.NNS, containerName string) (cid.ID, error) {
func resolveContainerIDWithNNS(resolver *resolver.NNS, cnrZone, cnrName, containerStr string) (cid.ID, error) {
var d container.Domain
d.SetName(containerName)
d.SetZone(cnrZone)
d.SetName(cnrName)
if cnrID, err := resolver.ResolveContainerDomain(d); err == nil {
return cnrID, err
}
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s'", containerName)
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s'", containerStr)
}
func (f *Fs) resolveContainerIDHelper(ctx context.Context, containerName string) (cid.ID, error) {
cnrIDStr, ok := f.containerIDCache[containerName]
func (f *Fs) resolveContainerIDHelper(ctx context.Context, containerStr string) (cid.ID, error) {
cnrIDStr, ok := f.containerIDCache[containerStr]
if ok {
return parseContainerID(cnrIDStr)
}
cnrZone, cnrName := f.getZoneAndContainerNames(containerStr)
if cnrName == "" {
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s'", containerStr)
}
if f.resolver != nil {
var err error
var cnrID cid.ID
if cnrID, err = resolveContainerIDWithNNS(f.resolver, containerName); err == nil {
f.containerIDCache[containerName] = cnrID.String()
if cnrID, err = resolveContainerIDWithNNS(f.resolver, cnrZone, cnrName, containerStr); err == nil {
f.containerIDCache[containerStr] = cnrID.String()
}
return cnrID, err
}
if dirEntries, err := f.listContainers(ctx); err == nil {
for _, dirEntry := range dirEntries {
if ok, cnrID, err := getContainerIDByName(dirEntry, containerName); ok {
if ok, cnrID, err := getContainerIDByZoneAndName(dirEntry, cnrZone, cnrName, f.opt.DefaultContainerZone); ok {
if err == nil {
f.containerIDCache[containerName] = cnrID.String()
f.containerIDCache[containerStr] = cnrID.String()
}
return cnrID, err
}
}
}
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s'", containerName)
return cid.ID{}, fmt.Errorf("couldn't resolve container '%s'", containerStr)
}
func (f *Fs) resolveContainerID(ctx context.Context, containerName string) (cid.ID, error) {
@ -1083,7 +1101,7 @@ func (f *Fs) listContainers(ctx context.Context) (fs.DirEntries, error) {
return nil, fmt.Errorf("couldn't get container '%s': %w", containerID, err)
}
res[i] = newDir(containerID, cnr)
res[i] = newDir(containerID, cnr, f.opt.DefaultContainerZone)
}
return res, nil

20
backend/frostfs/util.go
View file

@ -296,12 +296,16 @@ func formObject(own *user.ID, cnrID cid.ID, name string, header map[string]strin
return obj
}
func newDir(cnrID cid.ID, cnr container.Container) *fs.Dir {
func newDir(cnrID cid.ID, cnr container.Container, defaultZone string) *fs.Dir {
remote := cnrID.EncodeToString()
timestamp := container.CreatedAt(cnr)
if domain := container.ReadDomain(cnr); domain.Name() != "" {
remote = domain.Name()
if defaultZone != domain.Zone() {
remote = domain.Zone() + "." + domain.Name()
} else {
remote = domain.Name()
}
}
dir := fs.NewDir(remote, timestamp)
@ -309,3 +313,15 @@ func newDir(cnrID cid.ID, cnr container.Container) *fs.Dir {
return dir
}
func getZoneAndContainerNames(containerStr, defaultZone string) (cnrZone string, cnrName string) {
defer func() {
if len(cnrZone) == 0 {
cnrZone = "container"
}
}()
if idx := strings.Index(containerStr, "."); idx >= 0 {
return containerStr[:idx], containerStr[idx+1:]
}
return defaultZone, containerStr
}

68
backend/frostfs/util_test.go
View file

@ -7,6 +7,74 @@ import (
"github.com/stretchr/testify/require"
)
func TestGetZoneAndContainerNames(t *testing.T) {
for i, tc := range []struct {
cnrStr string
defZone string
expectedName string
expectedZone string
}{
{
cnrStr: "",
defZone: "",
expectedName: "",
expectedZone: "container",
},
{
cnrStr: "",
defZone: "def_zone",
expectedName: "",
expectedZone: "def_zone",
},
{
cnrStr: "cnr_name",
defZone: "",
expectedName: "cnr_name",
expectedZone: "container",
},
{
cnrStr: "cnr_name",
defZone: "def_zone",
expectedName: "cnr_name",
expectedZone: "def_zone",
},
{
cnrStr: ".cnr_name",
defZone: "",
expectedName: "cnr_name",
expectedZone: "container",
},
{
cnrStr: ".cnr_name",
defZone: "def_zone",
expectedName: "cnr_name",
expectedZone: "container",
},
{
cnrStr: ".cnr_name",
defZone: "",
expectedName: "cnr_name",
expectedZone: "container",
}, {
cnrStr: "cnr_zone.",
defZone: "def_zone",
expectedName: "",
expectedZone: "cnr_zone",
}, {
cnrStr: "cnr_zone.",
defZone: "",
expectedName: "",
expectedZone: "cnr_zone",
},
} {
t.Run(strconv.Itoa(i), func(t *testing.T) {
actualZone, actualName := getZoneAndContainerNames(tc.cnrStr, tc.defZone)
require.Equal(t, tc.expectedZone, actualZone)
require.Equal(t, tc.expectedName, actualName)
})
}
}
func TestParseContainerCreationPolicy(t *testing.T) {
for i, tc := range []struct {
ACLString string