TrueCloudLab/distribution
14
0
Fork 3
Code Issues 2 Pull requests Projects Releases 2 Packages Wiki Activity Actions

swift: correct segment path generation

Browse source 
When uploading segments to Swift, the registry generates a random file,
by taking the hash of the container path and 32-bytes of random data.
The registry attempts to shard across multiple directory paths, by
taking the first three hex characters as leader.

The implementation in registry, unfortunately, takes the hash of
nothing, and appends it to the path and random data. This results in all
segments being created in one directory.

Fixes: #2407
Fixes: #2311
Signed-off-by: Terin Stock <terinjokes@gmail.com>
This commit is contained in:
Terin Stock 2019-06-22 23:44:43 -07:00
parent 79f6bcbe16
commit b23dd1ef37
2 changed files with 42 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
registry/storage/driver/swift/swift.go
View file

@ -667,13 +667,16 @@ func (d *driver) swiftPath(path string) string {
return strings.TrimLeft(strings.TrimRight(d.Prefix+"/files"+path, "/"), "/")
}
// swiftSegmentPath returns a randomly generated path in the segments directory.
func (d *driver) swiftSegmentPath(path string) (string, error) {
checksum := sha1.New()
random := make([]byte, 32)
if _, err := rand.Read(random); err != nil {
checksum.Write([]byte(path))
if _, err := io.CopyN(checksum, rand.Reader, 32); err != nil {
return "", err
}
path = hex.EncodeToString(checksum.Sum(append([]byte(path), random...)))
path = hex.EncodeToString(checksum.Sum(nil))
return strings.TrimLeft(strings.TrimRight(d.Prefix+"/segments/"+path[0:3]+"/"+path[3:], "/"), "/"), nil
}

36
registry/storage/driver/swift/swift_test.go
View file

@ -243,3 +243,39 @@ func TestFilenameChunking(t *testing.T) {
t.Fatal("expected error for size = -1")
}
}
func TestSwiftSegmentPath(t *testing.T) {
d := &driver{
Prefix: "/test/segment/path",
}
s1, err := d.swiftSegmentPath("foo-baz")
if err != nil {
t.Fatalf("unexpected error generating segment path: %v", err)
}
s2, err := d.swiftSegmentPath("foo-baz")
if err != nil {
t.Fatalf("unexpected error generating segment path: %v", err)
}
if !strings.HasPrefix(s1, "test/segment/path/segments/") {
t.Fatalf("expected to be prefixed: %s", s1)
}
if !strings.HasPrefix(s1, "test/segment/path/segments/") {
t.Fatalf("expected to be prefixed: %s", s2)
}
if len(s1) != 68 {
t.Fatalf("unexpected segment path length, %d != %d", len(s1), 68)
}
if len(s2) != 68 {
t.Fatalf("unexpected segment path length, %d != %d", len(s2), 68)
}
if s1 == s2 {
t.Fatalf("expected segment paths to differ, %s == %s", s1, s2)
}
}