swift: correct segment path generation

When uploading segments to Swift, the registry generates a random file,
by taking the hash of the container path and 32-bytes of random data.
The registry attempts to shard across multiple directory paths, by
taking the first three hex characters as leader.

The implementation in registry, unfortunately, takes the hash of
nothing, and appends it to the path and random data. This results in all
segments being created in one directory.

Fixes: #2407
Fixes: #2311
Signed-off-by: Terin Stock <terinjokes@gmail.com>
This commit is contained in:
Terin Stock 2019-06-22 23:44:43 -07:00
parent 79f6bcbe16
commit b23dd1ef37
2 changed files with 42 additions and 3 deletions

View file

@ -667,13 +667,16 @@ func (d *driver) swiftPath(path string) string {
return strings.TrimLeft(strings.TrimRight(d.Prefix+"/files"+path, "/"), "/") return strings.TrimLeft(strings.TrimRight(d.Prefix+"/files"+path, "/"), "/")
} }
// swiftSegmentPath returns a randomly generated path in the segments directory.
func (d *driver) swiftSegmentPath(path string) (string, error) { func (d *driver) swiftSegmentPath(path string) (string, error) {
checksum := sha1.New() checksum := sha1.New()
random := make([]byte, 32) checksum.Write([]byte(path))
if _, err := rand.Read(random); err != nil {
if _, err := io.CopyN(checksum, rand.Reader, 32); err != nil {
return "", err return "", err
} }
path = hex.EncodeToString(checksum.Sum(append([]byte(path), random...)))
path = hex.EncodeToString(checksum.Sum(nil))
return strings.TrimLeft(strings.TrimRight(d.Prefix+"/segments/"+path[0:3]+"/"+path[3:], "/"), "/"), nil return strings.TrimLeft(strings.TrimRight(d.Prefix+"/segments/"+path[0:3]+"/"+path[3:], "/"), "/"), nil
} }

View file

@ -243,3 +243,39 @@ func TestFilenameChunking(t *testing.T) {
t.Fatal("expected error for size = -1") t.Fatal("expected error for size = -1")
} }
} }
func TestSwiftSegmentPath(t *testing.T) {
d := &driver{
Prefix: "/test/segment/path",
}
s1, err := d.swiftSegmentPath("foo-baz")
if err != nil {
t.Fatalf("unexpected error generating segment path: %v", err)
}
s2, err := d.swiftSegmentPath("foo-baz")
if err != nil {
t.Fatalf("unexpected error generating segment path: %v", err)
}
if !strings.HasPrefix(s1, "test/segment/path/segments/") {
t.Fatalf("expected to be prefixed: %s", s1)
}
if !strings.HasPrefix(s1, "test/segment/path/segments/") {
t.Fatalf("expected to be prefixed: %s", s2)
}
if len(s1) != 68 {
t.Fatalf("unexpected segment path length, %d != %d", len(s1), 68)
}
if len(s2) != 68 {
t.Fatalf("unexpected segment path length, %d != %d", len(s2), 68)
}
if s1 == s2 {
t.Fatalf("expected segment paths to differ, %s == %s", s1, s2)
}
}