From 05996103ecf7f33ad12b65b1ce1af3ce867e5bd5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Apr 2023 20:01:55 +0300 Subject: [PATCH] goodstor: initial implementation Signed-off-by: Evgenii Stratonikov --- .../blobstor/allocator/best_fit.go | 102 +++++++++++++++ .../blobstor/allocator/best_fit_test.go | 47 +++++++ .../blobstor/allocator/bitmap.go | 33 +++++ .../blobstor/allocator/bitmap_test.go | 1 + .../blobstor/allocator/cpu.pprof | Bin 0 -> 6160 bytes .../blobstor/allocator/types.go | 61 +++++++++ .../blobstor/allocator/util_test.go | 13 ++ .../blobstor/goodstor/control.go | 65 ++++++++++ .../blobstor/goodstor/delete.go | 27 ++++ .../blobstor/goodstor/exists.go | 27 ++++ .../blobstor/goodstor/generic_test.go | 72 +++++++++++ .../blobstor/goodstor/get.go | 34 +++++ .../blobstor/goodstor/get_range.go | 29 +++++ .../blobstor/goodstor/iterate.go | 7 ++ .../blobstor/goodstor/put.go | 35 ++++++ .../blobstor/goodstor/uring.go | 117 ++++++++++++++++++ .../blobstor/goodstor/util.go | 58 +++++++++ 17 files changed, 728 insertions(+) create mode 100644 pkg/local_object_storage/blobstor/allocator/best_fit.go create mode 100644 pkg/local_object_storage/blobstor/allocator/best_fit_test.go create mode 100644 pkg/local_object_storage/blobstor/allocator/bitmap.go create mode 100644 pkg/local_object_storage/blobstor/allocator/bitmap_test.go create mode 100644 pkg/local_object_storage/blobstor/allocator/cpu.pprof create mode 100644 pkg/local_object_storage/blobstor/allocator/types.go create mode 100644 pkg/local_object_storage/blobstor/allocator/util_test.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/control.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/delete.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/exists.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/generic_test.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/get.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/get_range.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/iterate.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/put.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/uring.go create mode 100644 pkg/local_object_storage/blobstor/goodstor/util.go diff --git a/pkg/local_object_storage/blobstor/allocator/best_fit.go b/pkg/local_object_storage/blobstor/allocator/best_fit.go new file mode 100644 index 000000000..201b094ba --- /dev/null +++ b/pkg/local_object_storage/blobstor/allocator/best_fit.go @@ -0,0 +1,102 @@ +package allocator + +import ( + "github.com/tidwall/btree" +) + +// BestFit implements best-fit allocation strategy. +type BestFit struct { + capacity uint64 + sizeToFree *btree.BTreeG[Region] + endToFree *btree.BTreeG[Region] +} + +func lessBySize(a, b Region) bool { + if a.Length() == b.Length() { + return a.Offset() < b.Offset() + } + return a.Length() < b.Length() +} + +func lessByEnd(a, b Region) bool { + return a.End() < b.End() +} + +var _ Allocator = (*BestFit)(nil) + +// NewBestFit returns new best-fit allocator instance. +func NewBestFit(capacity uint64) *BestFit { + a := &BestFit{ + capacity: capacity, + sizeToFree: btree.NewBTreeGOptions(lessBySize, btree.Options{NoLocks: true}), + endToFree: btree.NewBTreeGOptions(lessByEnd, btree.Options{NoLocks: true}), + } + for i := uint64(0); i+maxLength < capacity; i += maxLength { + a.addFreePortion(NewRegion(i*maxLength, maxLength)) + } + c := capacity / maxLength * maxLength + a.addFreePortion(NewRegion(c, capacity-c)) + return a +} + +// Get implements the Allocator interface. +func (b *BestFit) Get(size uint64) (Region, error) { + p := NewRegion(0, uint64(size)) + iter := b.sizeToFree.Iter() + if iter.Seek(p) { + free := iter.Item() + + b.deleteFreePortion(free) + + reg := free.allocate(size) + if free.Length() != 0 { + b.addFreePortion(free) + } + return reg, nil + } + + return 0, ErrOOM +} + +// Free implements the Allocator interface. +func (b *BestFit) Free(region Region) error { + p := b.mergeIfPossible(region) + b.addFreePortion(p) + return nil +} + +func (b *BestFit) mergeIfPossible(free Region) Region { + iter := b.endToFree.Iter() + + key := NewRegion(free.Offset(), 0) + if iter.Seek(key) { + prev := iter.Item() + if prev.End() == free.Offset() && prev.safeExtend(free.Length()) { + free = NewRegion(prev.Offset(), prev.Length()+free.Length()) + b.deleteFreePortion(prev) + } + } + + key = NewRegion(free.End(), 0) + if iter.Seek(key) { + next := iter.Item() + if next.Offset() == free.End() && free.safeExtend(next.Length()) { + free = NewRegion(free.Offset(), free.Length()+next.Length()) + b.deleteFreePortion(next) + } + } + + iter.Release() + + return free +} + +func (b *BestFit) deleteFreePortion(p Region) { + b.sizeToFree.Delete(p) + b.endToFree.Delete(p) +} + +func (b *BestFit) addFreePortion(p Region) { + b.sizeToFree.Set(p) + b.endToFree.Set(p) +} diff --git a/pkg/local_object_storage/blobstor/allocator/best_fit_test.go b/pkg/local_object_storage/blobstor/allocator/best_fit_test.go new file mode 100644 index 000000000..578c444bd --- /dev/null +++ b/pkg/local_object_storage/blobstor/allocator/best_fit_test.go @@ -0,0 +1,47 @@ +package allocator + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBestFit(t *testing.T) { + a := NewBestFit(4096) + testGet(t, a, 1024, NewRegion(0, 1024)) + testGet(t, a, 1024, NewRegion(1024, 1024)) + + require.NoError(t, a.Free(NewRegion(0, 1024))) + testGet(t, a, 1024, NewRegion(0, 1024)) + + require.NoError(t, a.Free(NewRegion(0, 1024))) + testGet(t, a, 1025, NewRegion(2048, 1025)) + testGet(t, a, 512, NewRegion(2048+1025, 512)) + testGet(t, a, 512, NewRegion(0, 512)) + testGet(t, a, 512, NewRegion(512, 512)) + + _, err := a.Get(512) + require.True(t, err == ErrOOM) + testGet(t, a, 511, NewRegion(2048+1025+512, 511)) + + require.NoError(t, a.Free(NewRegion(0, 1024))) + require.NoError(t, a.Free(NewRegion(2048, 2048))) + require.NoError(t, a.Free(NewRegion(1024, 1024))) + testGet(t, a, 4096, NewRegion(0, 4096)) +} + +func BenchmarkBestFit(b *testing.B) { + benchmarkAllocator(b, NewBestFit) +} + +func benchmarkAllocator[A Allocator](b *testing.B, constructor func(uint64) A) { + b.Run("by one", func(b *testing.B) { + a := constructor(uint64(b.N)) + for i := 0; i < b.N; i++ { + _, err := a.Get(1) + if err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/pkg/local_object_storage/blobstor/allocator/bitmap.go b/pkg/local_object_storage/blobstor/allocator/bitmap.go new file mode 100644 index 000000000..9158743a4 --- /dev/null +++ b/pkg/local_object_storage/blobstor/allocator/bitmap.go @@ -0,0 +1,33 @@ +package allocator + +import ( + "github.com/RoaringBitmap/roaring" +) + +// Bitmap allocates chunks of a constant size. +type Bitmap struct { + free roaring.Bitmap + allocated roaring.Bitmap +} + +func NewBitmap(capacity uint64) *Bitmap { + b := &Bitmap{} + b.free.AddRange(0, capacity) + return b +} + +func (b *Bitmap) Get(size int) (Region, error) { + if size != 1 { + panic("not implemented") + } + r := b.free.Minimum() + b.free.Remove(r) + b.allocated.Add(r) + return NewRegion(uint64(r), uint64(size)), nil +} + +func (b *Bitmap) Free(region Region) error { + b.allocated.RemoveRange(region.Offset(), region.End()) + b.free.AddRange(region.Offset(), region.End()) + return nil +} diff --git a/pkg/local_object_storage/blobstor/allocator/bitmap_test.go b/pkg/local_object_storage/blobstor/allocator/bitmap_test.go new file mode 100644 index 000000000..4dc59df21 --- /dev/null +++ b/pkg/local_object_storage/blobstor/allocator/bitmap_test.go @@ -0,0 +1 @@ +package allocator diff --git a/pkg/local_object_storage/blobstor/allocator/cpu.pprof b/pkg/local_object_storage/blobstor/allocator/cpu.pprof new file mode 100644 index 0000000000000000000000000000000000000000..c14b3a1c1caeb3180074b84c994006c4902bf17b GIT binary patch literal 6160 zcmV+r81LsFiwFP!00004|Fn5|d=zE!_xt4H$%G^|0g~cwn!`?jnWS@&C@Pm+atA~~ zgr1qsFfvD+nQ-W;ksAa-P+1Q&Dk3VliXyI{A|faXc;LmZE9$(|v8 zpVPeKqb#|EGyL|=B^h!yXRs?r&gL1go&zh(;|#JfTgp*(=9J~}3}}b#q#WfhIWq7J z-1?zdk_n&X3f)ZL$fX?R(_FcjXW}P|KhBV|cos~Nvw1eW&VZGKznR2xs?4(F6P$( zK|bb7F69d{&Y=Jcq(Wu3T*!0qlT{zl0dk>2?3@dLAQdX-$c4NeuHP(#E+`d37r4;2dvQwv)EXgBOgG` zSA;qPc5+}v9jx`Py!NSGfal}kS539Q6m%4QN5;uA|g zCUG{r!GNL+c>IhgYrq^%_ZzXg6Wna9?gV|YuhdIfO{+WOpIdIH)y43CjffUQKkO&m2k13!6FluTFn!0@yy z^vC{^OF2%S_QV}0gt!|VG{oIt01lvnBe8_*{~*NO;dw*c9R}h+$))Vcl#6*UJicCt zd%&(tIg9s%%SAAIfCoL2OL;Q0%*A_x7rj!UGD9xpGOo00U|<@CyLIo;Q|cufpy`db z{YIFT;47mNBq+yn$)$Y8=D+BL`pDY*R-ta;@A?^)TSS3{|d#L!%!7a;# zxDV_x#C@O|tEEciDH1FA-SXvRyf1ubkVRjp!5V3ha+*r46kl2*#Qk6^iCw%O)MBkP zNSQ1HFT@pZ9RDI7D5mBL^gER`s)k}(e+vnc2p zGS<;DmN(GjBpEOM^cRtVGI-c1fHF8A&zD@vRw{sU++bnu-Aq9#75MZSK$)3FDcJJ5SHZOoLDG4s#A1=TPqzjdN8F&q@*)IGY06%h(g8?uEhe$5v6qgJ6ApFcC!7^Y) zh1M=P)ir|%eTBRh@3b^Z!AX;~BnMuHhjy47EXf5{Q~(*!eBwZ^E4xlu0Ddm+Il6`* zHV}R`SYsdz#i7zLb8{Pi{= zR$+xf5-NJuz3i(LZ-B<*gQ9)&;sQy9vIqYRw*8)f=Y!bjtK zM+B0qU=7q5b`I957Wxvr?lvYtCS)p~)C!J9UVq*cS#JdyHhxEL>%DwPv1 z8TeSd>oy^lVV6s|bulh|L+q3RKePi@)YXc8(U`F&S6rO3U&zPd3d_aW!bNkaag8Gv zEv_Dq``!>Pc7@}aX}KD2Rcj%ifZtp2@1F+$2?qQp;%AQvBNBX00k`2l(WLK5xX(G( zIO=^;frSa+lkqEOofZ{XXU?8%=4}dY_+`{gA*Zao)#KV@W~9ERNZIUCp9&E0sra)M z&sAJRcpCo68nreMaBaSsI?bR0;FsV-_X}TZ;BUsrTm$29oa9noqee6x-%Miy8+(^b z7d`;L6c1bUN-f<{9+%oZ^fLVG2Vxteb35!@W~FWhervUkGdOgHgF|QHCuu&}2byWQ zbU8k;OpK5@u%BRFXyG;Ia%=dVg`ZmpAd0ky-&qC%uE4iiVd^VV&1*JpKV{OwqqL39 z9kXo+%)#&95q@`tkJ9lt$HF6U4VS)c=63_pt({-Z;kgg*x>d}sgJ7@G69&O}94}2! z3T5Da{M3rxZG>5S9{2e(?2a42H6MtrYTx}BpfGvcy4p!(~ z0o?($20UrO|C^k&&e@RKtAq#fQ|kivgnfZ?&_^((T+HXilwMtI0XcnQA}@82ev=Un(WTh8Vlc;09#=fY&1 zELAFh%yz81GDZ9I@H>kwk5GPX+w)Ae1RlntYsK(<9_%p2xbt8NPLb-BKT$X%_}8CS zP^TIU$Bafj7^dP>sa{#bW#CbKZH*A04+o6Z=fgCdCS9WJCd!QAC(DHR0@!VcFM#Pd zUAjbhKi9z};YhS3CIfH8)9;#4`!pSDjW*P-!fh7RM8IsQUF9III3A#6SZQ#$9#4@{ z0+(A*p_`qxhnR!nnpJj9=k5R#=~=3bax8s}!QI#XlSUfLVq0J*nEcyf&2q;Md`; z7W3{U4{XA`&SKu{ao_I+a=OEwbd~E36*q5Y*nC=qZ8$U=qxx`|g|nnS%D=K@H@|`5 zA?rQhAjP1Q)$7_fvm5Mp2jq(xe(+1QU=!A2Tiwl1M%zsngmzjq6PYy@m4w$0-s;0nA#noZ9XZoY!yv)_y2 zya*1_fRx8Cf;l)xs#k7nC%gGdhD*N_@{w?_5wDS;p(fQUEjhBAuVT3UrA=gi6ud@b zsEdyRANnM}BFS#Pn&Hh~iRGhVi?Mt(1TY}gD<4p}Z)AAOAt4_F-x$Xm13Kzbz4A7b z-Fywh1N(%0EWB;V$3g=(NcGBnBwx$$xy37K`@yh-($&pMSBK?n1zX8hvDNHGwuY@` zL@sWA6T|&!UET)UP0XqUH($r_RNDMytF2?!G=O|P!|iX0>%+xxKb>M6++uinF$6Iv z%~kHpD9hvHAcP@ln6iN)b~D2R4+|dm!29jNIbyG8H#3vA-FySXpRGg^|G1I7<|C-?`%iV8C%eMAwa@PQU z3BR4;sega`blAL|nYwO%2gBc5HU+oaJKtgM?D(|E%4-R~li~hfc&oE1=T0Ub>fHP; zhDVQz=x4wOL|*o>@1h_W$9D7048O9D@@=|%n{D@i-_7t}cZruj7bs~n{oRaHE53)} z+xx}y-2|8_o~0(hJe((mm2tA0w=mqYR)A|F{NCu)6Cr{TsZx2J%Wi%z!>t>HT!t?Q zEIlny72m^J*uBiWu+Za%F=M*5o$+tve80>LOZFYXPQrv};j_Rh4BXZ_YB;rru^g)Ij-xDQ21^!_$(iFG~ zuae@*yTnM_7;f1tUNP(86+>PR2~0?F5rbk(--~5K*YHKiC zM*y?6eq#smF@}4stIdm)b9*{?jEV5e$RB68>9D}+bl3@UHop`;0uhkuumBfGmC6n% z%j1{ALR=_SD$j!K<~tc4xKpgDh96wO9Ns_9b~5WeKEO;T@+TM`GrX^Y#r8XaBl}OJ zU?RKulMMG7O+N!}N-yIlNl%>J!4ubuQSgi*!oiKZ7@mCJ%+`L&mL0BLc8Z>2xZG-5 z^z!Gx>r;-VwVUDb(;KPm3gAR~RChDuQt9STGu)jvz1f}ZX=aUYZvI<_pITdr2MAm3 zw~hw*48whgMNhd54rIyMdj%=6e|)wutjqintx2y*8kpWw_gU{dWNBS!P`D-TXO* zC#`4NSBX9B+3q_EXXG%=#;ho9|<|?``3IKKx9jV^3lG z#I$BP@dCr$7Hrm%6E+LIkb;ejd_TiKz9&d)CVY`8Uah_}SbQd2gV#vEQa;agtljTi z`y#`~+a|mhZFM(4!0@em1oCR&1Qn`Xf(MLL&!!7R!4L_VPH2koX;ZoAL zl{Ahq{J|RWUZ7m{wJN6DA@(wRg&k(Eve(!VR&tabWAwmbguH~m&hTWL)6(nqwDbnU zoz{c%2I8Xj<}UmV#)rsm{wBjM*3JB5Qm{GUO^Xv8-$q-5K^He?(A?adfzA6q&QO-I zTGpi-&w%EZjUV3rLr-iz`33!yn)h*KIV)emx^(54(A=`@q>#3(pr2Cn9AzadUq!+! zXg;})K4Npr?etS>mXy`3{6^NL6VHa`eINb(tDk#fbMwdaQ&~f6a_9vA{F(kSY;LiB zRn{`~|ETQ}TBI?oC)8|ztSOpQGyRQCYF<=}#uBI){s0=hcHJ2w{5 zy@9Lsq(*x;&I@`Yu|TVKhRm<<)OgAbqd`wFcJ^B)^yKJJ^glRby}6+O**aCPC)LJa zDCud4#}dhggeMejh#Ee+G9ji#g{C1-L<`IhC3Lm4)g_A(v%{fi)56(WBv4c3 zNvIvHwTVTEY0;*H9++7F@7|kA-iRJC*3-7Bv!<4+JzCp1qDN?nWE%1)WsO=;7wIka zB>dWZJsQ*}Ms>Acs}_QOOOGt|Zqt&96sL1ZJ(5Uj{&{Ne)^^YmkyKc^w`pGOV~edr z#p{z=QZMy{qC^}`QNQXma;8?iDXJD(Mtng_eD*BpOXynMKUW=awxre&iUw5A*=h&$ zu%6Ua^*_@WynNPcMuQGXTpS|`N((Ynm zK`Dz6ZhWv-xw<%YnsxeG>WS+Op>Vj3Aw}qsZD%;dN18N@IY;ek9eY7Ml+;I?B+{sk)`^v}#G=G_Jwj9= z`hOd-NF=png0|{uuNDa}^)$qF5#w<&|2#d=hJpzI$Npzna3UwMK-U`$lcgT=4K~jw{94qQ5Zjmqf_@r&D6pxv9`uihYavIng8pH_@mhS|^jLhJ9v4|w+vvSP zG5DuwAbquclb{YUr70TKd|_Sf*;2qYJNRVXG#r&3w*Q@P%Xk zd1@!qMhf}E#OKr@Olq6pjGMu}!uJZh+XtQB%><;ZrwP^Z|HBKSz$)E3J#@cQi&8}x z4*C3^M9fp8DpgvQPxn^_>Z-Mh+VZNZs!F}KTCJc%LfGlbrqF$)fLs%Re{>dKuv93O>I?Gd99i= ibABKZo29yD&QB~#__c6&*8d9t0RR7sLL^`}G5`P$P65#X literal 0 HcmV?d00001 diff --git a/pkg/local_object_storage/blobstor/allocator/types.go b/pkg/local_object_storage/blobstor/allocator/types.go new file mode 100644 index 000000000..471bf5763 --- /dev/null +++ b/pkg/local_object_storage/blobstor/allocator/types.go @@ -0,0 +1,61 @@ +package allocator + +import ( + "errors" + "fmt" + "math" +) + +type Allocator interface { + Get(uint64) (Region, error) + Free(Region) error +} + +var ( + ErrOOM = errors.New("no free chunk is found") +) + +// Region describes continuous range of data as offset and length. +// Offset is 40-bit, for 512-byte chunks this means it can contain 2^40 * 2^9 = 512 TiB of data. +// Length is 24-bit, for 512-byte chunks this means it can contain 2^24 * 2^9 = 8 GiB of data. +type Region uint64 + +const ( + lenShift = 40 + maxLength = math.MaxUint64 >> lenShift + offsetMask = (1 << lenShift) - 1 +) + +func NewRegion(offset, length uint64) Region { + return Region((length << lenShift) | offset) +} + +func (r Region) Offset() uint64 { + return uint64(r & offsetMask) +} + +func (r Region) Length() uint64 { + return uint64(r >> lenShift) +} + +func (r Region) End() uint64 { + return r.Offset() + r.Length() +} + +func (r *Region) safeExtend(size uint64) bool { + return r.Length()+size < maxLength +} + +func (r *Region) allocate(size uint64) Region { + assert(size <= r.Length(), "invalid allocation size: have=%d, need=%d", r.Length(), size) + + result := NewRegion(r.Offset(), size) + *r = NewRegion(r.Offset()+size, r.Length()-size) + return result +} + +func assert(ok bool, msg string, args ...any) { + if !ok { + panic(fmt.Sprintf(msg, args...)) + } +} diff --git a/pkg/local_object_storage/blobstor/allocator/util_test.go b/pkg/local_object_storage/blobstor/allocator/util_test.go new file mode 100644 index 000000000..3fbaac7a6 --- /dev/null +++ b/pkg/local_object_storage/blobstor/allocator/util_test.go @@ -0,0 +1,13 @@ +package allocator + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func testGet(t *testing.T, a Allocator, size int, expected Region) { + r, err := a.Get(uint64(size)) + require.NoError(t, err) + require.Equal(t, expected, r) +} diff --git a/pkg/local_object_storage/blobstor/goodstor/control.go b/pkg/local_object_storage/blobstor/goodstor/control.go new file mode 100644 index 000000000..a670b4b40 --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/control.go @@ -0,0 +1,65 @@ +package goodstor + +import ( + "os" + "path/filepath" + + "codeberg.org/fyrchik/uring/loop" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" +) + +const idToStringBase = 16 + +type nameIDPair struct { + name string + id uint16 +} + +func (s *Storage) Open(readOnly bool) error { + s.readOnly = readOnly + + if s.loopSize != 0 { + lp, err := loop.New(s.loopSize, &s.loopParams) + if err != nil { + return err + } + s.loop = lp + } + + if !readOnly { + if err := util.MkdirAllX(s.path, os.ModePerm); err != nil { + return err + } + } + + p := filepath.Join(s.path, "backend") + b, err := s.openSlab(p) + if err != nil { + return err + } + + s.backend = b + return nil +} + +func (s *Storage) Init() error { + return nil +} + +func (s *Storage) Close() error { + if s.backend != nil { + if err := s.backend.Close(); err != nil { + return err + } + s.backend = nil + } + + if s.loop != nil { + if err := s.loop.Close(); err != nil { + return err + } + s.loop = nil + } + + return nil +} diff --git a/pkg/local_object_storage/blobstor/goodstor/delete.go b/pkg/local_object_storage/blobstor/goodstor/delete.go new file mode 100644 index 000000000..946d56cc4 --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/delete.go @@ -0,0 +1,27 @@ +package goodstor + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/allocator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" +) + +func (s *Storage) Delete(prm common.DeletePrm) (common.DeleteRes, error) { + if s.readOnly { + return common.DeleteRes{}, common.ErrReadOnly + } + + if prm.StorageID == nil { + return common.DeleteRes{}, errNotImplemented + } + + offset, length, err := parseStorageID(prm.StorageID) + if err != nil { + return common.DeleteRes{}, err + } + + s.allocMtx.Lock() + r := allocator.NewRegion(offset, (length+s.blockSize-1)/s.blockSize) + err = s.allocator.Free(r) + s.allocMtx.Unlock() + return common.DeleteRes{}, err +} diff --git a/pkg/local_object_storage/blobstor/goodstor/exists.go b/pkg/local_object_storage/blobstor/goodstor/exists.go new file mode 100644 index 000000000..b05da2261 --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/exists.go @@ -0,0 +1,27 @@ +package goodstor + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" +) + +func (s *Storage) Exists(prm common.ExistsPrm) (common.ExistsRes, error) { + //if prm.StorageID == nil { + return common.ExistsRes{}, errNotImplemented + //} + + // r, err := parseStorageID(prm.StorageID) + // if err != nil { + // return common.ExistsRes{}, err + // } + + // s.slabMtx.RLock() + // ss, ok := s.slabMap[slabID] + // s.slabMtx.RUnlock() + + // if !ok { + // return common.ExistsRes{}, nil + // } + + // exists, err := ss.Exists(offset) + // return common.ExistsRes{Exists: exists}, err +} diff --git a/pkg/local_object_storage/blobstor/goodstor/generic_test.go b/pkg/local_object_storage/blobstor/goodstor/generic_test.go new file mode 100644 index 000000000..dffee7610 --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/generic_test.go @@ -0,0 +1,72 @@ +package goodstor + +import ( + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "codeberg.org/fyrchik/uring/loop" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +func TestGeneric(t *testing.T) { + _ = os.RemoveAll(t.Name()) + defer func() { _ = os.RemoveAll(t.Name()) }() + + helper := func(t *testing.T, dir string) common.Storage { + s, err := New( + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithRootPath(dir), + WithBlockSize(8192), + WithCapacity(1024*1024), + WithLoopParams(32, loop.Params{ + RingCount: 1, + SubmissionTimer: 10 * time.Microsecond, + })) + require.NoError(t, err) + return s + } + + var n int + newUringStor := func(t *testing.T) common.Storage { + dir := filepath.Join(t.Name(), strconv.Itoa(n)) + return helper(t, dir) + } + + min := uint64(1024) + max := uint64(4096) + blobstortest.TestAll(t, newUringStor, min, max) + + t.Run("info", func(t *testing.T) { + dir := filepath.Join(t.Name(), "info") + blobstortest.TestInfo(t, func(t *testing.T) common.Storage { + return helper(t, dir) + }, Type, dir) + }) +} + +func TestControl(t *testing.T) { + _ = os.RemoveAll(t.Name()) + defer func() { _ = os.RemoveAll(t.Name()) }() + + var n int + newUringStor := func(t *testing.T) common.Storage { + dir := filepath.Join(t.Name(), strconv.Itoa(n)) + s, err := New( + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithRootPath(dir), + WithBlockSize(4096), + WithCapacity(1024*1024), + WithLoopParams(32, loop.Params{})) + require.NoError(t, err) + return s + } + + blobstortest.TestControl(t, newUringStor, 1024, 2048) +} diff --git a/pkg/local_object_storage/blobstor/goodstor/get.go b/pkg/local_object_storage/blobstor/goodstor/get.go new file mode 100644 index 000000000..d95684769 --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/get.go @@ -0,0 +1,34 @@ +package goodstor + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +var errInvalidStorageID = errors.New("invalid storage ID") + +func (s *Storage) Get(prm common.GetPrm) (common.GetRes, error) { + if prm.StorageID == nil { + return common.GetRes{}, errNotImplemented + } + + offset, length, err := parseStorageID(prm.StorageID) + if err != nil { + return common.GetRes{}, err + } + + data := make([]byte, length) + err = s.backend.ReadAt(data, int64(offset*s.blockSize)) + if err != nil { + return common.GetRes{}, err + } + + obj := objectSDK.New() + if err := obj.Unmarshal(data); err != nil { + return common.GetRes{}, err + } + + return common.GetRes{Object: obj, RawData: data}, nil +} diff --git a/pkg/local_object_storage/blobstor/goodstor/get_range.go b/pkg/local_object_storage/blobstor/goodstor/get_range.go new file mode 100644 index 000000000..15f16e6c5 --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/get_range.go @@ -0,0 +1,29 @@ +package goodstor + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" +) + +func (s *Storage) GetRange(prm common.GetRangePrm) (common.GetRangeRes, error) { + res, err := s.Get(common.GetPrm{ + Address: prm.Address, + StorageID: prm.StorageID, + }) + if err != nil { + return common.GetRangeRes{}, err + } + + payload := res.Object.Payload() + from := prm.Range.GetOffset() + to := from + prm.Range.GetLength() + + if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { + return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectOutOfRange{}) + } + + return common.GetRangeRes{ + Data: payload[from:to], + }, nil +} diff --git a/pkg/local_object_storage/blobstor/goodstor/iterate.go b/pkg/local_object_storage/blobstor/goodstor/iterate.go new file mode 100644 index 000000000..1e80fe393 --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/iterate.go @@ -0,0 +1,7 @@ +package goodstor + +import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + +func (s *Storage) Iterate(common.IteratePrm) (common.IterateRes, error) { + return common.IterateRes{}, errNotImplemented +} diff --git a/pkg/local_object_storage/blobstor/goodstor/put.go b/pkg/local_object_storage/blobstor/goodstor/put.go new file mode 100644 index 000000000..ae323783d --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/put.go @@ -0,0 +1,35 @@ +package goodstor + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" +) + +func (s *Storage) Put(prm common.PutPrm) (common.PutRes, error) { + if s.readOnly { + return common.PutRes{}, common.ErrReadOnly + } + if prm.RawData == nil { + panic("unexpected") + } + + up2 := s.blockSize + for ; up2 <= uint64(len(prm.RawData)); up2 *= 2 { + } + + s.allocMtx.Lock() + r, err := s.allocator.Get(up2 / s.blockSize) + s.allocMtx.Unlock() + if err != nil { + return common.PutRes{}, err + } + + err = s.backend.WriteAt(prm.RawData, int64(r.Offset()*s.blockSize)) + if err != nil { + s.allocMtx.Lock() + _ = s.allocator.Free(r) + s.allocMtx.Unlock() + } + + storageID := marshalStorageID(r.Offset(), uint64(len(prm.RawData))) + return common.PutRes{StorageID: storageID[:]}, nil +} diff --git a/pkg/local_object_storage/blobstor/goodstor/uring.go b/pkg/local_object_storage/blobstor/goodstor/uring.go new file mode 100644 index 000000000..564102e7d --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/uring.go @@ -0,0 +1,117 @@ +package goodstor + +import ( + "sync" + + "codeberg.org/fyrchik/uring/loop" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/allocator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/uringstor/slab" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "go.uber.org/zap" +) + +var _ common.Storage = (*Storage)(nil) + +type Option = func(*Storage) + +type Storage struct { + capacity uint64 + path string + blockSize uint64 + noSync bool + readOnly bool + loopSize int + loopParams loop.Params + loop *loop.Loop + log *logger.Logger + + backend slab.Backend + + allocMtx sync.Mutex + allocator allocator.Allocator +} + +const ( + defaultBlockSize = 512 + defaultUringLoopSize = 1024 +) + +func New(opts ...Option) (*Storage, error) { + var s Storage + s.blockSize = defaultBlockSize + s.loopSize = defaultUringLoopSize + s.log = &logger.Logger{Logger: zap.NewNop()} + for i := range opts { + opts[i](&s) + } + + s.allocator = allocator.NewBestFit(s.capacity / s.blockSize) + return &s, nil +} + +// Type is uring storage type used in logs and configuration. +const Type = "goodstor" + +// Type implements the common.Storage interface. +func (s *Storage) Type() string { + return Type +} + +// Path implements the common.Storage interface. +func (s *Storage) Path() string { + return s.path +} + +// SetCompressor implements the common.Storage interface. +func (s *Storage) SetCompressor(cc *compression.Config) {} + +// SetReportErrorFunc allows to provide a function to be called on disk errors. +// This function MUST be called before Open. +func (s *Storage) SetReportErrorFunc(f func(string, error)) {} + +// WithCapacity sets the max capacity of the storage. +func WithCapacity(capacity uint64) Option { + return func(s *Storage) { + s.capacity = capacity + } +} + +// WithRootPath sets the max capacity of the storage. +func WithRootPath(dir string) Option { + return func(s *Storage) { + s.path = dir + } +} + +func WithLoopParams(size int, p loop.Params) Option { + return func(s *Storage) { + s.loopSize = size + s.loopParams = p + } +} + +func WithOSBackend() Option { + return func(s *Storage) { + s.loopSize = 0 + } +} + +func WithBlockSize(size uint64) Option { + return func(s *Storage) { + s.blockSize = uint64(size) + } +} + +func WithLogger(l *logger.Logger) Option { + return func(s *Storage) { + s.log = &logger.Logger{Logger: l.With(zap.String("component", Type))} + } +} + +func WithNoSync(noSync bool) Option { + return func(s *Storage) { + s.noSync = noSync + } +} diff --git a/pkg/local_object_storage/blobstor/goodstor/util.go b/pkg/local_object_storage/blobstor/goodstor/util.go new file mode 100644 index 000000000..2c1f3e1ed --- /dev/null +++ b/pkg/local_object_storage/blobstor/goodstor/util.go @@ -0,0 +1,58 @@ +package goodstor + +import ( + "encoding/binary" + "fmt" + "os" + "syscall" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/uringstor/slab" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" +) + +var errNotImplemented = logicerr.Wrap(fmt.Errorf("not implemented: %w", apistatus.ObjectNotFound{})) + +func parseStorageID(storageID []byte) (uint64, uint64, error) { + if len(storageID) != 10 { + return 0, 0, errInvalidStorageID + } + + offset := binary.LittleEndian.Uint64(storageID[:8]) + length := binary.LittleEndian.Uint16(storageID[8:]) + return offset, uint64(length), nil +} + +func marshalStorageID(offset, length uint64) [10]byte { + var storageID [10]byte + binary.LittleEndian.PutUint64(storageID[:], offset) + binary.LittleEndian.PutUint16(storageID[8:], uint16(length)) + return storageID +} + +func assert(ok bool, msg string, args ...any) { + if !ok { + panic(fmt.Sprintf(msg, args...)) + } +} + +func (s *Storage) openSlab(p string) (slab.Backend, error) { + var b slab.Backend + if s.loop == nil { + b = slab.NewFileBackend(p) + } else { + b = slab.NewUringBackend(p, s.loop) + } + + var flags int + if s.readOnly { + flags |= os.O_RDONLY + } else { + flags |= os.O_RDWR | os.O_CREATE | syscall.O_DIRECT + } + + if !s.noSync { + flags |= os.O_SYNC + } + return b, b.Open(flags) +}