[#159] Add handle __SYSTEM__ sys attributes

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>

CHANGELOG.md

@@ -26,6 +26,7 @@ Changelog for FrostFS Node
 - `neofs-cli` buffer for object put increased from 4 KiB to 3 MiB (#2243)
 - Expired locked object is available for reading (#56)
 - Initialize write-cache asynchronously (#32)
+- Update system attribute names (#159)
 
 ### Fixed
 - Increase payload size metric on shards' `put` operation (#1794)
@@ -70,6 +71,9 @@ You need to change configuration environment variables to `FROSTFS_*` if you use
 New config field `object.delete.tombstone_lifetime` allows to set tombstone lifetime
 more appropriate for a specific deployment.
 
+Use `__SYSTEM__` prefix for system attributes instead of `__NEOFS__` 
+(existed objects with old attributes will be treated as before, but for new objects new attributes will be used).
+
 ## [0.35.0] - 2022-12-28 - Sindo (신도, 信島)
 
 ### Added

cmd/frostfs-cli/docs/storage-node-xheaders.md

@@ -9,16 +9,16 @@ duplicated header names or headers with empty values are considered invalid.
 
 ## Existing headers
 
-There are some "well-known" headers starting with `__FROSTFS__` prefix that
+There are some "well-known" headers starting with `__SYSTEM__` prefix that
 affect system behaviour. For backward compatibility, the same set of
 "well-known" headers may also use `__NEOFS__` prefix:
 
-* `__FROSTFS__NETMAP_EPOCH` - netmap epoch to use for object placement calculation. The `value` is string
+* `__SYSTEM__NETMAP_EPOCH` - netmap epoch to use for object placement calculation. The `value` is string
 encoded `uint64` in decimal presentation. If set to '0' or omitted, the
 current epoch only will be used.
-* `__FROSTFS__NETMAP_LOOKUP_DEPTH` - if object can't be found using current epoch's netmap, this header limits
+* `__SYSTEM__NETMAP_LOOKUP_DEPTH` - if object can't be found using current epoch's netmap, this header limits
 how many past epochs the node can look up through. Depth is applied to a current epoch or the value
-of `__FROSTFS__NETMAP_EPOCH` attribute. The `value` is string encoded `uint64` in decimal presentation.
+of `__SYSTEM__NETMAP_EPOCH` attribute. The `value` is string encoded `uint64` in decimal presentation.
 If set to '0' or not set, only the current epoch is used.
 
 ## `frostfs-cli` commands with `--xhdr`
@@ -30,5 +30,5 @@ List of commands with support of extended headers:
 
 Example:
 ```shell
-$ frostfs-cli object put -r s01.frostfs.devenv:8080 -w wallet.json --cid CID --file FILE --xhdr "__FROSTFS__NETMAP_EPOCH=777"
+$ frostfs-cli object put -r s01.frostfs.devenv:8080 -w wallet.json --cid CID --file FILE --xhdr "__SYSTEM__NETMAP_EPOCH=777"
 ```

cmd/frostfs-cli/modules/container/list.go

@@ -62,7 +62,7 @@ var listContainersCmd = &cobra.Command{
 				res, err := internalclient.GetContainer(prmGet)
 				if err == nil {
 					res.Container().IterateAttributes(func(key, val string) {
-						if !strings.HasPrefix(key, container.SysAttributePrefix) {
+						if !strings.HasPrefix(key, container.SysAttributePrefix) && !strings.HasPrefix(key, container.SysAttributePrefixNeoFS) {
 							// FIXME(@cthulhu-rider): neofs-sdk-go#314 use dedicated method to skip system attributes
 							cmd.Printf("  %s: %s\n", key, val)
 						}

cmd/frostfs-cli/modules/container/list_objects.go

@@ -70,7 +70,7 @@ var listContainerObjectsCmd = &cobra.Command{
 					attrs := resHead.Header().Attributes()
 					for i := range attrs {
 						attrKey := attrs[i].Key()
-						if !strings.HasPrefix(attrKey, v2object.SysAttributePrefix) {
+						if !strings.HasPrefix(attrKey, v2object.SysAttributePrefix) && !strings.HasPrefix(attrKey, v2object.SysAttributePrefixNeoFS) {
 							// FIXME(@cthulhu-rider): neofs-sdk-go#226 use dedicated method to skip system attributes
 							cmd.Printf("  %s: %s\n", attrKey, attrs[i].Value())
 						}

go.mod

@@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-node
 go 1.18
 
 require (
-	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51
+	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703
 	git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb
-	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599
+	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85
 	git.frostfs.info/TrueCloudLab/hrw v1.2.0
 	git.frostfs.info/TrueCloudLab/tzhash v1.8.0
 	github.com/cheggaaa/pb v1.0.29

go.sum

@@ -36,14 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51 h1:l4+K1hN+NuWNtlZZoV8yRRP3Uu7PifL05ukEqKcb0Ks=
-git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230307104236-f69d2ad83c51/go.mod h1:n0DxKYulu2Ar73R6OcNF34LiL/Xa+iDR7GZuaOChbLE=
+git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 h1:lxe0DtZq/uFZVZu9apx6OcIXCJskQBMd/GVeYGKA3wA=
+git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703/go.mod h1:gRd5iE5A84viily6AcNBsSlTx2XgoWrwRDz7z0MayDQ=
 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M=
 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o=
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599 h1:mzGX2RX8R8H/tUqrUu1TcYk4QRDBcBIWGYscPncfLOQ=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230307124721-94476f905599/go.mod h1:z7zcpGY+puI5puyy5oyFbf20vWp84WtslCxcr6/kv5c=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 h1:TUcJ5A0C1gWi3bAhw4b+V+iVM3E9mbBOdJIWWkAPNxo=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85/go.mod h1:23fUGlEv/ImaOi3vck6vZj0v0b4hteOhLLPnVWHSQeA=
 git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk=
 git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y=
 git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA=

pkg/core/object/fmt.go

@@ -353,7 +353,7 @@ func (v *FormatValidator) checkExpiration(obj *object.Object) error {
 
 func expirationEpochAttribute(obj *object.Object) (uint64, error) {
 	for _, a := range obj.Attributes() {
-		if a.Key() != objectV2.SysAttributeExpEpoch {
+		if a.Key() != objectV2.SysAttributeExpEpoch && a.Key() != objectV2.SysAttributeExpEpochNeoFS {
 			continue
 		}
 

pkg/local_object_storage/metabase/exists.go

@@ -110,20 +110,9 @@ func objectStatus(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) uint8 {
 	// GC is expected to collect all the objects that have
 	// expired previously for less than the one epoch duration
 
-	var expired bool
-
-	// bucket with objects that have expiration attr
-	attrKey := make([]byte, bucketKeySize+len(objectV2.SysAttributeExpEpoch))
-	expirationBucket := tx.Bucket(attributeBucketName(addr.Container(), objectV2.SysAttributeExpEpoch, attrKey))
-	if expirationBucket != nil {
-		// bucket that contains objects that expire in the current epoch
-		prevEpochBkt := expirationBucket.Bucket([]byte(strconv.FormatUint(currEpoch-1, 10)))
-		if prevEpochBkt != nil {
-			rawOID := objectKey(addr.Object(), make([]byte, objectKeySize))
-			if prevEpochBkt.Get(rawOID) != nil {
-				expired = true
-			}
-		}
+	expired := isExpiredWithAttribute(tx, objectV2.SysAttributeExpEpoch, addr, currEpoch)
+	if !expired {
+		expired = isExpiredWithAttribute(tx, objectV2.SysAttributeExpEpochNeoFS, addr, currEpoch)
 	}
 
 	if expired {
@@ -136,6 +125,24 @@ func objectStatus(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) uint8 {
 	return inGraveyardWithKey(addrKey, graveyardBkt, garbageBkt)
 }
 
+func isExpiredWithAttribute(tx *bbolt.Tx, attr string, addr oid.Address, currEpoch uint64) bool {
+	// bucket with objects that have expiration attr
+	attrKey := make([]byte, bucketKeySize+len(attr))
+	expirationBucket := tx.Bucket(attributeBucketName(addr.Container(), attr, attrKey))
+	if expirationBucket != nil {
+		// bucket that contains objects that expire in the current epoch
+		prevEpochBkt := expirationBucket.Bucket([]byte(strconv.FormatUint(currEpoch-1, 10)))
+		if prevEpochBkt != nil {
+			rawOID := objectKey(addr.Object(), make([]byte, objectKeySize))
+			if prevEpochBkt.Get(rawOID) != nil {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 func inGraveyardWithKey(addrKey []byte, graveyard, garbageBCK *bbolt.Bucket) uint8 {
 	if graveyard == nil {
 		// incorrect metabase state, does not make

pkg/local_object_storage/metabase/iterators.go

@@ -60,7 +60,10 @@ func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler)
 	err := tx.ForEach(func(name []byte, b *bbolt.Bucket) error {
 		cidBytes := cidFromAttributeBucket(name, objectV2.SysAttributeExpEpoch)
 		if cidBytes == nil {
-			return nil
+			cidBytes = cidFromAttributeBucket(name, objectV2.SysAttributeExpEpochNeoFS)
+			if cidBytes == nil {
+				return nil
+			}
 		}
 
 		var cnrID cid.ID

pkg/services/object_manager/tombstone/checker.go

@@ -73,7 +73,7 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr
 
 func (g *ExpirationChecker) handleTS(addr string, ts *object.Object, reqEpoch uint64) bool {
 	for _, atr := range ts.Attributes() {
-		if atr.Key() == objectV2.SysAttributeExpEpoch {
+		if atr.Key() == objectV2.SysAttributeExpEpoch || atr.Key() == objectV2.SysAttributeExpEpochNeoFS {
 			epoch, err := strconv.ParseUint(atr.Value(), 10, 64)
 			if err != nil {
 				g.log.Warn(