[#2246] node: Allow to configure tombsone lifetime

Currently, DELETE service sets tombstone expiration epoch to
`current epoch + 5`. This works less than ideal in private networks
where an epoch can be e.g. 10 minutes. In this case, after a node is
unavailable for more than 1 hour, already deleted objects have a chance
to reappear.

After this commit tombstone lifetime can be configured.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

CHANGELOG.md

@@ -9,6 +9,7 @@ Changelog for FrostFS Node
 - New `frostfs_node_object_container_size` metric for tracking size of reqular objects in a container (#2116)
 - New `frostfs_node_object_payload_size` metric for tracking size of reqular objects on a single shard (#1794)
 - Add command `frostfs-adm morph netmap-candidates` (#1889)
+- `object.delete.tombstone_lifetime` config parameter to set tombstone lifetime in the DELETE service (#2246)
 
 ### Changed
 - Change `frostfs_node_engine_container_size` to counting sizes of logical objects
@@ -46,7 +47,13 @@ Changelog for FrostFS Node
 - Minimum go version to v1.18
 
 ### Updating from v0.35.0
+<<<<<<< HEAD
 You need to change configuration environment variables to `FROSTFS_*` if you use any.
+||||||| parent of 00afc576d ([#2246] node: Allow to configure tombsone lifetime)
+=======
+New config field `object.delete.tombstone_lifetime` allows to set tombstone lifetime
+more appropriate for a specific deployment.
+>>>>>>> 00afc576d ([#2246] node: Allow to configure tombsone lifetime)
 
 ## [0.35.0] - 2022-12-28 - Sindo (신도, 信島)
 

cmd/frostfs-node/config.go

@@ -472,6 +472,8 @@ type cfgObject struct {
 	pool cfgObjectRoutines
 
 	cfgLocalStorage cfgLocalStorage
+
+	tombstoneLifetime uint64
 }
 
 type cfgNotifications struct {
@@ -598,7 +600,8 @@ func initCfg(appCfg *config.Config) *cfg {
 		proxyScriptHash: contractsconfig.Proxy(appCfg),
 	}
 	c.cfgObject = cfgObject{
-		pool: initObjectPool(appCfg),
+		pool:              initObjectPool(appCfg),
+		tombstoneLifetime: objectconfig.TombstoneLifetime(appCfg),
 	}
 	c.cfgReputation = cfgReputation{
 		scriptHash: contractsconfig.Reputation(appCfg),

cmd/frostfs-node/config/object/config_test.go

@@ -14,12 +14,14 @@ func TestObjectSection(t *testing.T) {
 		empty := configtest.EmptyConfig()
 
 		require.Equal(t, objectconfig.PutPoolSizeDefault, objectconfig.Put(empty).PoolSizeRemote())
+		require.EqualValues(t, objectconfig.DefaultTombstoneLifetime, objectconfig.TombstoneLifetime(empty))
 	})
 
 	const path = "../../../../config/example/node"
 
 	var fileConfigTest = func(c *config.Config) {
 		require.Equal(t, 100, objectconfig.Put(c).PoolSizeRemote())
+		require.EqualValues(t, 10, objectconfig.TombstoneLifetime(c))
 	}
 
 	configtest.ForEachFileType(path, fileConfigTest)

cmd/frostfs-node/config/object/delete.go

@@ -0,0 +1,19 @@
+package objectconfig
+
+import "github.com/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
+
+const (
+	deleteSubsection = "delete"
+
+	// DefaultTombstoneLifetime is the default value of tombstone lifetime in epochs.
+	DefaultTombstoneLifetime = 5
+)
+
+// TombstoneLifetime returns the value of `tombstone_lifetime` config parameter.
+func TombstoneLifetime(c *config.Config) uint64 {
+	ts := config.UintSafe(c.Sub(subsection).Sub(deleteSubsection), "tombstone_lifetime")
+	if ts <= 0 {
+		return DefaultTombstoneLifetime
+	}
+	return ts
+}

cmd/frostfs-node/object.go

@@ -316,7 +316,7 @@ func initObjectService(c *cfg) {
 		deletesvc.WithPutService(sPut),
 		deletesvc.WithNetworkInfo(&delNetInfo{
 			State:      c.cfgNetmap.state,
-			tsLifetime: 5,
+			tsLifetime: c.cfgObject.tombstoneLifetime,
 
 			cfg: c,
 		}),

config/example/node.env

@@ -85,6 +85,7 @@ FROSTFS_REPLICATOR_POOL_SIZE=10
 
 # Object service section
 FROSTFS_OBJECT_PUT_POOL_SIZE_REMOTE=100
+FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10
 
 # Storage engine section
 FROSTFS_STORAGE_SHARD_POOL_SIZE=15

config/example/node.json

@@ -128,6 +128,9 @@
     "put_timeout": "15s"
   },
   "object": {
+    "delete": {
+      "tombstone_lifetime": 10
+    },
     "put": {
       "pool_size_remote": 100
     }

config/example/node.yaml

@@ -107,6 +107,8 @@ replicator:
   pool_size: 10     # maximum amount of concurrent replications
 
 object:
+  delete:
+    tombstone_lifetime: 10 # tombstone "local" lifetime in epochs
   put:
     pool_size_remote: 100  # number of async workers for remote PUT operations
 

docs/storage-node-configuration.md

@@ -415,7 +415,7 @@ replicator:
 | `pool_size`   | `int`      | Equal to `object.put.pool_size_remote` | Maximum amount of concurrent replications.  |
 
 # `object` section
-Contains pool sizes for object operations with remote nodes.
+Contains object-service related parameters.
 
 ```yaml
 object:
@@ -423,6 +423,7 @@ object:
     pool_size_remote: 100
 ```
 
-| Parameter              | Type  | Default value | Description                                                                                    |
+| Parameter                   | Type  | Default value | Description                                                                                    |
-|------------------------|-------|---------------|------------------------------------------------------------------------------------------------|
+|-----------------------------|-------|---------------|------------------------------------------------------------------------------------------------|
-| `put.pool_size_remote` | `int` | `10`          | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. |
+| `delete.tombstone_lifetime` | `int` | `5`           | Tombstone lifetime for removed objects in epochs.                                              |
+| `put.pool_size_remote`      | `int` | `10`          | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. |