Compare commits

...

2 commits

Author SHA1 Message Date
76fd5c9706 [#180] preset_grpc: Add ability to attach rule for created container
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-11-15 13:55:30 +03:00
f0cbf9c301 [#180] preset_grpc: Remove deprecated parameter acl
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
2024-11-14 15:35:13 +03:00
5 changed files with 28 additions and 21 deletions

View file

@ -57,7 +57,7 @@ const frostfs_cli = native.connect("s01.frostfs.devenv:8080", "", 0, 0, false, 0
### Methods ### Methods
- `putContainer(params)`. The `params` is a dictionary (e.g. - `putContainer(params)`. The `params` is a dictionary (e.g.
`{acl:'public-read-write',placement_policy:'REP 3',name:'container-name',name_global_scope:'false'}`). `{placement_policy:'REP 3',name:'container-name',name_global_scope:'false'}`).
Returns dictionary with `success` Returns dictionary with `success`
boolean flag, `container_id` string, and `error` string. boolean flag, `container_id` string, and `error` string.
- `setBufferSize(size)`. Sets internal buffer size for data upload and - `setBufferSize(size)`. Sets internal buffer size for data upload and

View file

@ -13,7 +13,6 @@ export const options = {
export function setup() { export function setup() {
const params = { const params = {
acl: 'public-read-write',
placement_policy: 'REP 3', placement_policy: 'REP 3',
name: 'container-name', name: 'container-name',
name_global_scope: 'false' name_global_scope: 'false'

View file

@ -13,7 +13,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@ -265,16 +264,6 @@ func (c *Client) PutContainer(params map[string]string) PutContainerResponse {
container.SetCreationTime(&cnr, time.Now()) container.SetCreationTime(&cnr, time.Now())
cnr.SetOwner(usr) cnr.SetOwner(usr)
if basicACLStr, ok := params["acl"]; ok {
var basicACL acl.Basic
err := basicACL.DecodeString(basicACLStr)
if err != nil {
return c.putCnrErrorResponse(err)
}
cnr.SetBasicACL(basicACL)
}
placementPolicyStr, ok := params["placement_policy"] placementPolicyStr, ok := params["placement_policy"]
if ok { if ok {
var placementPolicy netmap.PlacementPolicy var placementPolicy netmap.PlacementPolicy

View file

@ -1,7 +1,7 @@
import re import re
from helpers.cmd import execute_cmd, log from helpers.cmd import execute_cmd, log
def create_container(endpoint, policy, container_creation_retry, wallet_path, config, acl, local=False, retry=0): def create_container(endpoint, policy, container_creation_retry, wallet_path, config, rules, local=False, retry=0):
if retry > int(container_creation_retry): if retry > int(container_creation_retry):
raise ValueError(f"unable to create container: too many unsuccessful attempts") raise ValueError(f"unable to create container: too many unsuccessful attempts")
@ -9,10 +9,8 @@ def create_container(endpoint, policy, container_creation_retry, wallet_path, co
wallet_file = f"--wallet {wallet_path}" wallet_file = f"--wallet {wallet_path}"
if config: if config:
wallet_config = f"--config {config}" wallet_config = f"--config {config}"
if acl:
acl_param = f"--basic-acl {acl}"
cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create {wallet_file} {wallet_config} " \ cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create {wallet_file} {wallet_config} " \
f" --policy '{policy}' {acl_param} --await" f" --policy '{policy}' --await"
output, success = execute_cmd(cmd_line) output, success = execute_cmd(cmd_line)
@ -36,6 +34,20 @@ def create_container(endpoint, policy, container_creation_retry, wallet_path, co
log(f"Created container: {cid} ({policy})", endpoint) log(f"Created container: {cid} ({policy})", endpoint)
# Add rule for container
if rules:
r = ""
for rule in rules:
r += f" --rule '{rule}' "
cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} ape-manager add {wallet_file} {wallet_config} " \
f" --chain-id 'chain-id' {r} --target-name '{cid}' --target-type 'container'"
output, success = execute_cmd(cmd_line)
if not success:
log(f"{cmd_line}\n"
f"Rule has not been added\n"
f"{output}", endpoint)
return False
if not local: if not local:
return cid return cid
@ -88,7 +100,7 @@ def create_container(endpoint, policy, container_creation_retry, wallet_path, co
return cid return cid
log(f"Created container {cid} is not stored on {endpoint}, creating another one...", endpoint) log(f"Created container {cid} is not stored on {endpoint}, creating another one...", endpoint)
return create_container(endpoint, policy, container_creation_retry, wallet_path, config, acl, local, retry + 1) return create_container(endpoint, policy, container_creation_retry, wallet_path, config, rules, local, retry + 1)
def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config): def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_config):

View file

@ -16,6 +16,7 @@ ERROR_WRONG_CONTAINERS_COUNT = 1
ERROR_WRONG_OBJECTS_COUNT = 2 ERROR_WRONG_OBJECTS_COUNT = 2
MAX_WORKERS = 50 MAX_WORKERS = 50
DEFAULT_POLICY = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" DEFAULT_POLICY = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
DEFAULT_RULES = ["allow Object.* *"]
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--size', help='Upload objects size in kb') parser.add_argument('--size', help='Upload objects size in kb')
@ -37,7 +38,10 @@ parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Def
parser.add_argument('--sleep', help='Time to sleep between containers creation and objects upload (in seconds), ' parser.add_argument('--sleep', help='Time to sleep between containers creation and objects upload (in seconds), '
'Default = 8', default=8) 'Default = 8', default=8)
parser.add_argument('--local', help='Create containers that store data on provided endpoints. Warning: additional empty containers may be created.', action='store_true') parser.add_argument('--local', help='Create containers that store data on provided endpoints. Warning: additional empty containers may be created.', action='store_true')
parser.add_argument('--acl', help='Container ACL. Default is public-read-write.', default='public-read-write') parser.add_argument(
'--rule',
help='Rule attached to created containers. All entries of CONTAINER_ID will be replaced with id of created container.',
action="append")
args: Namespace = parser.parse_args() args: Namespace = parser.parse_args()
print(args) print(args)
@ -56,6 +60,9 @@ def main():
wallet_config = args.config wallet_config = args.config
workers = int(args.workers) workers = int(args.workers)
objects_per_container = int(args.preload_obj) objects_per_container = int(args.preload_obj)
rules = args.rule
if not rules:
rules = DEFAULT_RULES
ignore_errors = args.ignore_errors ignore_errors = args.ignore_errors
if args.update: if args.update:
@ -68,7 +75,7 @@ def main():
containers_count = int(args.containers) containers_count = int(args.containers)
print(f"Create containers: {containers_count}") print(f"Create containers: {containers_count}")
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor: with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
containers_runs = [executor.submit(create_container, endpoint, policy, container_creation_retry, wallet, wallet_config, args.acl, args.local) containers_runs = [executor.submit(create_container, endpoint, policy, container_creation_retry, wallet, wallet_config, rules, args.local)
for _, endpoint, policy in for _, endpoint, policy in
zip(range(containers_count), cycle(endpoints), cycle(args.policy))] zip(range(containers_count), cycle(endpoints), cycle(args.policy))]