Vladimir Domnich
1cf53545f2
Registry module stores information about uploaded objects in bolt database and allows to verify their validity after a load test. Also, implemented logic to verify objects uploaded via gRPC and S3 protocols. Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
132 lines
4.3 KiB
Python
Executable file
132 lines
4.3 KiB
Python
Executable file
#!/usr/bin/python3
|
|
|
|
from multiprocessing import Process
|
|
import uuid
|
|
import shlex
|
|
from subprocess import check_output, CalledProcessError, STDOUT
|
|
import json
|
|
import os
|
|
import argparse, sys
|
|
|
|
from concurrent.futures import ProcessPoolExecutor
|
|
|
|
parser=argparse.ArgumentParser()
|
|
|
|
parser.add_argument('--size', help='Upload objects size in kb')
|
|
parser.add_argument('--containers', help='Number of containers to create')
|
|
parser.add_argument('--out', help='JSON file with output')
|
|
parser.add_argument('--preload_obj', help='Number of pre-loaded objects')
|
|
parser.add_argument('--endpoint', help='Node address')
|
|
parser.add_argument('--update', help='Save existed containers')
|
|
|
|
args=parser.parse_args()
|
|
print(args)
|
|
|
|
|
|
def main():
|
|
container_list = []
|
|
objects_struct = []
|
|
payload_filepath='/tmp/data_file'
|
|
|
|
if args.update:
|
|
# Open file
|
|
with open(args.out) as f:
|
|
data_json = json.load(f)
|
|
container_list = data_json['containers']
|
|
# Get CID list
|
|
else:
|
|
print(f"Create containers: {args.containers}")
|
|
with ProcessPoolExecutor(max_workers=10) as executor:
|
|
containers_runs = {executor.submit(create_container): _ for _ in range(int(args.containers))}
|
|
|
|
for run in containers_runs:
|
|
if run.result() is not None:
|
|
container_list.append(run.result())
|
|
|
|
print("Create containers: Completed")
|
|
|
|
print(f" > Containers: {container_list}")
|
|
|
|
print(f"Upload objects to each container: {args.preload_obj} ")
|
|
random_payload(payload_filepath)
|
|
print(" > Create random payload: Completed")
|
|
|
|
for container in container_list:
|
|
print(f" > Upload objects for container {container}")
|
|
with ProcessPoolExecutor(max_workers=50) as executor:
|
|
objects_runs = {executor.submit(upload_object, container, payload_filepath): _ for _ in range(int(args.preload_obj))}
|
|
|
|
for run in objects_runs:
|
|
if run.result() is not None:
|
|
objects_struct.append({'container': container, 'object': run.result()})
|
|
print(f" > Upload objects for container {container}: Completed")
|
|
|
|
|
|
print("Upload objects to each container: Completed")
|
|
|
|
data = { 'containers': container_list, 'objects': objects_struct, 'obj_size': args.size + " Kb" }
|
|
|
|
with open(args.out, 'w') as f:
|
|
json.dump(data, f, ensure_ascii=False)
|
|
|
|
print(f"Result:")
|
|
print(f" > Total Containers has been created: {len(container_list)}.")
|
|
print(f" > Total Objects has been created: {len(objects_struct)}.")
|
|
|
|
|
|
def random_payload(payload_filepath):
|
|
with open('%s'%payload_filepath, 'wb') as fout:
|
|
fout.write(os.urandom(1024*int(args.size)))
|
|
|
|
def execute_cmd(cmd_line):
|
|
args = shlex.split(cmd_line)
|
|
output = ""
|
|
try:
|
|
output = check_output(args, stderr=STDOUT).decode()
|
|
success = True
|
|
|
|
except CalledProcessError as e:
|
|
output = e.output.decode()
|
|
success = False
|
|
|
|
return output, success
|
|
|
|
|
|
def create_container():
|
|
cmd_line = f"neofs-cli --rpc-endpoint {args.endpoint} container create -g --policy 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X' --basic-acl public-read-write --await"
|
|
output, success = execute_cmd(cmd_line)
|
|
|
|
if not success:
|
|
print(f" > Container has not been created.")
|
|
else:
|
|
try:
|
|
fst_str = output.split('\n')[0]
|
|
except Exception:
|
|
print(f"Got empty output: {output}")
|
|
splitted = fst_str.split(": ")
|
|
if len(splitted) != 2:
|
|
raise ValueError(f"no CID was parsed from command output: \t{fst_str}")
|
|
return splitted[1]
|
|
|
|
|
|
def upload_object(container, payload_filepath):
|
|
object_name = ""
|
|
cmd_line = f"neofs-cli --rpc-endpoint {args.endpoint} object put -g --file {payload_filepath} --cid {container} --no-progress"
|
|
out, success = execute_cmd(cmd_line)
|
|
|
|
if not success:
|
|
print(f" > Object {object_name} has not been uploaded.")
|
|
else:
|
|
try:
|
|
# taking second string from command output
|
|
snd_str = out.split('\n')[1]
|
|
except:
|
|
print(f"Got empty input: {out}")
|
|
splitted = snd_str.split(": ")
|
|
if len(splitted) != 2:
|
|
raise Exception(f"no OID was parsed from command output: \t{snd_str}")
|
|
return splitted[1]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|