diff --git a/.drone.yml b/.drone.yml index d3c2070e..e254869f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -5,18 +5,21 @@ name: robot-image steps: - name: cleanup and build - image: docker:19.03.5 + image: docker:19.03.11-dind environment: - PASSWORD: - from_secret: docker_passwd IMG_NAME: 'registry.nspcc.ru/docker-local/robot' + REG_USR: 'admin' + REG_PWD: + from_secret: docker_passwd + JF_TOKEN: + from_secret: api_key volumes: - name: docker_sock path: /var/run/docker.sock commands: - ./images_cleanup.sh - docker build --no-cache -t $IMG_NAME:latest ./ - - docker login registry.nspcc.ru -u admin -p "$PASSWORD" + - docker login registry.nspcc.ru -u admin -p "$REG_PWD" - docker push $IMG_NAME:latest trigger: @@ -24,6 +27,7 @@ trigger: - master event: - push + - pull_request # host volume mount requires a repository to be `trusted` # in drone; this option enables in repository settings diff --git a/Dockerfile b/Dockerfile index 4a8c9b7f..117c5483 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,83 @@ -FROM python:3.6 +FROM docker:19.03.11-dind ENV WD / -ENV NEOFSCLI_VERSION '0.8.2' +ARG REG_USR +ARG REG_PWD +ARG JF_TOKEN +ARG BUILD_NEOFS_NODE +ENV REG_USR=${REG_USR} +ENV REG_PWD=${REG_PWD} +ENV NEOFSCLI_VERSION=0.9.0 +ENV JF_TOKEN=${JF_TOKEN} +ENV BUILD_NEOFS_NODE=${BUILD_NEOFS_NODE} -RUN apt-get update \ - && pip3 install robotframework==3.2.1 +ENV RF_VERSION 3.2.1 + +RUN apk add --no-cache openssh +RUN apk add --no-cache libressl-dev +RUN apk add --no-cache curl +RUN apk add --no-cache bash bash-doc bash-completion +RUN apk add --no-cache util-linux pciutils usbutils coreutils binutils findutils grep gcc libffi-dev openssl-dev +RUN apk add --no-cache sudo + +RUN apk --no-cache add \ + make \ + python3 \ + py3-pip + +RUN apk --no-cache add --virtual \ + .build-deps \ + build-base \ + python3-dev + +RUN addgroup nobody root && \ + echo "export PYTHONPATH=\$PYTHONPATH:/.local/lib/python3.8/site-packages" > /.profile && \ + mkdir -p /tests /reports /.local && \ + chgrp -R 0 /reports /.local && \ + chmod -R g=u /etc/passwd /reports /.local /.profile + +RUN pip3 install wheel +RUN pip3 install robotframework +RUN pip3 install neocore +RUN pip3 install requests + + +# Golang +ARG GOLANG_VERSION=1.14.3 +#we need the go version installed from apk to bootstrap the custom version built from source +RUN apk update && apk add go gcc bash musl-dev openssl-dev ca-certificates && update-ca-certificates +RUN wget https://dl.google.com/go/go$GOLANG_VERSION.src.tar.gz && tar -C /usr/local -xzf go$GOLANG_VERSION.src.tar.gz +RUN cd /usr/local/go/src && ./make.bash +ENV PATH=$PATH:/usr/local/go/bin +RUN rm go$GOLANG_VERSION.src.tar.gz +#we delete the apk installed version to avoid conflict +RUN apk del go +RUN go version + + +# Add the keys and set permissions +COPY ./ca/* /root/.ssh/ + +RUN chmod 600 /root/.ssh/id_rsa && \ + chmod 600 /root/.ssh/id_rsa.pub + +RUN pip3 install docker-compose + +RUN export DOCKER_HOST="${HOSTNAME}-docker" +RUN apk add --no-cache git \ + --repository https://alpine.global.ssl.fastly.net/alpine/v3.10/community \ + --repository https://alpine.global.ssl.fastly.net/alpine/v3.10/main + +RUN mkdir -p /robot/vendor + +RUN cd /robot/vendor \ + && git clone git@bitbucket.org:nspcc-dev/neofs-dev-env.git \ + && cd neofs-dev-env \ + && cp ca/* /usr/local/share/ca-certificates/ \ + && update-ca-certificates -RUN cd /tmp \ - && wget https://github.com/nspcc-dev/neofs-cli/releases/download/v${NEOFSCLI_VERSION}/neofs-cli_${NEOFSCLI_VERSION}_linux_x86_64.tar.gz \ - && tar xfz neofs-cli_${NEOFSCLI_VERSION}_linux_x86_64.tar.gz \ - && mv neofs-cli /usr/local/bin \ - && rm -rf /tmp/neofs-cli WORKDIR ${WD} COPY ./ ${WD} + +RUN cd ${WD} && chmod +x dockerd.sh diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..93cce3a5 --- /dev/null +++ b/Makefile @@ -0,0 +1,61 @@ +VERSION=0.0.17 +PREFIX= + +B=\033[0;1m +G=\033[0;92m +R=\033[0m + +.DEFAULT_GOAL := help +.PHONY: build-image + +DATE = $(shell date +%s) +NAME = "testcases_$(DATE)" + +build: + @echo "${B}${G}⇒ Build image ${R}" + @docker build \ + --build-arg REG_USR=$(REG_USR) \ + --build-arg REG_PWD=$(REG_PWD) \ + --build-arg JF_TOKEN=$(JF_TOKEN) \ + --build-arg BUILD_NEOFS_NODE=${BUILD_NEOFS_NODE} \ + --build-arg BUILD_CLI=${BUILD_CLI} \ + -f Dockerfile \ + -t robot:$(VERSION)$(PREFIX) . + +run_docker: + @echo "${B}${G}⇒ Test Run image $(NAME)${R}" + @mkdir artifacts_$(NAME) + @docker run --privileged=true \ + --name $(NAME) \ + --volume artifacts_$(NAME):/artifacts \ + --add-host bastion.localtest.nspcc.ru:192.168.123.10 \ + --add-host bastion.localtest.nspcc.ru:192.168.123.10 \ + --add-host cdn.fs.localtest.nspcc.ru:192.168.123.40 \ + --add-host main_chain.fs.localtest.nspcc.ru:192.168.123.50 \ + --add-host fs.localtest.nspcc.ru:192.168.123.20 \ + --add-host m01.fs.localtest.nspcc.ru:192.168.123.61 \ + --add-host m02.fs.localtest.nspcc.ru:192.168.123.62 \ + --add-host m03.fs.localtest.nspcc.ru:192.168.123.63 \ + --add-host m04.fs.localtest.nspcc.ru:192.168.123.64 \ + --add-host send.fs.localtest.nspcc.ru:192.168.123.30 \ + --add-host s01.fs.localtest.nspcc.ru:192.168.123.71 \ + --add-host s02.fs.localtest.nspcc.ru:192.168.123.72 \ + --add-host s03.fs.localtest.nspcc.ru:192.168.123.73 \ + --add-host s04.fs.localtest.nspcc.ru:192.168.123.74 \ + robot:$(VERSION)$(PREFIX) ./dockerd.sh & + @sleep 10; + @docker wait $(NAME); + @echo "${B}${G}⇒ Testsuite has been completed. ${R}"; + @echo "${B}${G}⇒ Copy Logs from container to ./artifacts/ ${R}"; + @docker cp $(NAME):/artifacts . + @docker rm $(NAME) + +run: + @echo "${B}${G}⇒ Test Run ${R}" + @robot --timestampoutputs --outputdir artifacts/ robot/testsuites/integration/object_suite.robot + +help: + @echo "${B}${G}⇒ build Build image ${R}" + @echo "${B}${G}⇒ run Run testcases ${R}" + @echo "${B}${G}⇒ run_docker Run in docker ${R}" + diff --git a/README.md b/README.md index 4d58ede1..3710c60e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,43 @@ -### Сборка образа с тестами + +### Локальный запуск тесткейсов +1. Устаносить зависимости (только для первого запуска): + - pip3 install robotframework + - pip3 install neocore + - pip3 install requests + +(pip3 заменить на соответсвующий менеджер пакетов python в системе). + +При этом должен быть запущен dev-env с тестируемым окружением. + +2. Выпольнить `make run` + +3. Логи будут доступны в папке artifacts/ после завершения тестов с любым из статусов. + + +### Запуск тесткейсов в докере +1. Задать переменные окружения для работы с dev-env: +``` + export REG_USR= + export REG_PWD= + export JF_TOKEN= +``` + +2. Выполнить `make build` + +3. Выполнить `make run_docker` + +4. Логи будут доступны в папке artifacts/ после завершения тестов с любым из статусов. + +### Запуск тесткейсов в докере с произвольными коммитами + +На данный момент доступны произовльные коммиты для NeoFS Node и NeoFS CLI. +Для этого достаточно задать переменные окружения перед запуском `make build`. +``` +export BUILD_NEOFS_NODE= +export BUILD_CLI= +``` + +## Сборка образа с тестами Чтобы тесты из этого репозитория были доступны к запуску из Drone CI, они должны быть упакованы в docker-имадж. Это делается в рамках CI, @@ -6,9 +45,6 @@ описывается в файлах `Dockerfile` и `.drone.yml` и осуществляется на каждый пуш в master. -В докерфайле указана версия `neofs-cli`, который используется в запусках -тестов. Каждый раз при сборке имаджа `neofs-cli` скачивается заново. - Тестовый образ имеет единственную версию -- `latest`. Ради экономии хранилища на машине-сборщике перед сборкой все ранее собранные образы удаляются. @@ -22,3 +58,5 @@ drone exec --trusted --secret-file=secrets.txt --volume /var/run/docker.sock В результате будет прогнан полный пайплайн, за исключением пуша образа в docker registry. Чтобы запушить образ, нужно указать пароль к реджистри в файле `secrets.txt`. + + diff --git a/ca/id_rsa b/ca/id_rsa new file mode 100644 index 00000000..8542365d --- /dev/null +++ b/ca/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAt28xenSOhtLv5sj2gMgp/SWNcIkTHP/OiHvZe2mgdXXDpyIG +iUVGR35c1bep5sHvDLnL2IeFqrWSuajCn2Sf2FTgb/a7dWrTd10fNDlCJK5ibXm9 +CvXXYmsGc+CVDH4E/44qkcm5kqur70D8N9MfnSwrhFlLlwuCXbweyt+cqzFFQeby +47lChQ5Aeim+r0zPVcjqCG66TF1llCSxMbDNYIVONYPQeTWDLut8azgVIUE4bhlR +B44w1mYS6BjMRcYKFZ672os6DLLc/qsRPquXR54AKuD2gg5sn9VgHplFrr2llPzL +Iwoiky6plIfQVgHn3BMGmePx/cNe51mq09AQwQIDAQABAoIBAQCCfJrZ3Wg2CH+X +0IVp/vm/lqMS1q++BUrKVC/VVsJKTEet8Mptg9YGraEkds5p1LNUfibAFUfEs/14 +DNDFyjLbFSXC/+VCFYfwdVHpOIIQzew+rEcKMO/Slwe0DqJ4jHzJvjwSEUntSCm6 +vKOuooTurakXMN5QyGMogtX0wzUToXajl8VPjGy1B2xlIM8KKmvUzsMMrIv4W157 +xV/9OnJ0vPn+3gh90F5Daxk069xNMDefpSbfx5DcZemhnZLLWddX+QmnsSOl5tfQ +8zRx2fkjmjqDCJJ2e4KqV1+7ePhJqARLPnHYxmxbgsfaRGU/RizZFI90Ou3Fma8E +9cc/mfABAoGBAPMtAachdkkuxMB4btTAsW5PgKod0JQhMcXajZJqsl35WczVHHPP +zPXm03Ggx+Yk4BLFBoQT7xFfQjQOo4uUQ30hgS5+lkpGRtJa5p2kLVoDlyuGi0ig +0V+1u/h7wUNfuxncFcFmT8CCjhLrv9WMH2aNGdeRMIHVuxbVIqCftqThAoGBAMEb +pcppZ4Wj7poN033Tq4i9vXoJ2p1d5AwEKWKiYxoHvGRthD/NAjekVrRQrmzVQd1f +/gIOtnfO39WVN7ocH6wRWTi/KBMIrR0oPhtor4MPsB+19JvDmNtq48b94lVz+Z4L +hOSqNdGzlsuy1vZagRzHkxD7VU7PlT+UBdDHpwfhAoGBAPHjZO+Ao46sTN4/bc+H +VXcq8gtF2QJf+oiam5R3ObGspRzRJ5ozq+c2kkFG81EEgTdqcM7UnUuke9AYd6oR +8wf3We6L0KdVPIFmFlvcwZf2VlrfXJEEFwCjX7UONPH1ucFBYQqd4NrXgsdjZdDf +ryRtWrVJIP0lQxK1M9qexClBAoGAI3SxFx4NTONRjuWU/Fhd+WhlHsAqbJRtp6sn +8h1AtunOtF3LV2+Lxa2d4dOigwcQ5dWXLMeIxyyrumqAZeJ+CjjROfMXJ4+DQYQ/ +CwdImnbJ7riY5fSe30Kb+dBpuyjlHxicWOPLp+oieNooT+lEJYWbQhXzjtncXGUQ +QEo4J+ECgYA1UiQr+C3NbMr27HOynXInv09f86HvHsHGooXuGoJ7GB518ZjtoiA7 +xefoSUR10NLmx8QhdGydaFH+wIdgAAwB7RjpsrA5XGnTEKj5yV00o272jXj2jygu +U5700lxwQFHbd5mNZXoywBP0kU/5I98A5OHGhpYthp8lGC+Vk1NZFA== +-----END RSA PRIVATE KEY----- diff --git a/ca/id_rsa.pub b/ca/id_rsa.pub new file mode 100644 index 00000000..c83e20f4 --- /dev/null +++ b/ca/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3bzF6dI6G0u/myPaAyCn9JY1wiRMc/86Ie9l7aaB1dcOnIgaJRUZHflzVt6nmwe8MucvYh4WqtZK5qMKfZJ/YVOBv9rt1atN3XR80OUIkrmJteb0K9ddiawZz4JUMfgT/jiqRybmSq6vvQPw30x+dLCuEWUuXC4JdvB7K35yrMUVB5vLjuUKFDkB6Kb6vTM9VyOoIbrpMXWWUJLExsM1ghU41g9B5NYMu63xrOBUhQThuGVEHjjDWZhLoGMxFxgoVnrvaizoMstz+qxE+q5dHngAq4PaCDmyf1WAemUWuvaWU/MsjCiKTLqmUh9BWAefcEwaZ4/H9w17nWarT0BDB neospcc@neospcc diff --git a/ca/known_hosts b/ca/known_hosts new file mode 100644 index 00000000..aa95d5b2 --- /dev/null +++ b/ca/known_hosts @@ -0,0 +1,2 @@ +|1|UPdjsdJ58lY6EqUaP7n833Fj7Ic=|hQkKlD6Ui6Ne03QrjHpsL7NU0qA= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== +|1|C5OFm8ugKVhcy7PRLErwNGAV/jw=|qMuNNLMrE8rDdleXMmX8N7YBEkM= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== \ No newline at end of file diff --git a/ca/nspcc-ca.pem b/ca/nspcc-ca.pem new file mode 100644 index 00000000..eb940cb9 --- /dev/null +++ b/ca/nspcc-ca.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDhDCCAmygAwIBAgIBATANBgkqhkiG9w0BAQsFADB2MQswCQYDVQQGEwJSVTEM +MAoGA1UECBMDU1BCMQwwCgYDVQQHEwNTUEIxDjAMBgNVBAoTBU5TUENDMQ4wDAYD +VQQLEwVOU1BDQzEOMAwGA1UEAxMFTlNQQ0MxGzAZBgkqhkiG9w0BCQEWDG9wc0Bu +c3BjYy5ydTAeFw0xOTEyMzAxNjQ0MDBaFw0yMDEyMzAxNjQ0MDBaMHYxCzAJBgNV +BAYTAlJVMQwwCgYDVQQIEwNTUEIxDDAKBgNVBAcTA1NQQjEOMAwGA1UEChMFTlNQ +Q0MxDjAMBgNVBAsTBU5TUENDMQ4wDAYDVQQDEwVOU1BDQzEbMBkGCSqGSIb3DQEJ +ARYMb3BzQG5zcGNjLnJ1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +q14BlfD4ZcNJz3HZGLj4ksqbi2tzogpnIlC2ccVcd8sk7mrkAjUFI1VqdJg57F7Y +kv8qYsyYwTaUQB5NAmSOQxNFKKsRc4LI3J8b7dE2q5+dDCWRN5tz8sIRfXryXyF2 +VpW1RnuxMGJ6SwgTn9Wb+R6o1WkwNtrDhhNkdWsBnrJzL37GIk6nhQJxm3OlrHgQ +IPz0bt2QlSYQwuTTd1V6vMgOpHfqkQiAJZLBvQ39SImAO+pNS1+CrJuh+zWqhKzq +0vagW0EA4mq7Z98WqGF/g9d2yj8WRpxoAwpJzUoExPstHyDYSvKhr6R8+QV0zhpF +KdjqggmAM9ixSonYC3txawIDAQABox0wGzAMBgNVHRMEBTADAQH/MAsGA1UdDwQE +AwICjDANBgkqhkiG9w0BAQsFAAOCAQEAQ3gBirQTNU9rWIT9Bwh4Hj1ft5jIepQ7 +yXmqFiOFKUli850t1yCre5/oGyOhQy8huUDL+qQDFeX63g5m3r1B3MsBfxKsukn6 +I2HqwNVEUrsnRtWDr/NXpwpRGfjg1zG9tUxg6Tbo5VD4HG7dUGBuyOXA+/tBYvUQ +3vr9bGFUpuYGJ6/KmnoEbR6B1cB4MwW+xorJFODCaEcd2aAuAABL7o84nx3UFTsb +CguS26u5dpA+32rwV9US17sJfJ301DirapG3CxlhxhUPMXqxuQH1Am8x4zaMz3dN +fRvPLar6g33xKcd6T/VAAp7kMpxyzWoenOYXxGfpgs7Rj/X6xBX3sw== +-----END CERTIFICATE----- diff --git a/dockerd.sh b/dockerd.sh new file mode 100644 index 00000000..8d27c623 --- /dev/null +++ b/dockerd.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +dockerd & +sleep 60 +export DOCKER_HOST=unix:///var/run/docker.sock +docker login registry.nspcc.ru -u ${REG_USR} -p ${REG_PWD} +make rebuild -C /robot/vendor/neofs-dev-env +make up -C /robot/vendor/neofs-dev-env +sleep 60 +robot --timestampoutputs --outputdir /artifacts/ /robot/testsuites/integration/object_suite.robot diff --git a/robot/resources/lib/environment.py b/robot/resources/lib/environment.py new file mode 100644 index 00000000..40c6c37b --- /dev/null +++ b/robot/resources/lib/environment.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 + +import subprocess +import re +import os +import shutil +import json +import binascii +import time + + +from robot.api.deco import keyword +from robot.api import logger + +import robot.errors +import requests +import uuid +from robot.libraries.BuiltIn import BuiltIn + +ROBOT_AUTO_KEYWORDS = False + +@keyword('Prepare Environment') +def prepare_environment(): + return + + +@keyword('Cleanup Environment') +def cleanup_environment(): + return + diff --git a/robot/resources/lib/kws_module.py b/robot/resources/lib/kws_module.py deleted file mode 100644 index 0186940b..00000000 --- a/robot/resources/lib/kws_module.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/python3 - -import subprocess -import re - -from robot.api.deco import keyword -from robot.api import logger - -ROBOT_AUTO_KEYWORDS = False - -NEOFS_ENDPOINT = '10.78.30.13:8080' -#NEOFS_ENDPOINT = '85.143.219.93:8080' -NEOFS_KEY = 'L3UcodxBNukNuXnMKzH7rUn3pvgLGrNkGqeUnvnPySxBFHVR8xmL' -#NEOFS_KEY = 'KxDgvEKzgSBPPfuVfw67oPQBSjidEiqTHURKSDL1R7yGaGYAeYnr' - -@keyword('Create container') -def create_container(): - logger.info("Creating container") - - createContainerCmd = f'neofs-cli --host {NEOFS_ENDPOINT} --key {NEOFS_KEY} container put --rule "RF 1 SELECT 2 Node"' - complProc = subprocess.run(createContainerCmd, check=True, universal_newlines=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) - cid = parse_cid(complProc.stdout) - return cid - -def parse_cid(output: str): - """ - This function parses CID from given CLI output. - Parameters: - - output: a string with command run output - """ - m = re.search(r'Success! Container <(([a-zA-Z0-9])+)> created', output) - if m.start() != m.end(): # e.g., if match found something - cid = m.group(1) - else: - logger.warn("no CID was parsed from command output: \t%s" % output) - return - return cid - -@keyword('Write object to NeoFS') -def write_object(path: str, cid: str): - logger.info("Going to put an object") - - putObjectCmd = f'neofs-cli --host {NEOFS_ENDPOINT} --key {NEOFS_KEY} object put --file {path} --cid {cid}' - complProc = subprocess.run(putObjectCmd, check=True, universal_newlines=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) - oid = parse_oid(complProc.stdout) - return oid - -def parse_oid(output: str): - """ - This function parses OID from given CLI output. - Parameters: - - output: a string with command run output - """ - m = re.search(r'ID: ([a-zA-Z0-9-]+)', output) - if m.start() != m.end(): # e.g., if match found something - oid = m.group(1) - else: - logger.warn("no OID was parsed from command output: \t%s" % output) - return - return oid - - -@keyword('Read object from NeoFS') -def read_object(cid: str, oid: str, read_object: str): - logger.info("Going to get an object") - - getObjectCmd = f'neofs-cli --host {NEOFS_ENDPOINT} --key {NEOFS_KEY} object get --cid {cid} --oid {oid} --file {read_object}' - complProc = subprocess.run(getObjectCmd, check=True, universal_newlines=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) - logger.info(complProc.stdout) diff --git a/robot/resources/lib/neo.py b/robot/resources/lib/neo.py new file mode 100644 index 00000000..29a863c5 --- /dev/null +++ b/robot/resources/lib/neo.py @@ -0,0 +1,104 @@ +#!/usr/bin/python3 + +import subprocess +import re +import json +import binascii + +from robot.api.deco import keyword +from robot.api import logger + +import robot.errors +import requests + +from robot.libraries.BuiltIn import BuiltIn +from neocore.KeyPair import KeyPair + +from Crypto import Random + +ROBOT_AUTO_KEYWORDS = False +NEOFS_NEO_API_ENDPOINT = "https://fs.localtest.nspcc.ru/neo_rpc/" + +@keyword('Generate Neo private key') +def generate_neo_private_key(): + """ + This function generates new random Neo private key. + Parameters: None + :rtype: 'bytes' object + """ + private_key = Random.get_random_bytes(32) + logger.info("Generated private key: %s" % binascii.hexlify(private_key)) + + return private_key + + +@keyword('Get Neo public key') +def get_neo_public_key(private_key: bytes): + """ + This function return neo public key. + Parameters: + :param private_key: neo private key + :rtype: string + """ + keypair_gen = KeyPair(bytes(private_key)) + pubkey = keypair_gen.PublicKey.encode_point(True).decode("utf-8") + logger.info("Generated public key: %s" % pubkey) + return pubkey + +@keyword('Get Neo address') +def get_neo_address(private_key: bytes): + """ + This function return neo address. + Parameters: + :param private_key: neo private key + :rtype: string + """ + keypair_gen = KeyPair(private_key) + address = keypair_gen.GetAddress() + logger.info("Generated Neo address: %s" % address) + return address + +@keyword('Transaction accepted in block') +def transaction_accepted_in_block(tx_id: str): + """ + This function return True in case of accepted TX. + Parameters: + :param tx_id: transaction is + :rtype: block number or Exception + """ + + logger.info("Transaction id: %s" % tx_id) + m = re.match(r"^\"0x+([\w.]+)", tx_id) + if m is None: + BuiltIn().fatal_error('Can not parse transaction id: "%s"' % tx_id) + + TX_request = 'curl -X POST '+NEOFS_NEO_API_ENDPOINT+' --cacert ca/nspcc-ca.pem -H \'Content-Type: application/json\' -d \'{ "jsonrpc": "2.0", "id": 5, "method": "gettransactionheight", "params": [\"'+m.group(1)+'\"] }\'' + complProc = subprocess.run(TX_request, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info(complProc.stdout) + response = json.loads(complProc.stdout) + + if (response['result'] == 0): + raise Exception( "Transaction is not found in the blocks." ) + + logger.info("Transaction has been found in the block %s." % response['result'] ) + return response['result'] + + +@keyword('Get Transaction') +def get_transaction(tx_id: str): + """ + This function return information about TX. + Parameters: + :param tx_id: transaction id + """ + + m = re.match(r"^\"0x+([\w.]+)", tx_id) + if m is None: + BuiltIn().fatal_error('Can not parse transaction id: "%s"' % tx_id) + + TX_request = 'curl -X POST '+NEOFS_NEO_API_ENDPOINT+' --cacert ca/nspcc-ca.pem -H \'Content-Type: application/json\' -d \'{ "jsonrpc": "2.0", "id": 5, "method": "getapplicationlog", "params": [\"'+m.group(1)+'\"] }\'' + complProc = subprocess.run(TX_request, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info(complProc.stdout) + \ No newline at end of file diff --git a/robot/resources/lib/neofs.py b/robot/resources/lib/neofs.py new file mode 100644 index 00000000..859fa41d --- /dev/null +++ b/robot/resources/lib/neofs.py @@ -0,0 +1,442 @@ +#!/usr/bin/python3 + +import subprocess +import os +import re +import binascii +import uuid +import hashlib +from robot.api.deco import keyword +from robot.api import logger + + +ROBOT_AUTO_KEYWORDS = False + +NEOFS_ENDPOINT = "192.168.123.71:8080" +CLI_PREFIX = "docker exec neofs-cli " + + +@keyword('Validate storage policy for object') +def validate_storage_policy_for_object(private_key: bytes, expected_copies, cid, oid): + storage_nodes = _get_storage_nodes(private_key) + copies = 0 + for node in storage_nodes: + if re.search(r'(%s: %s)' % (cid, oid), _search_object(node, private_key, cid, oid)): + copies += 1 + if copies < expected_copies: + raise Exception("Not enough object copies to match storage policyю Found: %s, expexted: %s." % (copies, expected_copies)) + + +@keyword('Create container') +def create_container(private_key: bytes): + rule = "RF 2 SELECT 2 Node" + createContainerCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} container put --rule "{rule}"' + + complProc = subprocess.run(createContainerCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=150, shell=True) + output = complProc.stdout + logger.info("Output: %s" % output) + cid = _parse_cid(output) + logger.info("Created container %s with rule '%s'" % (cid, rule)) + + return cid + + +@keyword('Container Existing') +def container_existing(private_key: bytes, cid: str): + Cmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} container list' + logger.info("CMD: %s" % Cmd) + complProc = subprocess.run(Cmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info("Output: %s" % complProc.stdout) + + _find_cid(complProc.stdout, cid) + return + + +@keyword('Generate file of bytes') +def generate_file_of_bytes(size): + """ + generate big binary file with the specified size in bytes + :param size: the size in bytes, can be declared as 6e+6 for example + :return:string filename + """ + + size = int(float(size)) + + filename = str(uuid.uuid4()) + with open('%s'%filename, 'wb') as fout: + fout.write(os.urandom(size)) + + logger.info("Random binary file with size %s bytes has been generated." % str(size)) + return filename + + +@keyword('Search object') +def search_object(private_key: bytes, cid: str, keys: str, *expected_objects_list, **kwargs ): + + logger.info(expected_objects_list) + logger.info(kwargs) + option = "" + + if kwargs: + for key, value in dict(kwargs).items(): + option = f'{option} {key} {value}' + + ObjectCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} object search {keys} --cid {cid} {option}' + logger.info("Cmd: %s" % ObjectCmd) + try: + complProc = subprocess.run(ObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + + logger.info("Output: %s" % complProc.stdout) + + if expected_objects_list is not None: + found_objects = re.findall(r'%s: ([\-\w]+)' % cid, complProc.stdout) + + if sorted(found_objects) == sorted(expected_objects_list): + logger.info("Found objects list '{}' is equal for expected list '{}'".format(found_objects, expected_objects_list)) + else: + raise Exception("Found object list '{}' is not equal to expected list '{}'".format(found_objects, expected_objects_list)) + + + + except subprocess.CalledProcessError as e: + raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + + +@keyword('Verify Head Tombstone') +def verify_head_tombstone(private_key: bytes, cid: str, oid: str): + + ObjectCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} object head --cid {cid} --oid {oid} --full-headers' + logger.info("Cmd: %s" % ObjectCmd) + try: + complProc = subprocess.run(ObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info("Output: %s" % complProc.stdout) + + if re.search(r'Type=Tombstone\s+Value=MARKED', complProc.stdout): + logger.info("Tombstone header 'Type=Tombstone Value=MARKED' was parsed from command output") + else: + raise Exception("Tombstone header 'Type=Tombstone Value=MARKED' was not found in the command output: \t%s" % (complProc.stdout)) + + except subprocess.CalledProcessError as e: + raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + + + + +def _exec_cli_cmd(private_key: bytes, postfix: str): + + # Get linked objects from first + ObjectCmd = f'{CLI_PREFIX}neofs-cli --raw --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} {postfix}' + logger.info("Cmd: %s" % ObjectCmd) + try: + complProc = subprocess.run(ObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info("Output: %s" % complProc.stdout) + + except subprocess.CalledProcessError as e: + raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + + return complProc.stdout + + +@keyword('Verify linked objects') +def verify_linked_objects(private_key: bytes, cid: str, oid: str, payload_size: float): + + payload_size = int(float(payload_size)) + + # Get linked objects from first + postfix = f'object head --cid {cid} --oid {oid} --full-headers' + output = _exec_cli_cmd(private_key, postfix) + child_obj_list = [] + + for m in re.finditer(r'Type=Child ID=([\w-]+)', output): + child_obj_list.append(m.group(1)) + + if not re.search(r'PayloadLength=0', output): + raise Exception("Payload is not equal to zero in the parent object %s." % obj) + + if not child_obj_list: + raise Exception("Child objects was not found.") + else: + logger.info("Child objects: %s" % child_obj_list) + + # HEAD and validate each child object: + payload = 0 + parent_id = "00000000-0000-0000-0000-000000000000" + first_obj = None + child_obj_list_headers = {} + + for obj in child_obj_list: + postfix = f'object head --cid {cid} --oid {obj} --full-headers' + output = _exec_cli_cmd(private_key, postfix) + child_obj_list_headers[obj] = output + if re.search(r'Type=Previous ID=00000000-0000-0000-0000-000000000000', output): + first_obj = obj + logger.info("First child object %s has been found" % first_obj) + + if not first_obj: + raise Exception("Can not find first object with zero Parent ID.") + else: + + _check_linked_object(first_obj, child_obj_list_headers, payload_size, payload, parent_id) + + return child_obj_list_headers.keys() + +def _check_linked_object(obj:str, child_obj_list_headers:dict, payload_size:int, payload:int, parent_id:str): + + output = child_obj_list_headers[obj] + logger.info("Verify headers of the child object %s" % obj) + + if not re.search(r'Type=Previous ID=%s' % parent_id, output): + raise Exception("Incorrect previos ID %s in the child object %s." % parent_id, obj) + else: + logger.info("Previous ID is equal for expected: %s" % parent_id) + + m = re.search(r'PayloadLength=(\d+)', output) + if m.start() != m.end(): + payload += int(m.group(1)) + else: + raise Exception("Can not get payload for the object %s." % obj) + + if payload > payload_size: + raise Exception("Payload exceeds expected total payload %s." % payload_size) + + elif payload == payload_size: + if not re.search(r'Type=Next ID=00000000-0000-0000-0000-000000000000', output): + raise Exception("Incorrect previos ID in the last child object %s." % obj) + else: + logger.info("Next ID is correct for the final child object: %s" % obj) + + else: + m = re.search(r'Type=Next ID=([\w-]+)', output) + if m: + # next object should be in the expected list + logger.info(m.group(1)) + if m.group(1) not in child_obj_list_headers.keys(): + raise Exception(f'Next object {m.group(1)} is not in the expected list: {child_obj_list_headers.keys()}.') + else: + logger.info(f'Next object {m.group(1)} is in the expected list: {child_obj_list_headers.keys()}.') + + _check_linked_object(m.group(1), child_obj_list_headers, payload_size, payload, obj) + + else: + raise Exception("Can not get Next object ID for the object %s." % obj) + + +@keyword('Head object') +def head_object(private_key: bytes, cid: str, oid: str, full_headers:bool=False, **user_headers_dict): + options = "" + if full_headers: + options = "--full-headers" + + ObjectCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} object head --cid {cid} --oid {oid} {options}' + logger.info("Cmd: %s" % ObjectCmd) + try: + complProc = subprocess.run(ObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info("Output: %s" % complProc.stdout) + + for key in user_headers_dict: + user_header = f'Key={key} Val={user_headers_dict[key]}' + if re.search(r'(%s)' % user_header, complProc.stdout): + logger.info("User header %s was parsed from command output" % user_header) + else: + raise Exception("User header %s was not found in the command output: \t%s" % (user_header, complProc.stdout)) + + except subprocess.CalledProcessError as e: + raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + + +@keyword('Delete object') +def delete_object(private_key: bytes, cid: str, oid: str): + ObjectCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} object delete --cid {cid} --oid {oid}' + try: + complProc = subprocess.run(ObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info("Output: %s" % complProc.stdout) + except subprocess.CalledProcessError as e: + raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + + +@keyword('Get file hash') +def get_file_hash(filename): + file_hash = _get_file_hash(filename) + return file_hash + + +@keyword('Verify file hash') +def verify_file_hash(filename, expected_hash): + file_hash = _get_file_hash(filename) + if file_hash == expected_hash: + logger.info("Hash is equal to expected: %s" % file_hash) + else: + raise Exception("File hash '{}' is not equal to {}".format(file_hash, expected_hash)) + + +@keyword('Create storage group') +def create_storage_group(private_key: bytes, cid: str, *objects_list): + objects = "" + + for oid in objects_list: + objects = f'{objects} --oid {oid}' + + ObjectCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} sg put --cid {cid} {objects}' + complProc = subprocess.run(ObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info("Output: %s" % complProc.stdout) + sgid = _parse_oid(complProc.stdout) + return sgid + + +@keyword('Get storage group') +def get_storage_group(private_key: bytes, cid: str, sgid: str): + ObjectCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} sg get --cid {cid} --sgid {sgid}' + logger.info("Cmd: %s" % ObjectCmd) + try: + complProc = subprocess.run(ObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info("Output: %s" % complProc.stdout) + except subprocess.CalledProcessError as e: + raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + + +@keyword('Cleanup File') +# remove temp files +def cleanup_file(filename: str): + if os.path.isfile(filename): + try: + os.remove(filename) + except OSError as e: + raise Exception("Error: '%s' - %s." % (e.filename, e.strerror)) + else: + raise Exception("Error: '%s' file not found" % filename) + + logger.info("File '%s' has been deleted." % filename) + + +@keyword('Put object to NeoFS') +def put_object(private_key: bytes, path: str, cid: str, **kwargs): + logger.info("Going to put the object") + user_headers = "" + user_headers_dict = kwargs + if kwargs: + logger.info(kwargs) + for key, value in dict(kwargs).items(): + user_headers = f'{user_headers} --user "{key}"="{value}"' + + + # Put object to cli container + putObjectCont = f'docker cp {path} neofs-cli:/ ' + logger.info("Cmd: %s" % putObjectCont) + complProc = subprocess.run(putObjectCont, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, shell=True) + + + putObjectCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} object put --verify --file {path} --cid {cid} {user_headers}' + logger.info("Cmd: %s" % putObjectCmd) + complProc = subprocess.run(putObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, shell=True) + logger.info("Output: %s" % complProc.stdout) + oid = _parse_oid(complProc.stdout) + return oid + + +@keyword('Get object from NeoFS') +def get_object(private_key: bytes, cid: str, oid: str, read_object: str): + ObjectCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} object get --cid {cid} --oid {oid} --file {read_object}' + + logger.info("Cmd: %s" % ObjectCmd) + try: + complProc = subprocess.run(ObjectCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, shell=True) + logger.info("Output: %s" % complProc.stdout) + except subprocess.CalledProcessError as e: + raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + + # Get object from cli container + getObjectCont = f'docker cp neofs-cli:/{read_object} . ' + logger.info("Cmd: %s" % getObjectCont) + complProc = subprocess.run(getObjectCont, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, shell=True) + + +def _get_file_hash(filename): + blocksize = 65536 + hash = hashlib.md5() + with open(filename, "rb") as f: + for block in iter(lambda: f.read(blocksize), b""): + hash.update(block) + logger.info("Hash: %s" % hash.hexdigest()) + + return hash.hexdigest() + +def _find_cid(output: str, cid: str): + """ + This function parses CID from given CLI output. + Parameters: + - output: a string with command run output + """ + + if re.search(r'(%s)' % cid, output): + logger.info("CID %s was parsed from command output: \t%s" % (cid, output)) + else: + raise Exception("no CID %s was parsed from command output: \t%s" % (cid, output)) + return cid + +def _parse_oid(output: str): + """ + This function parses OID from given CLI output. + Parameters: + - output: a string with command run output + """ + m = re.search(r'ID: ([a-zA-Z0-9-]+)', output) + if m.start() != m.end(): # e.g., if match found something + oid = m.group(1) + else: + raise Exception("no OID was parsed from command output: \t%s" % output) + + return oid + +def _parse_cid(output: str): + """ + This function parses CID from given CLI output. + Parameters: + - output: a string with command run output + """ + m = re.search(r'Success! Container <(([a-zA-Z0-9])+)> created', output) + if m.start() != m.end(): # e.g., if match found something + cid = m.group(1) + else: + raise Exception("no CID was parsed from command output: \t%s" % (output)) + + return cid + +def _get_storage_nodes(private_key: bytes): + storage_nodes = [] + NetmapCmd = f'{CLI_PREFIX}neofs-cli --host {NEOFS_ENDPOINT} --key {binascii.hexlify(private_key).decode()} status netmap' + complProc = subprocess.run(NetmapCmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + output = complProc.stdout + logger.info("Netmap: %s" % output) + for m in re.finditer(r'"address":"/ip4/(\d+\.\d+\.\d+\.\d+)/tcp/(\d+)"', output): + storage_nodes.append(m.group(1)+":"+m.group(2)) + + if not storage_nodes: + raise Exception("Storage nodes was not found.") + + logger.info("Storage nodes: %s" % storage_nodes) + return storage_nodes + + +def _search_object(node:str, private_key: bytes, cid:str, oid: str): + Cmd = f'{CLI_PREFIX}neofs-cli --host {node} --key {binascii.hexlify(private_key).decode()} object search --root --cid {cid} ID {oid} --ttl 1' + complProc = subprocess.run(Cmd, check=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=15, shell=True) + logger.info(Cmd) + logger.info("Output:") + logger.info(complProc.stdout) + + return complProc.stdout \ No newline at end of file diff --git a/robot/resources/lib/payment.py b/robot/resources/lib/payment.py new file mode 100644 index 00000000..4fd7ff31 --- /dev/null +++ b/robot/resources/lib/payment.py @@ -0,0 +1,83 @@ +#!/usr/bin/python3 + +import subprocess +import re + +from robot.api.deco import keyword +from robot.api import logger + +import logging +import robot.errors +import requests + +from robot.libraries.BuiltIn import BuiltIn +from neocore.KeyPair import KeyPair + +from Crypto import Random + +ROBOT_AUTO_KEYWORDS = False + +@keyword('Request NeoFS Deposit') +def request_neofs_deposit(public_key: str): + """ + This function requests Deposit to the selected public key. + :param public_key: neo public key + """ + + response = requests.get('https://fs.localtest.nspcc.ru/api/deposit/'+str(public_key), verify='ca/nspcc-ca.pem') + + if response.status_code != 200: + BuiltIn().fatal_error('Can not run Deposit to {} with error: {}'.format(public_key, response.text)) + else: + logger.info("Deposit has been completed for '%s'; tx: '%s'" % (public_key, response.text) ) + + return response.text + +@keyword('Get Balance') +def get_balance(public_key: str): + """ + This function returns NeoFS balance for selected public key. + :param public_key: neo public key + """ + + balance = _get_balance_request(public_key) + + return balance + +@keyword('Expected Balance') +def expected_balance(public_key: str, init_amount: float, deposit_size: float): + """ + This function returns NeoFS balance for selected public key. + :param public_key: neo public key + :param init_amount: initial number of tokens in the account + :param deposit_size: expected amount of the balance increasing + """ + + balance = _get_balance_request(public_key) + + deposit_change = round((float(balance) - init_amount),8) + if deposit_change != deposit_size: + raise Exception('Expected deposit increase: {}. This does not correspond to the actual change in account: {}'.format(deposit_size, deposit_change)) + + logger.info('Expected deposit increase: {}. This correspond to the actual change in account: {}'.format(deposit_size, deposit_change)) + + return deposit_change + + +def _get_balance_request(public_key: str): + ''' + Internal method. + ''' + response = requests.get('https://fs.localtest.nspcc.ru/api/balance/neofs/'+str(public_key)+'/', verify='ca/nspcc-ca.pem') + + if response.status_code != 200: + raise Exception('Can not get balance for {} with error: {}'.format(public_key, response.text)) + + m = re.match(r"\"+([\d.\.?\d*]+)", response.text ) + if m is None: + BuiltIn().fatal_error('Can not parse balance: "%s"' % response.text) + balance = m.group(1) + + logger.info("Balance for '%s' is '%s'" % (public_key, balance) ) + + return balance \ No newline at end of file diff --git a/robot/testsuites/integration/object_suite.robot b/robot/testsuites/integration/object_suite.robot new file mode 100644 index 00000000..c7a3bc9b --- /dev/null +++ b/robot/testsuites/integration/object_suite.robot @@ -0,0 +1,184 @@ +*** Settings *** +Variables ../../variables/common.py + +Library ${RESOURCES}/environment.py +Library ${RESOURCES}/neo.py +Library ${RESOURCES}/neofs.py +Library ${RESOURCES}/payment.py +Library ${RESOURCES}/assertions.py +Library ${RESOURCES}/neo.py + +*** Variables *** +&{FILE_USR_HEADER} = key1=1 key2='abc' + + +*** Test cases *** +NeoFS Simple Object Operations + [Documentation] Testcase to validate NeoFS operations with simple object. + [Tags] Object NeoFS NeoCLI + [Timeout] 20 min + + ${PRIV_KEY} = Generate Neo private key + ${PUB_KEY} = Get Neo public key ${PRIV_KEY} + ${ADDR} = Get Neo address ${PRIV_KEY} + ${TX} = Request NeoFS Deposit ${PUB_KEY} + Wait Until Keyword Succeeds 1 min 15 sec + ... Transaction accepted in block ${TX} + Get Transaction ${TX} + ${BALANCE} = Wait Until Keyword Succeeds 10 min 1 min + ... Get Balance ${PUB_KEY} + Expected Balance ${PUB_KEY} 0 50 + ${CID} = Create container ${PRIV_KEY} + Container Existing ${PRIV_KEY} ${CID} + Wait Until Keyword Succeeds 2 min 30 sec + ... Expected Balance ${PUB_KEY} ${BALANCE} -0.00001424 + ${FILE} = Generate file of bytes 1024 + ${FILE_HASH} = Get file hash ${FILE} + ${S_OID} = Put object to NeoFS ${PRIV_KEY} ${FILE} ${CID} + ${H_OID} = Put object to NeoFS ${PRIV_KEY} ${FILE} ${CID} &{FILE_USR_HEADER} +# Next keyword has been removed due to https://neospcc.atlassian.net/browse/NSPCC-1103 +# Validate storage policy for object ${PRIV_KEY} 2 ${CID} ${S_OID} + ${SGID} = Create storage group ${PRIV_KEY} ${CID} ${S_OID} ${H_OID} + + @{S_OBJ_SG} = Create List ${SGID} + @{S_OBJ_ALL} = Create List ${S_OID} ${H_OID} ${SGID} + @{S_OBJ_H} = Create List ${H_OID} + + Search object ${PRIV_KEY} ${CID} --sg @{S_OBJ_SG} + Get storage group ${PRIV_KEY} ${CID} ${SGID} + Get object from NeoFS ${PRIV_KEY} ${CID} ${S_OID} s_file_read + Get object from NeoFS ${PRIV_KEY} ${CID} ${S_OID} h_file_read + Search object ${PRIV_KEY} ${CID} ${EMPTY} @{S_OBJ_ALL} + Search object ${PRIV_KEY} ${CID} ${EMPTY} @{S_OBJ_H} &{FILE_USR_HEADER} + Head object ${PRIV_KEY} ${CID} ${S_OID} ${True} + Head object ${PRIV_KEY} ${CID} ${H_OID} ${True} &{FILE_USR_HEADER} + + Run Keyword And Expect Error REGEXP:User header (\\w+=\\w+\\s?)+ was not found + ... Head object ${PRIV_KEY} ${CID} ${H_OID} ${False} &{FILE_USR_HEADER} + + Verify file hash s_file_read ${FILE_HASH} + Verify file hash h_file_read ${FILE_HASH} + &{ID_OBJ_S} = Create Dictionary ID=${S_OID} + Delete object ${PRIV_KEY} ${CID} ${S_OID} + Verify Head tombstone ${PRIV_KEY} ${CID} ${S_OID} + Wait Until Keyword Succeeds 2 min 30 sec + ... Search object ${PRIV_KEY} ${CID} ${EMPTY} @{EMPTY} &{ID_OBJ_S} + + Run Keyword And Expect Error * + ... Get object from NeoFS ${PRIV_KEY} ${CID} ${S_OID} s_file_read_2 + &{ID_OBJ_H} = Create Dictionary ID=${H_OID} + Delete object ${PRIV_KEY} ${CID} ${H_OID} + Verify Head tombstone ${PRIV_KEY} ${CID} ${H_OID} + Search object ${PRIV_KEY} ${CID} ${EMPTY} @{EMPTY} &{FILE_USR_HEADER} + Wait Until Keyword Succeeds 2 min 30 sec + ... Search object ${PRIV_KEY} ${CID} ${EMPTY} @{EMPTY} &{ID_OBJ_H} + Run Keyword And Expect Error * + ... Get object from NeoFS ${PRIV_KEY} ${CID} ${H_OID} s_file_read_2 + + + &{SGID_OBJ} = Create Dictionary ID=${SGID} + Delete object ${PRIV_KEY} ${CID} ${SGID} + Verify Head tombstone ${PRIV_KEY} ${CID} ${SGID} + Search object ${PRIV_KEY} ${CID} --sg @{EMPTY} + Wait Until Keyword Succeeds 2 min 30 sec + ... Search object ${PRIV_KEY} ${CID} ${EMPTY} @{EMPTY} &{SGID_OBJ} + Run Keyword And Expect Error * + ... Get object from NeoFS ${PRIV_KEY} ${CID} ${SGID} s_file_read_2 + + Cleanup File ${FILE} + Cleanup File s_file_read + Cleanup File h_file_read + Run Keyword And Expect Error Error: 's_file_read_2' file not found + ... Cleanup File s_file_read_2 + + +NeoFS Complex Object Operations + [Documentation] Testcase to validate NeoFS operations with complex object. + [Tags] Object NeoFS NeoCLI + [Timeout] 15 min + + ${PRIV_KEY} = Generate Neo private key + ${PUB_KEY} = Get Neo public key ${PRIV_KEY} + ${ADDR} = Get Neo address ${PRIV_KEY} + ${TX} = Request NeoFS Deposit ${PUB_KEY} + Wait Until Keyword Succeeds 1 min 15 sec + ... Transaction accepted in block ${TX} + Get Transaction ${TX} + ${BALANCE} = Wait Until Keyword Succeeds 10 min 1 min + ... Get Balance ${PUB_KEY} + Expected Balance ${PUB_KEY} 0 50 + ${CID} = Create container ${PRIV_KEY} + Container Existing ${PRIV_KEY} ${CID} + Wait Until Keyword Succeeds 2 min 30 sec + ... Expected Balance ${PUB_KEY} ${BALANCE} -0.00001424 + + ${SIZE} = Set Variable 20e+6 + ${FILE} = Generate file of bytes ${SIZE} + ${FILE_HASH} = Get file hash ${FILE} + ${S_OID} = Put object to NeoFS ${PRIV_KEY} ${FILE} ${CID} + ${H_OID} = Put object to NeoFS ${PRIV_KEY} ${FILE} ${CID} &{FILE_USR_HEADER} + + @{Link_obj_S} = Verify linked objects ${PRIV_KEY} ${CID} ${S_OID} ${SIZE} + @{Link_obj_H} = Verify linked objects ${PRIV_KEY} ${CID} ${H_OID} ${SIZE} + + @{Full_obj_list} = Create List @{Link_obj_S} @{Link_obj_H} ${S_OID} ${H_OID} + Search object ${PRIV_KEY} ${CID} ${EMPTY} @{Full_obj_list} + +# Next keyword has been removed due to https://neospcc.atlassian.net/browse/NSPCC-1103 +# Validate storage policy for object ${PRIV_KEY} 2 ${CID} ${S_OID} + ${SGID} = Create storage group ${PRIV_KEY} ${CID} ${S_OID} ${H_OID} + + @{S_OBJ_SG} = Create List ${SGID} + @{S_OBJ_ALL} = Create List ${S_OID} ${H_OID} ${SGID} + @{S_OBJ_H} = Create List ${H_OID} + + + + Search object ${PRIV_KEY} ${CID} --sg @{S_OBJ_SG} + Get storage group ${PRIV_KEY} ${CID} ${SGID} + Get object from NeoFS ${PRIV_KEY} ${CID} ${S_OID} s_file_read + Get object from NeoFS ${PRIV_KEY} ${CID} ${S_OID} h_file_read + Search object ${PRIV_KEY} ${CID} --root @{S_OBJ_ALL} + # Check sub-objects + + Search object ${PRIV_KEY} ${CID} --root @{S_OBJ_H} &{FILE_USR_HEADER} + Head object ${PRIV_KEY} ${CID} ${S_OID} ${True} + Head object ${PRIV_KEY} ${CID} ${H_OID} ${True} &{FILE_USR_HEADER} + + Run Keyword And Expect Error REGEXP:User header (\\w+=\\w+\\s?)+ was not found + ... Head object ${PRIV_KEY} ${CID} ${H_OID} ${False} &{FILE_USR_HEADER} + + Verify file hash s_file_read ${FILE_HASH} + Verify file hash h_file_read ${FILE_HASH} + &{ID_OBJ_S} = Create Dictionary ID=${S_OID} + Delete object ${PRIV_KEY} ${CID} ${S_OID} + Verify Head tombstone ${PRIV_KEY} ${CID} ${S_OID} + Wait Until Keyword Succeeds 2 min 30 sec + ... Search object ${PRIV_KEY} ${CID} --root @{EMPTY} &{ID_OBJ_S} + + Run Keyword And Expect Error * + ... Get object from NeoFS ${PRIV_KEY} ${CID} ${S_OID} s_file_read_2 + &{ID_OBJ_H} = Create Dictionary ID=${H_OID} + Delete object ${PRIV_KEY} ${CID} ${H_OID} + Verify Head tombstone ${PRIV_KEY} ${CID} ${H_OID} + Search object ${PRIV_KEY} ${CID} --root @{EMPTY} &{FILE_USR_HEADER} + Wait Until Keyword Succeeds 2 min 30 sec + ... Search object ${PRIV_KEY} ${CID} --root @{EMPTY} &{ID_OBJ_H} + Run Keyword And Expect Error * + ... Get object from NeoFS ${PRIV_KEY} ${CID} ${H_OID} s_file_read_2 + + + &{SGID_OBJ} = Create Dictionary ID=${SGID} + Delete object ${PRIV_KEY} ${CID} ${SGID} + Verify Head tombstone ${PRIV_KEY} ${CID} ${SGID} + Search object ${PRIV_KEY} ${CID} --sg @{EMPTY} + Wait Until Keyword Succeeds 2 min 30 sec + ... Search object ${PRIV_KEY} ${CID} ${EMPTY} @{EMPTY} &{SGID_OBJ} + Run Keyword And Expect Error * + ... Get object from NeoFS ${PRIV_KEY} ${CID} ${SGID} s_file_read_2 + + Cleanup File ${FILE} + Cleanup File s_file_read + Cleanup File h_file_read + Run Keyword And Expect Error Error: 's_file_read_2' file not found + ... Cleanup File s_file_read_2 diff --git a/robot/testsuites/integration/rw_test.robot b/robot/testsuites/integration/rw_test.robot index c1f5719b..ed32308c 100644 --- a/robot/testsuites/integration/rw_test.robot +++ b/robot/testsuites/integration/rw_test.robot @@ -1,7 +1,7 @@ *** Settings *** Variables ../../variables/common.py -Library ${RESOURCES}/kws_module.py +Library ${RESOURCES}/neofs.py Library ${RESOURCES}/assertions.py *** Variables *** diff --git a/robot/variables/common.py b/robot/variables/common.py index 7b7f3b2a..6324653e 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -1,8 +1,16 @@ #!/usr/bin/python3 +import os ROOT='../..' RESOURCES="%s/resources/lib" % ROOT +CERT="%s/../../ca" % ROOT # path from repo root is required for object put and get # in case when test is run from root in docker ABSOLUTE_FILE_PATH="/robot/testsuites/integration" + +JF_TOKEN = os.getenv('JF_TOKEN') +REG_USR = os.getenv('REG_USR') +REG_PWD = os.getenv('REG_PWD') +NEOFS_ENDPOINT = "s01.fs.localtest.nspcc.ru:8080" +NEOFS_NEO_API_ENDPOINT = "https://fs.localtest.nspcc.ru/neo_rpc/" \ No newline at end of file