alexvanin/test-helper
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Initial commit

Browse source
This commit is contained in:
Alexey Vanin 2024-07-16 20:47:56 +03:00
commit 02e08b813e
8 changed files with 317 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
acl-migrate/.gitignore vendored Normal file
View file

121
acl-migrate/README.md Normal file
View file

@ -0,0 +1,121 @@
# Access control checker
This repo contains `prepare-*` and `check-*` scripts to verify migration of eACL
policies.
`prepare-*` scripts must be invoked before update to create some buckets and
containers with variety of policies.
`check-*` scripts must be invoked after
update to verify that expected access control behavior is intact.
`*-aws` scripts invoke AWS CLI to check S3 gateway behaviour.
`*-ffs` scripts invoke FrostFS CLI to check storage behaviour.
## Prerequisites
Make sure you have `aws` and `frostfs-cli` commands available.
To run these scripts create `env` file `cp env.example env`
### FILE
Path for a file with the size of a simple object.
```
FILE=./data/cat.jpg
```
### COMPLEXFILE
Path for a file with the size of a complex object that should be split during
put operation.
```
COMPLEXFILE=./data/70m
```
### S3ENDPOINT
S3 Gateway endpoint.
```
S3ENDPOINT=http://localhost:8084
```
### S3PROF
Profile name with AWS credentials for content owner
```
$ aws configure --profile main
S3PROF=main
```
### S3PROFEXT
Profile name with AWS credentials for other user without specific permissions.
```
$ aws configure --profile ext
S3PROFEXT=ext
```
### S3PREFIX
Bucket prefix for all created containers. Modify between consecutive runs.
```
S3PREFIX=av01
```
### S3KEY
Object name stored in buckets.
```
S3KEY=some/object
```
### FFSCONF
Path to FrostFS CLI config file with content owner credentials
```
FFSCONF=./data/ffs-cli.yaml
```
### FFSCONFEXT
Path to FrostFS CLI config file with other user without specific permissions.
```
FFSCONFEXT=./data/ffs-cli-ext.yaml
```
### PLACEMENT
Policy for FrostFS containers
```
PLACEMENT="REP 1"
```
### CHECKFILE
Path to file with state between `prepare-ffs.sh` and `chech-ffs.sh` runs.
```
CHECKFILE=checkfile.txt
```
## Run
After configuring `env` file, run `prepare-*` scripts in any order. Make sure
to save logs as they can be useful for debugging.
```
$ ./prepare-aws.sh | tee prepare-aws.log
$ ./prepare-ffs.sh | tee prepare-aws.log
```
Then run check scripts after update.
```
$ ./check-ffs.sh | tee check-ffs.log
$ ./check-aws.sh | tee check-aws.log
```
In case of any failures, scripts return non-zero exit code.

29
acl-migrate/check-aws.sh Executable file
View file

@ -0,0 +1,29 @@
#!/bin/bash -x
source ./env
# check public read-write bucket with object inside
aws s3api get-bucket-acl --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT || exit 1
aws s3api get-object-acl --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --key $S3KEY || exit 1
aws s3api get-object --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --key $S3KEY out || exit 1 # allow owner to get object
aws s3api put-object --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --key $S3KEY-new --body $FILE || exit 1 # allow owner to put object
aws s3api get-object --profile $S3PROFEXT --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --key $S3KEY out || exit 1 # allow others to get object
aws s3api put-object --profile $S3PROFEXT --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --key $S3KEY-new --body $FILE || exit 1 # allow others to put object
# check public read bucket with object inside
aws s3api get-bucket-acl --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT || exit 1
aws s3api get-object-acl --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --key $S3KEY || exit 1
aws s3api get-object --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --key $S3KEY out || exit 1 # allow owner to get object
aws s3api put-object --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --key $S3KEY-new --body $FILE || exit 1 # allow owner to put object
aws s3api get-object --profile $S3PROFEXT --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --key $S3KEY out || exit 1 # allow others to get object
aws s3api put-object --profile $S3PROFEXT --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --key $S3KEY-new --body $FILE && exit 1 # deny others to put object
# check private read bucket with object inside
aws s3api get-bucket-acl --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT || exit 1
aws s3api get-object-acl --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --key $S3KEY || exit 1
aws s3api get-object --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --key $S3KEY out || exit 1 # allow owner to get object
aws s3api put-object --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --key $S3KEY-new --body $FILE || exit 1 # allow owner to put object
aws s3api get-object --profile $S3PROFEXT --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --key $S3KEY out && exit 1 # deny others to get object
aws s3api put-object --profile $S3PROFEXT --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --key $S3KEY-new --body $FILE && exit 1 # deny others to put object
rm out && exit 0

63
acl-migrate/check-ffs.sh Executable file
View file

@ -0,0 +1,63 @@
#!/bin/bash -x
source ./env
readarray -t arr < $CHECKFILE # read object and container ids
# check private container with object inside
CID=${arr[0]}
OID=${arr[1]}
frostfs-cli --config $FFSCONF object get --cid $CID --oid $OID --file out || exit 1 # allow owner to get object
frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --no-progress || exit 1 # allow owner to put object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID --file out && exit 1 # deny others to get object
frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $FILE --no-progress && exit 1 # deny others to put object
# check public read container with object inside
CID=${arr[2]}
OID=${arr[3]}
frostfs-cli --config $FFSCONF object get --cid $CID --oid $OID --file out || exit 1 # allow owner to get object
frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --no-progress || exit 1 # allow owner to put object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID --file out || exit 1 # allow others to get object
frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $FILE --no-progress && exit 1 # deny others to put object
# check public read write container with object inside
CID=${arr[4]}
OID=${arr[5]}
frostfs-cli --config $FFSCONF object get --cid $CID --oid $OID --file out || exit 1 # allow owner to get object
frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --no-progress || exit 1 # allow owner to put object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID --file out || exit 1 # allow others to get object
frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $FILE --no-progress || exit 1 # allow others to put object
# check allow get / put specific objects in container
CID=${arr[6]}
OID_S_ATTR=${arr[7]}
OID_C_ATTR=${arr[8]}
OID_S_NOATTR=${arr[9]}
OID_C_NOATTR=${arr[10]}
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID_S_ATTR --file out || exit 1 # allow others to get small attr object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID_C_ATTR --file out || exit 1 # allow others to get complex attr object
frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $FILE --attributes "Foo=Bar" --no-progress || exit 1 # allow others to put simple attr object
frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $COMPLEXFILE --attributes "Foo=Bar" --no-progress || exit 1 # allow others to put complex attr object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID_S_NOATTR --file out && exit 1 # deny others to get small attr object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID_C_NOATTR --file out && exit 1 # deny others to get complex attr object
# ! DIDN'T WORK v1.5.0 ! frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $FILE --no-progress && exit 1 # deny others to put simple attr object
# ! DIDN'T WORK v1.5.0 ! frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $COMPLEXFILE --no-progress && exit 1 # deny others to put complex attr object
# check deny get / put specific objects in container
CID=${arr[11]}
OID_S_ATTR=${arr[12]}
OID_C_ATTR=${arr[13]}
OID_S_NOATTR=${arr[14]}
OID_C_NOATTR=${arr[15]}
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID_S_ATTR --file out && exit 1 # deny others to get small attr object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID_C_ATTR --file out && exit 1 # deny others to get complex attr object
# ! DIDN'T WORK v1.5.0 ! frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $FILE --attributes "Foo=Bar" --no-progress && exit 1 # deny others to put simple attr object
# ! DIDN'T WORK v1.5.0 ! frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $COMPLEXFILE --attributes "Foo=Bar" --no-progress && exit 1 # deny others to put complex attr object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID_S_NOATTR --file out || exit 1 # allow others to get small attr object
frostfs-cli --config $FFSCONFEXT object get --cid $CID --oid $OID_C_NOATTR --file out || exit 1 # allow others to get complex attr object
frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $FILE --no-progress || exit 1 # allow others to put simple attr object
frostfs-cli --config $FFSCONFEXT object put --cid $CID --file $COMPLEXFILE --no-progress || exit 1 # allow others to put complex attr object
rm out && exit 0

0
acl-migrate/data/.gitkeep Normal file
View file

13
acl-migrate/env.example Normal file
View file

@ -0,0 +1,13 @@
FILE=./data/cat.jpg
COMPLEXFILE=./data/70m
S3ENDPOINT=http://localhost:8084
S3PROF=main
S3PROFEXT=ext
S3PREFIX=av04
S3KEY=some/object
FFSCONF=./data/ffs-cli.yaml
FFSCONFEXT=./data/ffs-cli-ext.yaml
PLACEMENT="REP 1"
CHECKFILE=checkfile.txt

25
acl-migrate/prepare-aws.sh Executable file
View file

@ -0,0 +1,25 @@
#!/bin/bash -x
source ./env
export AWS_PAGER=""
# public read-write bucket with object inside
aws s3api create-bucket --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --acl public-read-write || exit 1
aws s3api head-bucket --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --debug 2>&1 | grep 'X-Container-Id' || exit 1
aws s3api put-object --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --key $S3KEY --body $FILE || exit 1
aws s3api get-bucket-acl --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT || exit 1
aws s3api get-object-acl --profile $S3PROF --bucket $S3PREFIX-public-rw --endpoint $S3ENDPOINT --key $S3KEY || exit 1
# public read bucket with object inside
aws s3api create-bucket --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --acl public-read || exit 1
aws s3api head-bucket --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --debug 2>&1 | grep 'X-Container-Id' || exit 1
aws s3api put-object --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --key $S3KEY --body $FILE || exit 1
aws s3api get-bucket-acl --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT || exit 1
aws s3api get-object-acl --profile $S3PROF --bucket $S3PREFIX-public-r --endpoint $S3ENDPOINT --key $S3KEY || exit 1
# private read bucket with object inside
aws s3api create-bucket --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --acl private || exit 1
aws s3api head-bucket --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --debug 2>&1 | grep 'X-Container-Id' || exit 1
aws s3api put-object --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --key $S3KEY --body $FILE || exit 1
aws s3api get-bucket-acl --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT || exit 1
aws s3api get-object-acl --profile $S3PROF --bucket $S3PREFIX-private --endpoint $S3ENDPOINT --key $S3KEY || exit 1

66
acl-migrate/prepare-ffs.sh Executable file
View file

@ -0,0 +1,66 @@
#!/bin/bash -x
source ./env
> $CHECKFILE # clear file
# private container with object inside
CID=$(frostfs-cli --config $FFSCONF container create -p "$PLACEMENT" --await --basic-acl eacl-private | head -1 | awk '{print $2}')
OID=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID || exit 1
echo $CID >> $CHECKFILE
echo $OID >> $CHECKFILE
# public read container with object inside
CID=$(frostfs-cli --config $FFSCONF container create -p "$PLACEMENT" --await --basic-acl eacl-public-read | head -1 | awk '{print $2}')
OID=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID || exit 1
echo $CID >> $CHECKFILE
echo $OID >> $CHECKFILE
# public read write container with object inside
CID=$(frostfs-cli --config $FFSCONF container create -p "$PLACEMENT" --await --basic-acl eacl-public-read-write | head -1 | awk '{print $2}')
OID=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID || exit 1
echo $CID >> $CHECKFILE
echo $OID >> $CHECKFILE
# allow get / put specific objects in private container
CID=$(frostfs-cli --config $FFSCONF container create -p "$PLACEMENT" --await --basic-acl eacl-public-read-write | head -1 | awk '{print $2}')
OID_S_ATTR=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --attributes "Foo=Bar" --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID_S_ATTR || exit 1
OID_C_ATTR=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $COMPLEXFILE --attributes "Foo=Bar" --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID_C_ATTR || exit 1
OID_S_NOATTR=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID_S_NOATTR || exit 1
OID_C_NOATTR=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $COMPLEXFILE --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID_C_NOATTR || exit 1
frostfs-cli --config $FFSCONF acl extended create --cid $CID -r 'allow get obj:Foo=Bar others' -r 'allow put obj:Foo=Bar others' -r 'deny get others' -r 'deny put others' -o acl.json || exit 1
frostfs-cli --config $FFSCONF container set-eacl --cid $CID --table acl.json --await || exit 1
echo $CID >> $CHECKFILE
echo $OID_S_ATTR >> $CHECKFILE
echo $OID_C_ATTR >> $CHECKFILE
echo $OID_S_NOATTR >> $CHECKFILE
echo $OID_C_NOATTR >> $CHECKFILE
rm acl.json
# deny get / put specific objects in private container
CID=$(frostfs-cli --config $FFSCONF container create -p "$PLACEMENT" --await --basic-acl eacl-public-read-write | head -1 | awk '{print $2}')
OID_S_ATTR=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --attributes "Foo=Bar" --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID_S_ATTR || exit 1
OID_C_ATTR=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $COMPLEXFILE --attributes "Foo=Bar" --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID_C_ATTR || exit 1
OID_S_NOATTR=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $FILE --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID_S_NOATTR || exit 1
OID_C_NOATTR=$(frostfs-cli --config $FFSCONF object put --cid $CID --file $COMPLEXFILE --no-progress | tail -2 | head -1 | awk '{print $2}')
frostfs-cli --config $FFSCONF object head --cid $CID --oid $OID_C_NOATTR || exit 1
frostfs-cli --config $FFSCONF acl extended create --cid $CID -r 'deny get obj:Foo=Bar others' -r 'deny put obj:Foo=Bar others' -r 'allow get others' -r 'allow put others' -o acl.json || exit 1
frostfs-cli --config $FFSCONF container set-eacl --cid $CID --table acl.json --await || exit 1
echo $CID >> $CHECKFILE
echo $OID_S_ATTR >> $CHECKFILE
echo $OID_C_ATTR >> $CHECKFILE
echo $OID_S_NOATTR >> $CHECKFILE
echo $OID_C_NOATTR >> $CHECKFILE
rm acl.json
exit 0