forked from TrueCloudLab/s3-tests
s3select predicate tests
Signed-off-by: Albin Antony <aantony@redhat.com>
This commit is contained in:
Albin Antony
Ali Maredia
parent
dd163877d4
commit
adad16121f
1 changed files with 154 additions and 0 deletions
|
|
@ -1,5 +1,6 @@
|
||||||
import nose
|
import nose
|
||||||
import random
|
import random
|
||||||
|
import string
|
||||||
from nose.plugins.attrib import attr
|
from nose.plugins.attrib import attr
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -110,6 +111,23 @@ def create_random_csv_object(rows,columns,col_delim=",",record_delim="\n",csv_sc
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def create_random_csv_object_string(rows,columns,col_delim=",",record_delim="\n",csv_schema=""):
|
||||||
|
result = ""
|
||||||
|
if len(csv_schema)>0 :
|
||||||
|
result = csv_schema + record_delim
|
||||||
|
|
||||||
|
for _ in range(rows):
|
||||||
|
row = ""
|
||||||
|
for _ in range(columns):
|
||||||
|
if random.randint(0,9) == 5:
|
||||||
|
row = row + "{}{}".format(''.join(random.choice(string.ascii_letters) for m in range(10)) + "aeiou",col_delim)
|
||||||
|
else:
|
||||||
|
row = row + "{}{}".format(''.join("cbcd" + random.choice(string.ascii_letters) for m in range(10)) + "vwxyzzvwxyz" ,col_delim)
|
||||||
|
|
||||||
|
result += row + record_delim
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def upload_csv_object(bucket_name,new_key,obj):
|
def upload_csv_object(bucket_name,new_key,obj):
|
||||||
|
|
||||||
|
|
@ -195,6 +213,12 @@ def test_column_sum_min_max():
|
||||||
|
|
||||||
nose.tools.assert_equal( int(res_s3select), int(res_target))
|
nose.tools.assert_equal( int(res_s3select), int(res_target))
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select avg(int(_6)) from stdin;") ).replace(",","")
|
||||||
|
list_int = create_list_of_int( 6 , csv_obj )
|
||||||
|
res_target = float(sum(list_int ))/10000
|
||||||
|
|
||||||
|
nose.tools.assert_equal( float(res_s3select), float(res_target))
|
||||||
|
|
||||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select max(int(_4)) from stdin;") ).replace(",","")
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select max(int(_4)) from stdin;") ).replace(",","")
|
||||||
list_int = create_list_of_int( 4 , csv_obj )
|
list_int = create_list_of_int( 4 , csv_obj )
|
||||||
res_target = max( list_int )
|
res_target = max( list_int )
|
||||||
|
|
@ -230,6 +254,136 @@ def test_column_sum_min_max():
|
||||||
|
|
||||||
nose.tools.assert_equal( int(count)*4 , int(sum1)-int(sum2) )
|
nose.tools.assert_equal( int(count)*4 , int(sum1)-int(sum2) )
|
||||||
|
|
||||||
|
@attr('s3select')
|
||||||
|
def test_nullif_expressions():
|
||||||
|
|
||||||
|
csv_obj = create_random_csv_object(10000,10)
|
||||||
|
|
||||||
|
csv_obj_name = get_random_string()
|
||||||
|
bucket_name = "test"
|
||||||
|
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||||
|
|
||||||
|
res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where nullif(_1,_2) is null ;") ).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 == _2 ;") ).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_nullif, res_s3select)
|
||||||
|
|
||||||
|
res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where not nullif(_1,_2) is null ;") ).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_nullif, res_s3select)
|
||||||
|
|
||||||
|
res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where nullif(_1,_2) == _1 ;") ).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_nullif, res_s3select)
|
||||||
|
|
||||||
|
@attr('s3select')
|
||||||
|
def test_lowerupper_expressions():
|
||||||
|
|
||||||
|
csv_obj = create_random_csv_object(1,10)
|
||||||
|
|
||||||
|
csv_obj_name = get_random_string()
|
||||||
|
bucket_name = "test"
|
||||||
|
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select lower("AB12cd$$") from stdin ;') ).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select, "ab12cd$$,")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select upper("ab12CD$$") from stdin ;') ).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select, "AB12CD$$,")
|
||||||
|
|
||||||
|
@attr('s3select')
|
||||||
|
def test_in_expressions():
|
||||||
|
|
||||||
|
# purpose of test: engine is process correctly several projections containing aggregation-functions
|
||||||
|
csv_obj = create_random_csv_object(10000,10)
|
||||||
|
|
||||||
|
csv_obj_name = get_random_string()
|
||||||
|
bucket_name = "test"
|
||||||
|
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) in(1);')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1;')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) in(1,0);')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1 or int(_1) == 0;')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) in(1,0,2);')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) == 1 or int(_2) == 0 or int(_2) == 2;')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5);')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5;')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where characterlength(_1) == 2 and substr(_1,2,1) in ("3");')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where _1 like "_3";')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
@attr('s3select')
|
||||||
|
def test_like_expressions():
|
||||||
|
|
||||||
|
csv_obj = create_random_csv_object_string(10000,10)
|
||||||
|
|
||||||
|
csv_obj_name = get_random_string()
|
||||||
|
bucket_name = "test"
|
||||||
|
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%aeio%";')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substr(_1,11,4) == "aeio" ;')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "cbcd%";')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substr(_1,1,4) == "cbcd";')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "%y[y-z]";')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substr(_3,charlength(_3),1) between "y" and "z" and substr(_3,charlength(_3)-1,1) == "y";')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%yz";')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substr(_2,charlength(_2),1) == "z" and substr(_2,charlength(_2)-1,1) == "y";')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "c%z";')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substr(_3,charlength(_3),1) == "z" and substr(_3,1,1) == "c";')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%xy_";')).replace("\n","")
|
||||||
|
|
||||||
|
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substr(_2,charlength(_2)-1,1) == "y" and substr(_2,charlength(_2)-2,1) == "x";')).replace("\n","")
|
||||||
|
|
||||||
|
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||||
|
|
||||||
|
|
||||||
@attr('s3select')
|
@attr('s3select')
|
||||||
def test_complex_expressions():
|
def test_complex_expressions():
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue