forked from TrueCloudLab/s3-tests
the upgraded CSV parser has some impact on engine result. (redundant comma, quote behavior)
Signed-off-by: galsalomon66 <gal.salomon@gmail.com>
(cherry picked from commit 89f97ed35c)
This commit is contained in:
galsalomon66
Casey Bodley
parent
02ee15041a
commit
89b10274dd
1 changed files with 160 additions and 17 deletions
|
|
@ -15,6 +15,10 @@ from . import (
|
|||
import logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
#TODO the following resolve an issue with python version (should be conditional)
|
||||
import collections
|
||||
collections.Callable = collections.abc.Callable
|
||||
|
||||
region_name = ''
|
||||
|
||||
# recurssion function for generating arithmetical expression
|
||||
|
|
@ -749,6 +753,136 @@ def test_truefalselike_expressions():
|
|||
|
||||
s3select_assert_result( res_s3select_like, res_s3select )
|
||||
|
||||
@attr('s3select')
|
||||
def test_nullif_expressions():
|
||||
|
||||
csv_obj = create_random_csv_object(10000,10)
|
||||
|
||||
csv_obj_name = get_random_string()
|
||||
bucket_name = "test"
|
||||
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||
|
||||
res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where nullif(_1,_2) is null ;") ).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 = _2 ;") ).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_nullif, res_s3select)
|
||||
|
||||
res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where not nullif(_1,_2) is null ;") ).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_nullif, res_s3select)
|
||||
|
||||
res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where nullif(_1,_2) = _1 ;") ).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_nullif, res_s3select)
|
||||
|
||||
@attr('s3select')
|
||||
def test_lowerupper_expressions():
|
||||
|
||||
csv_obj = create_random_csv_object(1,10)
|
||||
|
||||
csv_obj_name = get_random_string()
|
||||
bucket_name = "test"
|
||||
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select lower("AB12cd$$") from stdin ;') ).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select, "ab12cd$$")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select upper("ab12CD$$") from stdin ;') ).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select, "AB12CD$$")
|
||||
|
||||
@attr('s3select')
|
||||
def test_in_expressions():
|
||||
|
||||
# purpose of test: engine is process correctly several projections containing aggregation-functions
|
||||
csv_obj = create_random_csv_object(10000,10)
|
||||
|
||||
csv_obj_name = get_random_string()
|
||||
bucket_name = "test"
|
||||
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) in(1);')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) = 1;')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) in(1,0);')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) = 1 or int(_1) = 0;')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) in(1,0,2);')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) = 1 or int(_2) = 0 or int(_2) = 2;')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5);')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 = int(_3)*2 or int(_2)*2 = int(_4)*3 or int(_2)*2 = int(_5)*5;')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where character_length(_1) = 2 and substring(_1,2,1) in ("3");')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where _1 like "_3";')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
@attr('s3select')
|
||||
def test_like_expressions():
|
||||
|
||||
csv_obj = create_random_csv_object_string(10000,10)
|
||||
|
||||
csv_obj_name = get_random_string()
|
||||
bucket_name = "test"
|
||||
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%aeio%";')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,11,4) = "aeio" ;')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "cbcd%";')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,1,4) = "cbcd";')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "%y[y-z]";')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_3,character_length(_3),1) between "y" and "z" and substring(_3,character_length(_3)-1,1) = "y";')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%yz";')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_2,character_length(_2),1) = "z" and substring(_2,character_length(_2)-1,1) = "y";')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "c%z";')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_3,character_length(_3),1) = "z" and substring(_3,1,1) = "c";')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%xy_";')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_2,character_length(_2)-1,1) = "y" and substring(_2,character_length(_2)-2,1) = "x";')).replace("\n","")
|
||||
|
||||
nose.tools.assert_equal( res_s3select_in, res_s3select )
|
||||
|
||||
|
||||
@attr('s3select')
|
||||
def test_complex_expressions():
|
||||
|
||||
|
|
@ -891,7 +1025,7 @@ def test_csv_parser():
|
|||
# purpuse: test default csv values(, \n " \ ), return value may contain meta-char
|
||||
# NOTE: should note that default meta-char for s3select are also for python, thus for one example double \ is mandatory
|
||||
|
||||
csv_obj = ',first,,,second,third="c31,c32,c33",forth="1,2,3,4",fifth="my_string=\\"any_value\\" , my_other_string=\\"aaaa,bbb\\" ",' + "\n"
|
||||
csv_obj = r',first,,,second,third="c31,c32,c33",forth="1,2,3,4",fifth=my_string=\"any_value\" \, my_other_string=\"aaaa\,bbb\" ,' + "\n"
|
||||
csv_obj_name = get_random_string()
|
||||
bucket_name = "test"
|
||||
|
||||
|
|
@ -899,15 +1033,15 @@ def test_csv_parser():
|
|||
|
||||
# return value contain comma{,}
|
||||
res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _6 from s3object;") ).replace("\n","")
|
||||
s3select_assert_result( res_s3select_alias, 'third="c31,c32,c33"')
|
||||
s3select_assert_result( res_s3select_alias, 'third=c31,c32,c33')
|
||||
|
||||
# return value contain comma{,}
|
||||
res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _7 from s3object;") ).replace("\n","")
|
||||
s3select_assert_result( res_s3select_alias, 'forth="1,2,3,4"')
|
||||
s3select_assert_result( res_s3select_alias, 'forth=1,2,3,4')
|
||||
|
||||
# return value contain comma{,}{"}, escape-rule{\} by-pass quote{"} , the escape{\} is removed.
|
||||
res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _8 from s3object;") ).replace("\n","")
|
||||
s3select_assert_result( res_s3select_alias, 'fifth="my_string="any_value" , my_other_string="aaaa,bbb" "')
|
||||
s3select_assert_result( res_s3select_alias, 'fifth=my_string="any_value" , my_other_string="aaaa,bbb" ')
|
||||
|
||||
# return NULL as first token
|
||||
res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _1 from s3object;") ).replace("\n","")
|
||||
|
|
@ -1246,31 +1380,40 @@ def test_output_serial_expressions():
|
|||
bucket_name = "test"
|
||||
upload_csv_object(bucket_name,csv_obj_name,csv_obj)
|
||||
|
||||
res_s3select_1 = remove_xml_tags_from_result( run_s3select_output(bucket_name,csv_obj_name,"select _1, _2 from s3object where nullif(_1,_2) is null ;", "ALWAYS") ).replace("\n",",")
|
||||
res_s3select_1 = remove_xml_tags_from_result( run_s3select_output(bucket_name,csv_obj_name,"select _1, _2 from s3object where nullif(_1,_2) is null ;", "ALWAYS") ).replace("\n",",").replace(",","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _1, _2 from s3object where _1 = _2 ;") ).replace("\n",",")
|
||||
|
||||
res_s3select_list = res_s3select.split(',')
|
||||
res_s3select_final = (','.join('"' + item + '"' for item in res_s3select_list)).replace('""','') # remove empty result(first,last)
|
||||
|
||||
s3select_assert_result( res_s3select_1, res_s3select_final)
|
||||
res_s3select_list.pop()
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select_output(bucket_name,csv_obj_name,'select int(_1) from s3object where (int(_1) in(int(_2)));', "ASNEEDED", '$', '#')).replace("\n","")
|
||||
res_s3select_final = (''.join('"' + item + '"' for item in res_s3select_list))
|
||||
|
||||
s3select_assert_result( '""'+res_s3select_1+'""', res_s3select_final)
|
||||
|
||||
|
||||
res_s3select_in = remove_xml_tags_from_result( run_s3select_output(bucket_name,csv_obj_name,'select int(_1) from s3object where (int(_1) in(int(_2)));', "ASNEEDED", '$', '#')).replace("\n","#") ## TODO why \n appears in output?
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) = int(_2);')).replace("\n","#")
|
||||
res_s3select = res_s3select[1:len(res_s3select)] # remove first redundant
|
||||
res_s3select_final = res_s3select[0:len(res_s3select)-1] # remove last redundant
|
||||
|
||||
res_s3select_list = res_s3select.split('#')
|
||||
|
||||
res_s3select_list.pop()
|
||||
|
||||
res_s3select_final = (''.join(item + '#' for item in res_s3select_list))
|
||||
|
||||
|
||||
s3select_assert_result(res_s3select_in , res_s3select_final )
|
||||
|
||||
s3select_assert_result( res_s3select_in, res_s3select_final )
|
||||
|
||||
res_s3select_quot = remove_xml_tags_from_result( run_s3select_output(bucket_name,csv_obj_name,'select int(_1) from s3object where (int(_1) in(int(_2)));', "ALWAYS", '$', '#')).replace("\n","")
|
||||
|
||||
res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) = int(_2);')).replace("\n","#")
|
||||
res_s3select = res_s3select[1:len(res_s3select)] # remove first redundant
|
||||
res_s3select = res_s3select[0:len(res_s3select)-1] # remove last redundant
|
||||
|
||||
res_s3select_list = res_s3select.split('#')
|
||||
res_s3select_final = ('#'.join('"' + item + '"' for item in res_s3select_list)).replace('""','')
|
||||
|
||||
s3select_assert_result( res_s3select_quot, res_s3select_final )
|
||||
|
||||
res_s3select_list.pop()
|
||||
|
||||
res_s3select_final = (''.join('"' + item + '"' + '#' for item in res_s3select_list))
|
||||
|
||||
s3select_assert_result( '""#'+res_s3select_quot+'""#', res_s3select_final )
|
||||
|
|
|
|||
Loading…
Reference in a new issue