From 4a86ebbe8b666dbbcf9c5b6b7935129dfd242b28 Mon Sep 17 00:00:00 2001 From: Albin Antony Date: Wed, 20 Jan 2021 13:16:05 +0530 Subject: [PATCH 1/4] s3select: align s3-tests with new changes in s3select Fix when then, date functions and NULL, add escape, trim tests Signed-off-by: Albin Antony --- s3tests_boto3/functional/test_s3select.py | 368 +++++++++++++++++++++- 1 file changed, 357 insertions(+), 11 deletions(-) diff --git a/s3tests_boto3/functional/test_s3select.py b/s3tests_boto3/functional/test_s3select.py index adc3812..3fef0c3 100644 --- a/s3tests_boto3/functional/test_s3select.py +++ b/s3tests_boto3/functional/test_s3select.py @@ -100,7 +100,7 @@ def create_csv_object_for_datetime(rows,columns): for _ in range(rows): row = "" for _ in range(columns): - row = row + "{}{:02d}{:02d}-{:02d}{:02d}{:02d},".format(random.randint(0,100)+1900,random.randint(1,12),random.randint(1,28),random.randint(0,23),random.randint(0,59),random.randint(0,59),) + row = row + "{}{:02d}{:02d}T{:02d}{:02d}{:02d}Z,".format(random.randint(0,100)+1900,random.randint(1,12),random.randint(1,28),random.randint(0,23),random.randint(0,59),random.randint(0,59),) result += row + "\n" return result @@ -135,6 +135,50 @@ def create_random_csv_object_string(rows,columns,col_delim=",",record_delim="\n" return result +def create_random_csv_object_trim(rows,columns,col_delim=",",record_delim="\n",csv_schema=""): + result = "" + if len(csv_schema)>0 : + result = csv_schema + record_delim + + for _ in range(rows): + row = "" + for _ in range(columns): + row = row + "{}{}".format(''.join(" aeiou ") ,col_delim) + + result += row + record_delim + + return result + +def create_random_csv_object_escape(rows,columns,col_delim=",",record_delim="\n",csv_schema=""): + result = "" + if len(csv_schema)>0 : + result = csv_schema + record_delim + + for _ in range(rows): + row = "" + for _ in range(columns): + if random.randint(0,9) == 5: + row = row + "{}{}".format(''.join("_ar") ,col_delim) + else: + row = row + "{}{}".format(''.join("aeio_") ,col_delim) + + result += row + record_delim + + return result + +def create_random_csv_object_null(rows,columns,col_delim=",",record_delim="\n",csv_schema=""): + result = "" + if len(csv_schema)>0 : + result = csv_schema + record_delim + + for _ in range(rows): + row = "" + for _ in range(columns): + row = row + "{}{}".format(''.join("") ,col_delim) + + result += row + record_delim + + return result def upload_csv_object(bucket_name,new_key,obj): @@ -294,6 +338,64 @@ def test_nullif_expressions(): nose.tools.assert_equal( res_s3select_nullif, res_s3select) + csv_obj = create_random_csv_object_null(10000,10) + + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where nullif(_1,null) is null;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where _1 is null;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_nullif, res_s3select) + +@attr('s3select') +def test_nulliftrue_expressions(): + + csv_obj = create_random_csv_object(10000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where (nullif(_1,_2) is null) == true ;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 == _2 ;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_nullif, res_s3select) + + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where not (nullif(_1,_2) is null) == true ;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_nullif, res_s3select) + + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where (nullif(_1,_2) == _1 == true) ;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_nullif, res_s3select) + +@attr('s3select') +def test_is_not_null_expressions(): + + csv_obj = create_random_csv_object(10000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_null = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where nullif(_1,_2) is not null ;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where _1 != _2 ;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_null, res_s3select) + + res_s3select_null = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where (nullif(_1,_1) and _1 == _2) is not null ;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where _1 != _2 ;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_null, res_s3select) + @attr('s3select') def test_lowerupper_expressions(): @@ -351,6 +453,45 @@ def test_in_expressions(): nose.tools.assert_equal( res_s3select_in, res_s3select ) +@attr('s3select') +def test_true_false_in_expressions(): + + csv_obj = create_random_csv_object(10000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where (int(_1) in(1)) == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where (int(_1) in(1,0)) == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1 or int(_1) == 0;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where (int(_2) in(1,0,2)) == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) == 1 or int(_2) == 0 or int(_2) == 2;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where (int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5)) == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where (character_length(_1) == 2) == true and (substring(_1,2,1) in ("3")) == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where _1 like "_3";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + @attr('s3select') def test_like_expressions(): @@ -396,6 +537,50 @@ def test_like_expressions(): nose.tools.assert_equal( res_s3select_in, res_s3select ) +@attr('s3select') +def test_truefalselike_expressions(): + + csv_obj = create_random_csv_object_string(1000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_1 like "%aeio%") == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,11,4) == "aeio" ;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_1 like "cbcd%") == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,1,4) == "cbcd";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_3 like "%y[y-z]") == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_3,char_length(_3),1) between "y" and "z") == true and (substring(_3,char_length(_3)-1,1) == "y") == true;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_2 like "%yz") == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_2,char_length(_2),1) == "z") == true and (substring(_2,char_length(_2)-1,1) == "y") == true;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_3 like "c%z") == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_3,char_length(_3),1) == "z") == true and (substring(_3,1,1) == "c") == true;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_2 like "%xy_") == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_2,char_length(_2)-1,1) == "y") == true and (substring(_2,char_length(_2)-2,1) == "x") == true;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) @attr('s3select') def test_complex_expressions(): @@ -481,24 +666,57 @@ def test_datetime(): upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where extract("year",timestamp(_1)) > 1950 and extract("year",timestamp(_1)) < 1960;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where extract(year from to_timestamp(_1)) > 1950 and extract(year from to_timestamp(_1)) < 1960;') ) res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where int(substring(_1,1,4))>1950 and int(substring(_1,1,4))<1960;') ) nose.tools.assert_equal( res_s3select_date_time, res_s3select_substring) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where datediff("month",timestamp(_1),dateadd("month",2,timestamp(_1)) ) == 2;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where date_diff(month,to_timestamp(_1),date_add(month,2,to_timestamp(_1)) ) == 2;') ) res_s3select_count = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin;') ) nose.tools.assert_equal( res_s3select_date_time, res_s3select_count) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where datediff("year",timestamp(_1),dateadd("day", 366 ,timestamp(_1))) == 1 ;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where date_diff(year,to_timestamp(_1),date_add(day, 366 ,to_timestamp(_1))) == 1 ;') ) nose.tools.assert_equal( res_s3select_date_time, res_s3select_count) # validate that utcnow is integrate correctly with other date-time functions - res_s3select_date_time_utcnow = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where datediff("hours",utcnow(),dateadd("day",1,utcnow())) == 24 ;') ) + res_s3select_date_time_utcnow = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where date_diff(hour,utcnow(),date_add(day,1,utcnow())) == 24 ;') ) + + nose.tools.assert_equal( res_s3select_date_time_utcnow, res_s3select_count) + +@attr('s3select') +def test_true_false_datetime(): + + # purpose of test is to validate date-time functionality is correct, + # by creating same groups with different functions (nested-calls) ,which later produce the same result + + csv_obj = create_csv_object_for_datetime(10000,1) + + csv_obj_name = get_random_string() + bucket_name = "test" + + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where (extract(year from to_timestamp(_1)) > 1950) == true and (extract(year from to_timestamp(_1)) < 1960) == true;') ) + + res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where int(substring(_1,1,4))>1950 and int(substring(_1,1,4))<1960;') ) + + nose.tools.assert_equal( res_s3select_date_time, res_s3select_substring) + + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where (date_diff(month,to_timestamp(_1),date_add(month,2,to_timestamp(_1)) ) == 2) == true;') ) + + res_s3select_count = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin;') ) + + nose.tools.assert_equal( res_s3select_date_time, res_s3select_count) + + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where (date_diff(year,to_timestamp(_1),date_add(day, 366 ,to_timestamp(_1))) == 1) == true ;') ) + + nose.tools.assert_equal( res_s3select_date_time, res_s3select_count) + + res_s3select_date_time_utcnow = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where (date_diff(hour,utcnow(),date_add(day,1,utcnow())) == 24) == true ;') ) nose.tools.assert_equal( res_s3select_date_time_utcnow, res_s3select_count) @@ -528,19 +746,19 @@ def test_csv_parser(): # return NULL as first token res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _1 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, ',') + nose.tools.assert_equal( res_s3select_alias, 'null,') # return NULL in the middle of line res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _3 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, ',') + nose.tools.assert_equal( res_s3select_alias, 'null,') # return NULL in the middle of line (successive) res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _4 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, ',') + nose.tools.assert_equal( res_s3select_alias, 'null,') # return NULL at the end line res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _9 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, ',') + nose.tools.assert_equal( res_s3select_alias, 'null,') @attr('s3select') def test_csv_definition(): @@ -605,7 +823,7 @@ def test_schema_definition(): assert res_multiple_defintion.find("multiple definition of column {c4} as schema-column and alias") > 0 @attr('s3select') -def test_when_than_else_expressions(): +def test_when_then_else_expressions(): csv_obj = create_random_csv_object(10000,10) @@ -613,7 +831,7 @@ def test_when_than_else_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select case when cast(_1 as int)>100 and cast(_1 as int)<200 than "(100-200)" when cast(_1 as int)>200 and cast(_1 as int)<300 than "(200-300)" else "NONE" end from s3object;') ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select case when cast(_1 as int)>100 and cast(_1 as int)<200 then "(100-200)" when cast(_1 as int)>200 and cast(_1 as int)<300 then "(200-300)" else "NONE" end from s3object;') ).replace("\n","") count1 = res_s3select.count("(100-200)") @@ -694,3 +912,131 @@ def test_version(): nose.tools.assert_equal( res_version, "41.a," ) +@attr('s3select') +def test_trim_expressions(): + + csv_obj = create_random_csv_object_trim(10000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(_1) == "aeiou";')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1 from 4 for 5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(both from _1) == "aeiou";')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(trailing from _1) == " aeiou";')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(leading from _1) == "aeiou ";')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(trim(leading from _1)) == "aeiou";')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + +@attr('s3select') +def test_truefalse_trim_expressions(): + + csv_obj = create_random_csv_object_trim(10000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(_1) == "aeiou" == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1 from 4 for 5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(both from _1) == "aeiou" == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(trailing from _1) == " aeiou" == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(leading from _1) == "aeiou " == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(trim(leading from _1)) == "aeiou" == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_trim, res_s3select ) + +@attr('s3select') +def test_escape_expressions(): + + csv_obj = create_random_csv_object_escape(10000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_escape = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%_ar" escape "%";')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,char_length(_1),1) == "r" and substring(_1,char_length(_1)-1,1) == "a" and substring(_1,char_length(_1)-2,1) == "_";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_escape, res_s3select ) + + res_s3select_escape = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%aeio$_" escape "$";')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,1,5) == "aeio_";')).replace("\n","") + + nose.tools.assert_equal( res_s3select_escape, res_s3select ) + +@attr('s3select') +def test_case_value_expressions(): + + csv_obj = create_random_csv_object(10000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_case = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select case cast(_1 as int) when cast(_2 as int) then "case_1_1" else "case_2_2" end from s3object;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select case when cast(_1 as int) == cast(_2 as int) then "case_1_1" else "case_2_2" end from s3object;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_case, res_s3select ) + +@attr('s3select') +def test_bool_cast_expressions(): + + csv_obj = create_random_csv_object(10000,10) + + csv_obj_name = get_random_string() + bucket_name = "test" + upload_csv_object(bucket_name,csv_obj_name,csv_obj) + + res_s3select_cast = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where cast(int(_1) as bool) == true ;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where cast(_1 as int) != 0 ;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_cast, res_s3select ) \ No newline at end of file From 60dd3444b3a68c64d845540e1bffe17ab029e718 Mon Sep 17 00:00:00 2001 From: Albin Antony Date: Mon, 22 Mar 2021 16:24:26 +0530 Subject: [PATCH 2/4] tests Signed-off-by: Albin Antony --- s3tests_boto3/functional/test_s3select.py | 132 ++++++++++++++++++---- 1 file changed, 108 insertions(+), 24 deletions(-) diff --git a/s3tests_boto3/functional/test_s3select.py b/s3tests_boto3/functional/test_s3select.py index 3fef0c3..e2f8f91 100644 --- a/s3tests_boto3/functional/test_s3select.py +++ b/s3tests_boto3/functional/test_s3select.py @@ -326,12 +326,24 @@ def test_nullif_expressions(): nose.tools.assert_equal( res_s3select_nullif, res_s3select) + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (nullif(_1,_2) is null) from stdin ;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (_1 == _2) from stdin ;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_nullif, res_s3select) + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where not nullif(_1,_2) is null ;") ).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","") nose.tools.assert_equal( res_s3select_nullif, res_s3select) + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (nullif(_1,_2) is not null) from stdin ;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (_1 != _2) from stdin ;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_nullif, res_s3select) + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where nullif(_1,_2) == _1 ;") ).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","") @@ -348,6 +360,12 @@ def test_nullif_expressions(): nose.tools.assert_equal( res_s3select_nullif, res_s3select) + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (nullif(_1,null) is null) from s3object;") ).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (_1 is null) from s3object;") ).replace("\n","") + + nose.tools.assert_equal( res_s3select_nullif, res_s3select) + @attr('s3select') def test_nulliftrue_expressions(): @@ -429,30 +447,60 @@ def test_in_expressions(): nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) in(1)) from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) == 1) from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) in(1,0);')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1 or int(_1) == 0;')).replace("\n","") nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) in(1,0)) from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) == 1 or int(_1) == 0) from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) in(1,0,2);')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) == 1 or int(_2) == 0 or int(_2) == 2;')).replace("\n","") nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2) in(1,0,2)) from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2) == 1 or int(_2) == 0 or int(_2) == 2) from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5);')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5;')).replace("\n","") nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5)) from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5) from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where character_length(_1) == 2 and substring(_1,2,1) in ("3");')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where _1 like "_3";')).replace("\n","") nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (character_length(_1) == 2 and substring(_1,2,1) in ("3")) from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "_3") from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + @attr('s3select') def test_true_false_in_expressions(): @@ -501,41 +549,77 @@ def test_like_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%aeio%";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%aeio%";')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,11,4) == "aeio" ;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "cbcd%";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "%aeio%") from stdin ;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_1,11,4) == "aeio") from stdin ;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_like, res_s3select ) + + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "cbcd%";')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,1,4) == "cbcd";')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "%y[y-z]";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "cbcd%") from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_1,1,4) == "cbcd") from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_like, res_s3select ) + + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "%y[y-z]";')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_3,char_length(_3),1) between "y" and "z" and substring(_3,char_length(_3)-1,1) == "y";')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%yz";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_3 like "%y[y-z]") from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_3,char_length(_3),1) between "y" and "z" and substring(_3,char_length(_3)-1,1) == "y") from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_like, res_s3select ) + + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%yz";')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_2,char_length(_2),1) == "z" and substring(_2,char_length(_2)-1,1) == "y";')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "c%z";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_2 like "%yz") from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_2,char_length(_2),1) == "z" and substring(_2,char_length(_2)-1,1) == "y") from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_like, res_s3select ) + + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "c%z";')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_3,char_length(_3),1) == "z" and substring(_3,1,1) == "c";')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%xy_";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_3 like "c%z") from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_3,char_length(_3),1) == "z" and substring(_3,1,1) == "c") from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_like, res_s3select ) + + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%xy_";')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_2,char_length(_2)-1,1) == "y" and substring(_2,char_length(_2)-2,1) == "x";')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) + + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_2 like "%xy_") from stdin;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_2,char_length(_2)-1,1) == "y" and substring(_2,char_length(_2)-2,1) == "x") from stdin;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_like, res_s3select ) @attr('s3select') def test_truefalselike_expressions(): @@ -546,41 +630,41 @@ def test_truefalselike_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_1 like "%aeio%") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_1 like "%aeio%") == true;')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,11,4) == "aeio" ;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_1 like "cbcd%") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_1 like "cbcd%") == true;')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,1,4) == "cbcd";')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_3 like "%y[y-z]") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_3 like "%y[y-z]") == true;')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_3,char_length(_3),1) between "y" and "z") == true and (substring(_3,char_length(_3)-1,1) == "y") == true;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_2 like "%yz") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_2 like "%yz") == true;')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_2,char_length(_2),1) == "z") == true and (substring(_2,char_length(_2)-1,1) == "y") == true;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_3 like "c%z") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_3 like "c%z") == true;')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_3,char_length(_3),1) == "z") == true and (substring(_3,1,1) == "c") == true;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_2 like "%xy_") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_2 like "%xy_") == true;')).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_2,char_length(_2)-1,1) == "y") == true and (substring(_2,char_length(_2)-2,1) == "x") == true;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + nose.tools.assert_equal( res_s3select_like, res_s3select ) @attr('s3select') def test_complex_expressions(): From e7102e8cb0b499daceb1d2915591e6c2dd3151a0 Mon Sep 17 00:00:00 2001 From: Albin Antony Date: Mon, 29 Mar 2021 18:56:38 +0530 Subject: [PATCH 3/4] tests Signed-off-by: Albin Antony --- s3tests_boto3/functional/test_s3select.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/s3tests_boto3/functional/test_s3select.py b/s3tests_boto3/functional/test_s3select.py index e2f8f91..ee2eeb9 100644 --- a/s3tests_boto3/functional/test_s3select.py +++ b/s3tests_boto3/functional/test_s3select.py @@ -1,6 +1,7 @@ import nose import random import string +import re from nose.plugins.attrib import attr import uuid @@ -174,7 +175,10 @@ def create_random_csv_object_null(rows,columns,col_delim=",",record_delim="\n",c for _ in range(rows): row = "" for _ in range(columns): + if random.randint(0,5) == 2: row = row + "{}{}".format(''.join("") ,col_delim) + else: + row = row + "{}{}".format(''.join("abc") ,col_delim) result += row + record_delim @@ -219,6 +223,9 @@ def remove_xml_tags_from_result(obj): continue result += rec + "\n" # remove by split + result_strip= result.strip() + x = bool(re.search("^failure.*$", result_strip)) + nose.tools.assert_equal(x, False) return result def create_list_of_int(column_pos,obj,field_split=",",row_split="\n"): @@ -540,6 +547,12 @@ def test_true_false_in_expressions(): nose.tools.assert_equal( res_s3select_in, res_s3select ) + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) in (1,2,0)) as a1 from s3object where a1 == true;')).replace("\n","") + + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select \"true\"from stdin where (int(_1) in (1,0,2)) ;')).replace("\n","") + + nose.tools.assert_equal( res_s3select_in, res_s3select ) + @attr('s3select') def test_like_expressions(): @@ -688,11 +701,11 @@ def test_complex_expressions(): nose.tools.assert_equal( res_s3select, __res ) # purpose of test that all where conditions create the same group of values, thus same result - res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where substring(_2,1,1) == "1"')).replace("\n","") + res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where substring(_2,1,1) == "1" and char_length(_2) == 3;')).replace("\n","") - res_s3select_between_numbers = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where int(_2)>=100 and int(_2)<200')).replace("\n","") + res_s3select_between_numbers = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where int(_2)>=100 and int(_2)<200;')).replace("\n","") - res_s3select_eq_modolu = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where int(_2)/100 == 1 or int(_2)/10 == 1 or int(_2) == 1')).replace("\n","") + res_s3select_eq_modolu = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where int(_2)/100 == 1 and character_length(_2) == 3;')).replace("\n","") nose.tools.assert_equal( res_s3select_substring, res_s3select_between_numbers) From c6a4ab9d125c074b5aa3ac79180fc93e4cfae6a0 Mon Sep 17 00:00:00 2001 From: Albin Antony Date: Wed, 31 Mar 2021 16:30:30 +0530 Subject: [PATCH 4/4] tests Signed-off-by: Albin Antony --- s3tests_boto3/functional/test_s3select.py | 540 +++++++++++----------- 1 file changed, 281 insertions(+), 259 deletions(-) diff --git a/s3tests_boto3/functional/test_s3select.py b/s3tests_boto3/functional/test_s3select.py index ee2eeb9..9a3a20f 100644 --- a/s3tests_boto3/functional/test_s3select.py +++ b/s3tests_boto3/functional/test_s3select.py @@ -11,6 +11,9 @@ from . import ( get_client ) +import logging +logging.basicConfig(level=logging.INFO) + region_name = '' # recurssion function for generating arithmetical expression @@ -36,11 +39,11 @@ def generate_s3select_where_clause(bucket_name,obj_name): # generate s3select statement using generated randome expression # upon count(0)>0 it means true for the where clause expression # the python-engine {eval( conditional expression )} should return same boolean result. - s3select_stmt = "select count(0) from stdin where " + a + s + b + ";" + s3select_stmt = "select count(0) from s3object where " + a + s + b + ";" res = remove_xml_tags_from_result( run_s3select(bucket_name,obj_name,s3select_stmt) ).replace(",","") - nose.tools.assert_equal(int(res)>0 , eval( a + s + b )) + s3select_assert_result(int(res)>0 , eval( a + s + b )) def generate_s3select_expression_projection(bucket_name,obj_name): @@ -58,7 +61,7 @@ def generate_s3select_expression_projection(bucket_name,obj_name): if eval( e ) == 0: return - res = remove_xml_tags_from_result( run_s3select(bucket_name,obj_name,"select " + e + " from stdin;",) ).replace(",","") + res = remove_xml_tags_from_result( run_s3select(bucket_name,obj_name,"select " + e + " from s3object;",) ).replace(",","") # accuracy level epsilon = float(0.000001) @@ -94,7 +97,18 @@ def test_generate_projection(): for _ in range(100): generate_s3select_expression_projection(bucket_name,obj_name) + +def s3select_assert_result(a,b): + if type(a) == str: + a_strip = a.strip() + b_strip = b.strip() + assert a_strip != "" + assert b_strip != "" + else: + assert a != "" + assert b != "" + nose.tools.assert_equal(a,b) def create_csv_object_for_datetime(rows,columns): result = "" @@ -144,7 +158,12 @@ def create_random_csv_object_trim(rows,columns,col_delim=",",record_delim="\n",c for _ in range(rows): row = "" for _ in range(columns): + if random.randint(0,5) == 2: row = row + "{}{}".format(''.join(" aeiou ") ,col_delim) + else: + row = row + "{}{}".format(''.join("abcd") ,col_delim) + + result += row + record_delim @@ -225,7 +244,10 @@ def remove_xml_tags_from_result(obj): result_strip= result.strip() x = bool(re.search("^failure.*$", result_strip)) + if x: + logging.info(result) nose.tools.assert_equal(x, False) + return result def create_list_of_int(column_pos,obj,field_split=",",row_split="\n"): @@ -249,9 +271,9 @@ def test_count_operation(): num_of_rows = 1234 obj_to_load = create_random_csv_object(num_of_rows,10) upload_csv_object(bucket_name,csv_obj_name,obj_to_load) - res = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin;") ).replace(",","") + res = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object;") ).replace(",","") - nose.tools.assert_equal( num_of_rows, int( res )) + s3select_assert_result( num_of_rows, int( res )) @attr('s3select') def test_column_sum_min_max(): @@ -265,58 +287,58 @@ def test_column_sum_min_max(): bucket_name_2 = "testbuck2" upload_csv_object(bucket_name_2,csv_obj_name_2,csv_obj) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_1)) from stdin;") ).replace(",","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_1)) from s3object;") ).replace(",","") list_int = create_list_of_int( 1 , csv_obj ) res_target = min( list_int ) - nose.tools.assert_equal( int(res_s3select), int(res_target)) + s3select_assert_result( int(res_s3select), int(res_target)) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_4)) from stdin;") ).replace(",","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_4)) from s3object;") ).replace(",","") list_int = create_list_of_int( 4 , csv_obj ) res_target = min( list_int ) - nose.tools.assert_equal( int(res_s3select), int(res_target)) + s3select_assert_result( int(res_s3select), int(res_target)) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select avg(int(_6)) from stdin;") ).replace(",","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select avg(int(_6)) from s3object;") ).replace(",","") list_int = create_list_of_int( 6 , csv_obj ) res_target = float(sum(list_int ))/10000 - nose.tools.assert_equal( float(res_s3select), float(res_target)) + s3select_assert_result( float(res_s3select), float(res_target)) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select max(int(_4)) from stdin;") ).replace(",","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select max(int(_4)) from s3object;") ).replace(",","") list_int = create_list_of_int( 4 , csv_obj ) res_target = max( list_int ) - nose.tools.assert_equal( int(res_s3select), int(res_target)) + s3select_assert_result( int(res_s3select), int(res_target)) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select max(int(_7)) from stdin;") ).replace(",","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select max(int(_7)) from s3object;") ).replace(",","") list_int = create_list_of_int( 7 , csv_obj ) res_target = max( list_int ) - nose.tools.assert_equal( int(res_s3select), int(res_target)) + s3select_assert_result( int(res_s3select), int(res_target)) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select sum(int(_4)) from stdin;") ).replace(",","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select sum(int(_4)) from s3object;") ).replace(",","") list_int = create_list_of_int( 4 , csv_obj ) res_target = sum( list_int ) - nose.tools.assert_equal( int(res_s3select), int(res_target)) + s3select_assert_result( int(res_s3select), int(res_target)) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select sum(int(_7)) from stdin;") ).replace(",","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select sum(int(_7)) from s3object;") ).replace(",","") list_int = create_list_of_int( 7 , csv_obj ) res_target = sum( list_int ) - nose.tools.assert_equal( int(res_s3select) , int(res_target) ) + s3select_assert_result( int(res_s3select) , int(res_target) ) # the following queries, validates on *random* input an *accurate* relation between condition result,sum operation and count operation. - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name_2,csv_obj_name_2,"select count(0),sum(int(_1)),sum(int(_2)) from stdin where (int(_1)-int(_2)) == 2;" ) ) + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name_2,csv_obj_name_2,"select count(0),sum(int(_1)),sum(int(_2)) from s3object where (int(_1)-int(_2)) == 2;" ) ) count,sum1,sum2,d = res_s3select.split(",") - nose.tools.assert_equal( int(count)*2 , int(sum1)-int(sum2 ) ) + s3select_assert_result( int(count)*2 , int(sum1)-int(sum2 ) ) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0),sum(int(_1)),sum(int(_2)) from stdin where (int(_1)-int(_2)) == 4;" ) ) + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0),sum(int(_1)),sum(int(_2)) from s3object where (int(_1)-int(_2)) == 4;" ) ) count,sum1,sum2,d = res_s3select.split(",") - nose.tools.assert_equal( int(count)*4 , int(sum1)-int(sum2) ) + s3select_assert_result( int(count)*4 , int(sum1)-int(sum2) ) @attr('s3select') def test_nullif_expressions(): @@ -327,35 +349,35 @@ def test_nullif_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where nullif(_1,_2) is null ;") ).replace("\n","") + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where nullif(_1,_2) is null ;") ).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 == _2 ;") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where _1 == _2 ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) - res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (nullif(_1,_2) is null) from stdin ;") ).replace("\n","") + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (nullif(_1,_2) is null) from s3object ;") ).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (_1 == _2) from stdin ;") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (_1 == _2) from s3object ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) - res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where not nullif(_1,_2) is null ;") ).replace("\n","") + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where not nullif(_1,_2) is null ;") ).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where _1 != _2 ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) - res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (nullif(_1,_2) is not null) from stdin ;") ).replace("\n","") + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (nullif(_1,_2) is not null) from s3object ;") ).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (_1 != _2) from stdin ;") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (_1 != _2) from s3object ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) - res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where nullif(_1,_2) == _1 ;") ).replace("\n","") + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where nullif(_1,_2) == _1 ;") ).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where _1 != _2 ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) csv_obj = create_random_csv_object_null(10000,10) @@ -365,13 +387,13 @@ def test_nullif_expressions(): res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where _1 is null;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (nullif(_1,null) is null) from s3object;") ).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select (_1 is null) from s3object;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) @attr('s3select') def test_nulliftrue_expressions(): @@ -382,23 +404,23 @@ def test_nulliftrue_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where (nullif(_1,_2) is null) == true ;") ).replace("\n","") + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where (nullif(_1,_2) is null) == true ;") ).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 == _2 ;") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where _1 == _2 ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) - res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where not (nullif(_1,_2) is null) == true ;") ).replace("\n","") + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where not (nullif(_1,_2) is null) == true ;") ).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where _1 != _2 ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) - res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where (nullif(_1,_2) == _1 == true) ;") ).replace("\n","") + res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where (nullif(_1,_2) == _1 == true) ;") ).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where _1 != _2 ;") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where _1 != _2 ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_nullif, res_s3select) + s3select_assert_result( res_s3select_nullif, res_s3select) @attr('s3select') def test_is_not_null_expressions(): @@ -413,13 +435,13 @@ def test_is_not_null_expressions(): res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where _1 != _2 ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_null, res_s3select) + s3select_assert_result( res_s3select_null, res_s3select) res_s3select_null = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where (nullif(_1,_1) and _1 == _2) is not null ;") ).replace("\n","") res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where _1 != _2 ;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_null, res_s3select) + s3select_assert_result( res_s3select_null, res_s3select) @attr('s3select') def test_lowerupper_expressions(): @@ -430,13 +452,13 @@ def test_lowerupper_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select lower("AB12cd$$") from stdin ;') ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select lower("AB12cd$$") from s3object ;') ).replace("\n","") - nose.tools.assert_equal( res_s3select, "ab12cd$$,") + s3select_assert_result( res_s3select, "ab12cd$$,") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select upper("ab12CD$$") from stdin ;') ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select upper("ab12CD$$") from s3object ;') ).replace("\n","") - nose.tools.assert_equal( res_s3select, "AB12CD$$,") + s3select_assert_result( res_s3select, "AB12CD$$,") @attr('s3select') def test_in_expressions(): @@ -448,65 +470,65 @@ def test_in_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) in(1);')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) in(1);')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) == 1;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) in(1)) from stdin;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) in(1)) from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) == 1) from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) == 1) from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) in(1,0);')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) in(1,0);')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1 or int(_1) == 0;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) == 1 or int(_1) == 0;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) in(1,0)) from stdin;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) in(1,0)) from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) == 1 or int(_1) == 0) from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) == 1 or int(_1) == 0) from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) in(1,0,2);')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from s3object where int(_2) in(1,0,2);')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) == 1 or int(_2) == 0 or int(_2) == 2;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from s3object where int(_2) == 1 or int(_2) == 0 or int(_2) == 2;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2) in(1,0,2)) from stdin;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2) in(1,0,2)) from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2) == 1 or int(_2) == 0 or int(_2) == 2) from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2) == 1 or int(_2) == 0 or int(_2) == 2) from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5);')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from s3object where int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5);')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from s3object where int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5)) from stdin;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5)) from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5) from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5) from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where character_length(_1) == 2 and substring(_1,2,1) in ("3");')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where character_length(_1) == 2 and substring(_1,2,1) in ("3");')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where _1 like "_3";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where _1 like "_3";')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (character_length(_1) == 2 and substring(_1,2,1) in ("3")) from stdin;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (character_length(_1) == 2 and substring(_1,2,1) in ("3")) from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "_3") from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "_3") from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) @attr('s3select') def test_true_false_in_expressions(): @@ -517,41 +539,41 @@ def test_true_false_in_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where (int(_1) in(1)) == true;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where (int(_1) in(1)) == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) == 1;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where (int(_1) in(1,0)) == true;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where (int(_1) in(1,0)) == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) == 1 or int(_1) == 0;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) == 1 or int(_1) == 0;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where (int(_2) in(1,0,2)) == true;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from s3object where (int(_2) in(1,0,2)) == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2) == 1 or int(_2) == 0 or int(_2) == 2;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from s3object where int(_2) == 1 or int(_2) == 0 or int(_2) == 2;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where (int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5)) == true;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from s3object where (int(_2)*2 in(int(_3)*2,int(_4)*3,int(_5)*5)) == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from stdin where int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_2) from s3object where int(_2)*2 == int(_3)*2 or int(_2)*2 == int(_4)*3 or int(_2)*2 == int(_5)*5;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) - res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where (character_length(_1) == 2) == true and (substring(_1,2,1) in ("3")) == true;')).replace("\n","") + res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where (character_length(_1) == 2) == true and (substring(_1,2,1) in ("3")) == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where _1 like "_3";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where _1 like "_3";')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (int(_1) in (1,2,0)) as a1 from s3object where a1 == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select \"true\"from stdin where (int(_1) in (1,0,2)) ;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select \"true\"from s3object where (int(_1) in (1,0,2)) ;')).replace("\n","") - nose.tools.assert_equal( res_s3select_in, res_s3select ) + s3select_assert_result( res_s3select_in, res_s3select ) @attr('s3select') def test_like_expressions(): @@ -562,77 +584,77 @@ def test_like_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%aeio%";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _1 like "%aeio%";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,11,4) == "aeio" ;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,11,4) == "aeio" ;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "%aeio%") from stdin ;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "%aeio%") from s3object ;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_1,11,4) == "aeio") from stdin ;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_1,11,4) == "aeio") from s3object ;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "cbcd%";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _1 like "cbcd%";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,1,4) == "cbcd";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,1,4) == "cbcd";')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "cbcd%") from stdin;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_1 like "cbcd%") from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_1,1,4) == "cbcd") from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_1,1,4) == "cbcd") from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "%y[y-z]";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _3 like "%y[y-z]";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_3,char_length(_3),1) between "y" and "z" and substring(_3,char_length(_3)-1,1) == "y";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_3,char_length(_3),1) between "y" and "z" and substring(_3,char_length(_3)-1,1) == "y";')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_3 like "%y[y-z]") from stdin;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_3 like "%y[y-z]") from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_3,char_length(_3),1) between "y" and "z" and substring(_3,char_length(_3)-1,1) == "y") from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_3,char_length(_3),1) between "y" and "z" and substring(_3,char_length(_3)-1,1) == "y") from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%yz";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _2 like "%yz";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_2,char_length(_2),1) == "z" and substring(_2,char_length(_2)-1,1) == "y";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_2,char_length(_2),1) == "z" and substring(_2,char_length(_2)-1,1) == "y";')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_2 like "%yz") from stdin;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_2 like "%yz") from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_2,char_length(_2),1) == "z" and substring(_2,char_length(_2)-1,1) == "y") from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_2,char_length(_2),1) == "z" and substring(_2,char_length(_2)-1,1) == "y") from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _3 like "c%z";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _3 like "c%z";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_3,char_length(_3),1) == "z" and substring(_3,1,1) == "c";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_3,char_length(_3),1) == "z" and substring(_3,1,1) == "c";')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_3 like "c%z") from stdin;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_3 like "c%z") from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_3,char_length(_3),1) == "z" and substring(_3,1,1) == "c") from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_3,char_length(_3),1) == "z" and substring(_3,1,1) == "c") from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _2 like "%xy_";')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _2 like "%xy_";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_2,char_length(_2)-1,1) == "y" and substring(_2,char_length(_2)-2,1) == "x";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_2,char_length(_2)-1,1) == "y" and substring(_2,char_length(_2)-2,1) == "x";')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_2 like "%xy_") from stdin;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select (_2 like "%xy_") from s3object;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_2,char_length(_2)-1,1) == "y" and substring(_2,char_length(_2)-2,1) == "x") from stdin;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select (substring(_2,char_length(_2)-1,1) == "y" and substring(_2,char_length(_2)-2,1) == "x") from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) @attr('s3select') def test_truefalselike_expressions(): @@ -643,41 +665,41 @@ def test_truefalselike_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_1 like "%aeio%") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where (_1 like "%aeio%") == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,11,4) == "aeio" ;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,11,4) == "aeio" ;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_1 like "cbcd%") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where (_1 like "cbcd%") == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,1,4) == "cbcd";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,1,4) == "cbcd";')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_3 like "%y[y-z]") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where (_3 like "%y[y-z]") == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_3,char_length(_3),1) between "y" and "z") == true and (substring(_3,char_length(_3)-1,1) == "y") == true;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where (substring(_3,char_length(_3),1) between "y" and "z") == true and (substring(_3,char_length(_3)-1,1) == "y") == true;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_2 like "%yz") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where (_2 like "%yz") == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_2,char_length(_2),1) == "z") == true and (substring(_2,char_length(_2)-1,1) == "y") == true;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where (substring(_2,char_length(_2),1) == "z") == true and (substring(_2,char_length(_2)-1,1) == "y") == true;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_3 like "c%z") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where (_3 like "c%z") == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_3,char_length(_3),1) == "z") == true and (substring(_3,1,1) == "c") == true;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where (substring(_3,char_length(_3),1) == "z") == true and (substring(_3,1,1) == "c") == true;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) - res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where (_2 like "%xy_") == true;')).replace("\n","") + res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where (_2 like "%xy_") == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where (substring(_2,char_length(_2)-1,1) == "y") == true and (substring(_2,char_length(_2)-2,1) == "x") == true;')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where (substring(_2,char_length(_2)-1,1) == "y") == true and (substring(_2,char_length(_2)-2,1) == "x") == true;')).replace("\n","") - nose.tools.assert_equal( res_s3select_like, res_s3select ) + s3select_assert_result( res_s3select_like, res_s3select ) @attr('s3select') def test_complex_expressions(): @@ -689,7 +711,7 @@ def test_complex_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_1)),max(int(_2)),min(int(_3))+1 from stdin;")).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_1)),max(int(_2)),min(int(_3))+1 from s3object;")).replace("\n","") min_1 = min ( create_list_of_int( 1 , csv_obj ) ) max_2 = max ( create_list_of_int( 2 , csv_obj ) ) @@ -698,18 +720,18 @@ def test_complex_expressions(): __res = "{},{},{},".format(min_1,max_2,min_3) # assert is according to radom-csv function - nose.tools.assert_equal( res_s3select, __res ) + s3select_assert_result( res_s3select, __res ) # purpose of test that all where conditions create the same group of values, thus same result - res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where substring(_2,1,1) == "1" and char_length(_2) == 3;')).replace("\n","") + res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from s3object where substring(_2,1,1) == "1" and char_length(_2) == 3;')).replace("\n","") - res_s3select_between_numbers = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where int(_2)>=100 and int(_2)<200;')).replace("\n","") + res_s3select_between_numbers = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from s3object where int(_2)>=100 and int(_2)<200;')).replace("\n","") - res_s3select_eq_modolu = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from stdin where int(_2)/100 == 1 and character_length(_2) == 3;')).replace("\n","") + res_s3select_eq_modolu = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select min(int(_2)),max(int(_2)) from s3object where int(_2)/100 == 1 and character_length(_2) == 3;')).replace("\n","") - nose.tools.assert_equal( res_s3select_substring, res_s3select_between_numbers) + s3select_assert_result( res_s3select_substring, res_s3select_between_numbers) - nose.tools.assert_equal( res_s3select_between_numbers, res_s3select_eq_modolu) + s3select_assert_result( res_s3select_between_numbers, res_s3select_eq_modolu) @attr('s3select') def test_alias(): @@ -725,11 +747,11 @@ def test_alias(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3 from stdin where a3>100 and a3<300;") ).replace(",","") + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3 from s3object where a3>100 and a3<300;") ).replace(",","") - res_s3select_no_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(_1),int(_2),int(_1)+int(_2) from stdin where (int(_1)+int(_2))>100 and (int(_1)+int(_2))<300;") ).replace(",","") + res_s3select_no_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(_1),int(_2),int(_1)+int(_2) from s3object where (int(_1)+int(_2))>100 and (int(_1)+int(_2))<300;") ).replace(",","") - nose.tools.assert_equal( res_s3select_alias, res_s3select_no_alias) + s3select_assert_result( res_s3select_alias, res_s3select_no_alias) @attr('s3select') @@ -744,7 +766,7 @@ def test_alias_cyclic_refernce(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(_1) as a1,int(_2) as a2, a1+a4 as a3, a5+a1 as a4, int(_3)+a3 as a5 from stdin;") ) + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(_1) as a1,int(_2) as a2, a1+a4 as a3, a5+a1 as a4, int(_3)+a3 as a5 from s3object;") ) find_res = res_s3select_alias.find("number of calls exceed maximum size, probably a cyclic reference to alias") @@ -763,26 +785,26 @@ def test_datetime(): upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where extract(year from to_timestamp(_1)) > 1950 and extract(year from to_timestamp(_1)) < 1960;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where extract(year from to_timestamp(_1)) > 1950 and extract(year from to_timestamp(_1)) < 1960;') ) - res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where int(substring(_1,1,4))>1950 and int(substring(_1,1,4))<1960;') ) + res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where int(substring(_1,1,4))>1950 and int(substring(_1,1,4))<1960;') ) - nose.tools.assert_equal( res_s3select_date_time, res_s3select_substring) + s3select_assert_result( res_s3select_date_time, res_s3select_substring) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where date_diff(month,to_timestamp(_1),date_add(month,2,to_timestamp(_1)) ) == 2;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where date_diff(month,to_timestamp(_1),date_add(month,2,to_timestamp(_1)) ) == 2;') ) - res_s3select_count = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin;') ) + res_s3select_count = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object;') ) - nose.tools.assert_equal( res_s3select_date_time, res_s3select_count) + s3select_assert_result( res_s3select_date_time, res_s3select_count) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where date_diff(year,to_timestamp(_1),date_add(day, 366 ,to_timestamp(_1))) == 1 ;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where date_diff(year,to_timestamp(_1),date_add(day, 366 ,to_timestamp(_1))) == 1 ;') ) - nose.tools.assert_equal( res_s3select_date_time, res_s3select_count) + s3select_assert_result( res_s3select_date_time, res_s3select_count) # validate that utcnow is integrate correctly with other date-time functions - res_s3select_date_time_utcnow = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where date_diff(hour,utcnow(),date_add(day,1,utcnow())) == 24 ;') ) + res_s3select_date_time_utcnow = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where date_diff(hour,utcnow(),date_add(day,1,utcnow())) == 24 ;') ) - nose.tools.assert_equal( res_s3select_date_time_utcnow, res_s3select_count) + s3select_assert_result( res_s3select_date_time_utcnow, res_s3select_count) @attr('s3select') def test_true_false_datetime(): @@ -797,25 +819,25 @@ def test_true_false_datetime(): upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where (extract(year from to_timestamp(_1)) > 1950) == true and (extract(year from to_timestamp(_1)) < 1960) == true;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where (extract(year from to_timestamp(_1)) > 1950) == true and (extract(year from to_timestamp(_1)) < 1960) == true;') ) - res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where int(substring(_1,1,4))>1950 and int(substring(_1,1,4))<1960;') ) + res_s3select_substring = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where int(substring(_1,1,4))>1950 and int(substring(_1,1,4))<1960;') ) - nose.tools.assert_equal( res_s3select_date_time, res_s3select_substring) + s3select_assert_result( res_s3select_date_time, res_s3select_substring) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where (date_diff(month,to_timestamp(_1),date_add(month,2,to_timestamp(_1)) ) == 2) == true;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where (date_diff(month,to_timestamp(_1),date_add(month,2,to_timestamp(_1)) ) == 2) == true;') ) - res_s3select_count = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin;') ) + res_s3select_count = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object;') ) - nose.tools.assert_equal( res_s3select_date_time, res_s3select_count) + s3select_assert_result( res_s3select_date_time, res_s3select_count) - res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where (date_diff(year,to_timestamp(_1),date_add(day, 366 ,to_timestamp(_1))) == 1) == true ;') ) + res_s3select_date_time = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where (date_diff(year,to_timestamp(_1),date_add(day, 366 ,to_timestamp(_1))) == 1) == true ;') ) - nose.tools.assert_equal( res_s3select_date_time, res_s3select_count) + s3select_assert_result( res_s3select_date_time, res_s3select_count) - res_s3select_date_time_utcnow = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from stdin where (date_diff(hour,utcnow(),date_add(day,1,utcnow())) == 24) == true ;') ) + res_s3select_date_time_utcnow = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(0) from s3object where (date_diff(hour,utcnow(),date_add(day,1,utcnow())) == 24) == true ;') ) - nose.tools.assert_equal( res_s3select_date_time_utcnow, res_s3select_count) + s3select_assert_result( res_s3select_date_time_utcnow, res_s3select_count) @attr('s3select') def test_csv_parser(): @@ -830,32 +852,32 @@ def test_csv_parser(): upload_csv_object(bucket_name,csv_obj_name,csv_obj) # return value contain comma{,} - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _6 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, 'third="c31,c32,c33",') + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _6 from s3object;") ).replace("\n","") + s3select_assert_result( res_s3select_alias, 'third="c31,c32,c33",') # return value contain comma{,} - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _7 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, 'forth="1,2,3,4",') + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _7 from s3object;") ).replace("\n","") + s3select_assert_result( res_s3select_alias, 'forth="1,2,3,4",') # return value contain comma{,}{"}, escape-rule{\} by-pass quote{"} , the escape{\} is removed. - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _8 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, 'fifth="my_string="any_value" , my_other_string="aaaa,bbb" ",') + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _8 from s3object;") ).replace("\n","") + s3select_assert_result( res_s3select_alias, 'fifth="my_string="any_value" , my_other_string="aaaa,bbb" ",') # return NULL as first token - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _1 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, 'null,') + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _1 from s3object;") ).replace("\n","") + s3select_assert_result( res_s3select_alias, 'null,') # return NULL in the middle of line - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _3 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, 'null,') + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _3 from s3object;") ).replace("\n","") + s3select_assert_result( res_s3select_alias, 'null,') # return NULL in the middle of line (successive) - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _4 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, 'null,') + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _4 from s3object;") ).replace("\n","") + s3select_assert_result( res_s3select_alias, 'null,') # return NULL at the end line - res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _9 from stdin;") ).replace("\n","") - nose.tools.assert_equal( res_s3select_alias, 'null,') + res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _9 from s3object;") ).replace("\n","") + s3select_assert_result( res_s3select_alias, 'null,') @attr('s3select') def test_csv_definition(): @@ -871,20 +893,20 @@ def test_csv_definition(): upload_csv_object(bucket_name,csv_obj_name,csv_obj) # purpose of tests is to parse correctly input with different csv defintions - res = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin;","|","\t") ).replace(",","") + res = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object;","|","\t") ).replace(",","") - nose.tools.assert_equal( number_of_rows, int(res)) + s3select_assert_result( number_of_rows, int(res)) # assert is according to radom-csv function # purpose of test is validate that tokens are processed correctly - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_1)),max(int(_2)),min(int(_3))+1 from stdin;","|","\t") ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_1)),max(int(_2)),min(int(_3))+1 from s3object;","|","\t") ).replace("\n","") min_1 = min ( create_list_of_int( 1 , csv_obj , "|","\t") ) max_2 = max ( create_list_of_int( 2 , csv_obj , "|","\t") ) min_3 = min ( create_list_of_int( 3 , csv_obj , "|","\t") ) + 1 __res = "{},{},{},".format(min_1,max_2,min_3) - nose.tools.assert_equal( res_s3select, __res ) + s3select_assert_result( res_s3select, __res ) @attr('s3select') @@ -901,21 +923,21 @@ def test_schema_definition(): upload_csv_object(bucket_name,csv_obj_name,csv_obj) # ignoring the schema on first line and retrieve using generic column number - res_ignore = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _1,_3 from stdin;",csv_header_info="IGNORE") ).replace("\n","") + res_ignore = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select _1,_3 from s3object;",csv_header_info="IGNORE") ).replace("\n","") # using the scheme on first line, query is using the attach schema - res_use = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select c1,c3 from stdin;",csv_header_info="USE") ).replace("\n","") + res_use = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select c1,c3 from s3object;",csv_header_info="USE") ).replace("\n","") # result of both queries should be the same - nose.tools.assert_equal( res_ignore, res_use) + s3select_assert_result( res_ignore, res_use) # using column-name not exist in schema - res_multiple_defintion = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select c1,c10,int(c11) from stdin;",csv_header_info="USE") ).replace("\n","") + res_multiple_defintion = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select c1,c10,int(c11) from s3object;",csv_header_info="USE") ).replace("\n","") assert res_multiple_defintion.find("alias {c11} or column not exist in schema") > 0 # alias-name is identical to column-name - res_multiple_defintion = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(c1)+int(c2) as c4,c4 from stdin;",csv_header_info="USE") ).replace("\n","") + res_multiple_defintion = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(c1)+int(c2) as c4,c4 from s3object;",csv_header_info="USE") ).replace("\n","") assert res_multiple_defintion.find("multiple definition of column {c4} as schema-column and alias") > 0 @@ -942,11 +964,11 @@ def test_when_then_else_expressions(): res2 = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where cast(_1 as int)<=100 or cast(_1 as int)>=300 or cast(_1 as int)==200 ;') ).replace("\n","") - nose.tools.assert_equal( str(count1) + ',', res) + s3select_assert_result( str(count1) + ',', res) - nose.tools.assert_equal( str(count2) + ',', res1) + s3select_assert_result( str(count2) + ',', res1) - nose.tools.assert_equal( str(count3) + ',', res2) + s3select_assert_result( str(count3) + ',', res2) @attr('s3select') def test_coalesce_expressions(): @@ -961,13 +983,13 @@ def test_coalesce_expressions(): res_null = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where cast(_3 as int)>99 and cast(_4 as int)>99 and coalesce(nullif(cast(substring(_3,1,2) as int),cast(substring(_4,1,2) as int)),7) == 7;' ) ).replace("\n","") - nose.tools.assert_equal( res_s3select, res_null) + s3select_assert_result( res_s3select, res_null) - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select coalesce(nullif(_5,_5),nullif(_1,_1),_2) from stdin;') ).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select coalesce(nullif(_5,_5),nullif(_1,_1),_2) from s3object;') ).replace("\n","") - res_coalesce = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select coalesce(_2) from stdin;') ).replace("\n","") + res_coalesce = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select coalesce(_2) from s3object;') ).replace("\n","") - nose.tools.assert_equal( res_s3select, res_coalesce) + s3select_assert_result( res_s3select, res_coalesce) @attr('s3select') @@ -983,13 +1005,13 @@ def test_cast_expressions(): res = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where char_length(_3)>3;') ).replace("\n","") - nose.tools.assert_equal( res_s3select, res) + s3select_assert_result( res_s3select, res) res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where cast(_3 as int)>99 and cast(_3 as int)<1000;') ).replace("\n","") res = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where char_length(_3)==3;') ).replace("\n","") - nose.tools.assert_equal( res_s3select, res) + s3select_assert_result( res_s3select, res) @attr('s3select') def test_version(): @@ -1005,9 +1027,9 @@ def test_version(): upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_version = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select version() from stdin;") ).replace("\n","") + res_version = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select version() from s3object;") ).replace("\n","") - nose.tools.assert_equal( res_version, "41.a," ) + s3select_assert_result( res_version, "41.a," ) @attr('s3select') def test_trim_expressions(): @@ -1018,35 +1040,35 @@ def test_trim_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(_1) == "aeiou";')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(_1) == "aeiou";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1 from 4 for 5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1 from 4 for 5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(both from _1) == "aeiou";')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(both from _1) == "aeiou";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,4,5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(trailing from _1) == " aeiou";')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(trailing from _1) == " aeiou";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,4,5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(leading from _1) == "aeiou ";')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(leading from _1) == "aeiou ";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,4,5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(trim(leading from _1)) == "aeiou";')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(trim(leading from _1)) == "aeiou";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,4,5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) @attr('s3select') def test_truefalse_trim_expressions(): @@ -1057,35 +1079,35 @@ def test_truefalse_trim_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(_1) == "aeiou" == true;')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(_1) == "aeiou" == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1 from 4 for 5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1 from 4 for 5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(both from _1) == "aeiou" == true;')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(both from _1) == "aeiou" == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,4,5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(trailing from _1) == " aeiou" == true;')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(trailing from _1) == " aeiou" == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,4,5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(leading from _1) == "aeiou " == true;')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(leading from _1) == "aeiou " == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,4,5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) - res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where trim(trim(leading from _1)) == "aeiou" == true;')).replace("\n","") + res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(trim(leading from _1)) == "aeiou" == true;')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,4,5) == "aeiou";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,4,5) == "aeiou";')).replace("\n","") - nose.tools.assert_equal( res_s3select_trim, res_s3select ) + s3select_assert_result( res_s3select_trim, res_s3select ) @attr('s3select') def test_escape_expressions(): @@ -1096,17 +1118,17 @@ def test_escape_expressions(): bucket_name = "test" upload_csv_object(bucket_name,csv_obj_name,csv_obj) - res_s3select_escape = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%_ar" escape "%";')).replace("\n","") + res_s3select_escape = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _1 like "%_ar" escape "%";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,char_length(_1),1) == "r" and substring(_1,char_length(_1)-1,1) == "a" and substring(_1,char_length(_1)-2,1) == "_";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,char_length(_1),1) == "r" and substring(_1,char_length(_1)-1,1) == "a" and substring(_1,char_length(_1)-2,1) == "_";')).replace("\n","") - nose.tools.assert_equal( res_s3select_escape, res_s3select ) + s3select_assert_result( res_s3select_escape, res_s3select ) - res_s3select_escape = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%aeio$_" escape "$";')).replace("\n","") + res_s3select_escape = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _1 like "%aeio$_" escape "$";')).replace("\n","") - res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from stdin where substring(_1,1,5) == "aeio_";')).replace("\n","") + res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where substring(_1,1,5) == "aeio_";')).replace("\n","") - nose.tools.assert_equal( res_s3select_escape, res_s3select ) + s3select_assert_result( res_s3select_escape, res_s3select ) @attr('s3select') def test_case_value_expressions(): @@ -1121,7 +1143,7 @@ def test_case_value_expressions(): res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select case when cast(_1 as int) == cast(_2 as int) then "case_1_1" else "case_2_2" end from s3object;')).replace("\n","") - nose.tools.assert_equal( res_s3select_case, res_s3select ) + s3select_assert_result( res_s3select_case, res_s3select ) @attr('s3select') def test_bool_cast_expressions(): @@ -1136,4 +1158,4 @@ def test_bool_cast_expressions(): res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name, 'select count(*) from s3object where cast(_1 as int) != 0 ;')).replace("\n","") - nose.tools.assert_equal( res_s3select_cast, res_s3select ) \ No newline at end of file + s3select_assert_result( res_s3select_cast, res_s3select ) \ No newline at end of file