diff --git a/contrib/pax_storage/src/api/python3/setup-debug.py b/contrib/pax_storage/src/api/python3/setup-debug.py index 13eadaa357d..da02e2ab237 100644 --- a/contrib/pax_storage/src/api/python3/setup-debug.py +++ b/contrib/pax_storage/src/api/python3/setup-debug.py @@ -51,7 +51,7 @@ def abs_path(file_name): version = '1.0', description = 'PAXPY is the PYTHON3 API of PAX', author = 'jiaqizho', - author_email = 'jiaqizho@hashdata.cn', + author_email = 'jiaqizho@apache.org', url = '-', ext_modules = [paxpy_module] -) \ No newline at end of file +) diff --git a/contrib/pax_storage/src/api/python3/setup.py b/contrib/pax_storage/src/api/python3/setup.py index a60cc779fd9..b739cacc465 100644 --- a/contrib/pax_storage/src/api/python3/setup.py +++ b/contrib/pax_storage/src/api/python3/setup.py @@ -48,7 +48,7 @@ def abs_path(file_name): version = '1.0', description = 'PAXPY is the PYTHON3 API of PAX', author = 'jiaqizho', - author_email = 'jiaqizho@hashdata.cn', + author_email = 'jiaqizho@apache.org', url = '-', ext_modules = [paxpy_module] -) \ No newline at end of file +) diff --git a/contrib/pax_storage/src/cpp/catalog/manifest_api.h b/contrib/pax_storage/src/cpp/catalog/manifest_api.h index f9e8f7c9eb1..3978caf9097 100644 --- a/contrib/pax_storage/src/cpp/catalog/manifest_api.h +++ b/contrib/pax_storage/src/cpp/catalog/manifest_api.h @@ -28,7 +28,7 @@ #pragma once /* - * abstract interface of hashdata manifest + * abstract interface of manifest */ #ifdef __cplusplus @@ -42,7 +42,7 @@ extern "C" { * memory management. */ -/* hashdata manifest support following data types for fields */ +/* manifest support following data types for fields */ typedef enum MetaFieldType { Meta_Field_Type_Int = 1, diff --git a/contrib/pax_storage/src/test/regress/expected/create_index.out b/contrib/pax_storage/src/test/regress/expected/create_index.out index ac4958d56dc..963ce9a6b39 100644 --- a/contrib/pax_storage/src/test/regress/expected/create_index.out +++ b/contrib/pax_storage/src/test/regress/expected/create_index.out @@ -2690,7 +2690,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index diff --git a/contrib/pax_storage/src/test/regress/expected/create_index_optimizer.out b/contrib/pax_storage/src/test/regress/expected/create_index_optimizer.out index 8d0a41352a8..81728a31260 100644 --- a/contrib/pax_storage/src/test/regress/expected/create_index_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/create_index_optimizer.out @@ -2717,7 +2717,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index diff --git a/contrib/pax_storage/src/test/regress/sql/create_index.sql b/contrib/pax_storage/src/test/regress/sql/create_index.sql index a0e3fc5ccdc..2cb3cc77fd0 100644 --- a/contrib/pax_storage/src/test/regress/sql/create_index.sql +++ b/contrib/pax_storage/src/test/regress/sql/create_index.sql @@ -1111,7 +1111,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -- These are the toast table and index of pg_authid. diff --git a/gpcontrib/gpmapreduce/output/mapred_1.source b/gpcontrib/gpmapreduce/output/mapred_1.source new file mode 100644 index 00000000000..05c29807feb --- /dev/null +++ b/gpcontrib/gpmapreduce/output/mapred_1.source @@ -0,0 +1,798 @@ +-- +-- map/reduce tests +-- +-- Some notes: the PATH variable doesn't necessarily work correctly in +-- the regression context, so use the "gpwhich" token to get the full +-- path from your environment (see gpstringsubs for more details). And +-- to make this work correctly with gpsourcify, you need to add your +-- gpwhich token to gptokencheck. +-- start_matchsubs +-- m|mapreduce_\d+_run| +-- s|mapreduce_\d+_run|mapreduce_PID_run| +-- end_matchsubs +-- +-- This test makes use of plperlu +-- +-- start_ignore +create language plperlu; +create language plpython3u; +-- Take a look at the installed languages, plperl and plpython should be in the list. +-- + pg_pltemplate contains the list of languages that can be simply installed +-- + pl_language contains the list of languages that are actually installed +select * from pg_pltemplate; + tmplname | tmpltrusted | tmplhandler | tmplvalidator | tmpllibrary | tmplacl +-----------+-------------+-----------------------+-------------------+------------------+--------- + plpgsql | t | plpgsql_call_handler | plpgsql_validator | $libdir/plpgsql | + pltcl | t | pltcl_call_handler | | $libdir/pltcl | + pltclu | f | pltclu_call_handler | | $libdir/pltcl | + plperl | t | plperl_call_handler | plperl_validator | $libdir/plperl | + plperlu | f | plperl_call_handler | plperl_validator | $libdir/plperl | + plpython3u | f | plpython_call_handler | | $libdir/plpython | + plr | f | plr_call_handler | | $libdir/plr | +(7 rows) + +select lanname, lanispl, lanpltrusted from pg_language; + lanname | lanispl | lanpltrusted +-----------+---------+-------------- + internal | f | f + c | f | f + sql | f | t + plpgsql | t | t + plperlu | t | f + plpython3u | t | f +(6 rows) + +-- Check environment variables that should have been set by greenplum_path.sh +-- +-- 1) We need to check these on all segments and on the master. +-- 2) We do this via external table rather than perl/python in case it is part +-- of the cause of a mis-installed plperl/plpython. +-- 3) It is normal for the master to have a slightly different enviornment from +-- the segments (but perhaps not desirable?) +-- +CREATE EXTERNAL WEB TABLE env_segment(var text, value text) +EXECUTE 'env | grep "^[^=]*=[^=]*$"' format 'text' (delimiter '='); +CREATE EXTERNAL WEB TABLE env_master(var text, value text) +EXECUTE 'env | grep "^[^=]*=[^=]*$"' ON COORDINATOR format 'text' (delimiter '='); +CREATE VIEW env AS + SELECT gp_execution_segment(), * FROM env_segment + UNION ALL + SELECT gp_execution_segment(), * FROM env_master; +SELECT * FROM env WHERE var in ( + 'GPHOME', + 'DYLD_LIBRARY_PATH', + 'LD_LIBRARY_PATH', + 'PATH' +) ORDER BY var, gp_execution_segment; + gp_execution_segment | var | value +----------------------+-------------------+---------------------------------------------------------------------------------------------------------------------------------------- + -1 | DYLD_LIBRARY_PATH | @gphome@/lib:@gphome@/ext/python/lib:$DYLD_LIBRARY_PATH + 0 | DYLD_LIBRARY_PATH | @gphome@/lib:@gphome@/ext/python/lib: + 1 | DYLD_LIBRARY_PATH | @gphome@/lib:@gphome@/ext/python/lib: + -1 | GPHOME | @gphome@ + 0 | GPHOME | @gphome@ + 1 | GPHOME | @gphome@ + -1 | LD_LIBRARY_PATH | @gphome@/lib:@gphome@/ext/python/lib: + -1 | PATH | @gphome@/bin:/usr/gnu/bin:/usr/local/bin:/bin:/usr/bin:/usr/sbin:/sbin:/usr/texbin:/usr/X11/bin:/sw/bin:/opt/local/bin:/opt/local/sbin + 0 | PATH | @gphome@/ext/python/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin + 1 | PATH | @gphome@/ext/python/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin +(10 rows) + +-- end_ignore +-- start_ignore +-- +-- Some checks to verify what versions of perl/python we have. +-- If everything has been configured correctly this should be constant +-- across all our installations. +-- +-- All of these checks should return a single row because it should produce +-- a constant across all segments and the master. +-- +-- The current expected version is 2.6.2 +-- +CREATE OR REPLACE FUNCTION python_version() returns text as $$ +import sys +return sys.version_info +$$ language plpython3u NO SQL; +SELECT python_version() FROM env GROUP BY python_version; + python_version +----------------------- + (2, 6, 2, 'final', 0) +(1 row) + +-- +-- Same check for perl version +-- +-- Expected version is perl 5.008xxx +-- +CREATE OR REPLACE FUNCTION perl_version() returns text as $$ +return "Perl $]" +$$ language plperlu NO SQL; +SELECT perl_version() FROM env GROUP BY perl_version; + perl_version +--------------- + Perl 5.008005 +(1 row) + +-- +-- The following two checks need to be put into big ignore blocks +-- because paths can be of differing lengths +-- +CREATE OR REPLACE FUNCTION python_path() returns text as $$ +import sys +return sys.path[0] +$$ language plpython3u NO SQL; +SELECT python_path() FROM env GROUP BY python_path; + python_path +--------------------------------------------- + @gphome@/lib/python +(1 row) + +CREATE OR REPLACE FUNCTION perl_path() returns text as $$ +return join(':', @INC) +$$ language plperlu NO SQL; +SELECT perl_path() FROM env GROUP BY perl_path; + perl_path +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + /System/Library/Perl/5.8.8/darwin-thread-multi-2level:/System/Library/Perl/5.8.8:/Library/Perl/5.8.8/darwin-thread-multi-2level:/Library/Perl/5.8.8:/Library/Perl:/Network/Library/Perl/5.8.8/darwin-thread-multi-2level:/Network/Library/Perl/5.8.8:/Network/Library/Perl:/System/Library/Perl/Extras/5.8.8/darwin-thread-multi-2level:/System/Library/Perl/Extras/5.8.8:/Library/Perl/5.8.6:/Library/Perl/5.8.1:. +(1 row) + +-- end_ignore +-- +-- Create a harness to run shell commands and output stderr and stdout +-- +CREATE OR REPLACE FUNCTION execute(cmd text) returns text as $$ +import subprocess +p = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) +r = p.communicate() +header = "---------------------\n" +return header + r[0].decode() +$$ LANGUAGE plpython3u NO SQL; +-- +-- Create a harness to run mapreduce jobs on the correct host/port +-- +CREATE OR REPLACE FUNCTION mapreduce(file text) returns setof text as $$ +import subprocess + +rv = plpy.execute("select hostname, port, user as user, " + + " current_database() as db from gp_segment_configuration where content=-1") + +cmd = ['@gpwhich_gpmapreduce@', + '--host=%s' % rv[0]['hostname'], + '--port=%s' % rv[0]['port'], + '--file=%s' % file, + rv[0]['db'], rv[0]['user'] ] +p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +r = p.communicate() +header = "---------------------" +err = '\nSTDERR> '.join(('STDERR> ' + r[1].decode()).split('\n')).replace('\t', ' ') +out = 'STDOUT>\n' + r[0].decode().replace('\t', ', ') +return [header, err, out] +$$ LANGUAGE plpython3u READS SQL DATA; +CREATE OR REPLACE FUNCTION mapreduce(file text, keys text) returns setof text as $$ +import subprocess + +rv = plpy.execute("select hostname, port, user as user, " + + " current_database() as db from gp_segment_configuration where content=-1") + +cmd = ['@gpwhich_gpmapreduce@', + '--host=%s' % rv[0]['hostname'], + '--port=%s' % rv[0]['port'], + '--file=%s' % file, + rv[0]['db'], rv[0]['user'] ] +for key in keys.split(';'): + cmd.append('--key=%s' % key) +p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +r = p.communicate() +header = "---------------------" +err = '\nSTDERR> '.join(('STDERR> ' + r[1].decode()).split('\n')).replace('\t', ' ') +out = 'STDOUT>\n' + r[0].decode().replace('\t', ', ') +return [header, err, out] +$$ LANGUAGE plpython3u READS SQL DATA; +-- +-- CHECK 1) make sure plperlu is really installed: +-- +-- Note: if not, you might need to 'export PG_LANG=true' and reconfigure/rebuild +-- +SELECT lanname, lanispl, lanpltrusted FROM pg_language WHERE lanname = 'plperlu'; + lanname | lanispl | lanpltrusted +---------+---------+-------------- + plperlu | t | f +(1 row) + +-- +-- Since many of these tests will end up having variable length output depending +-- on local paths we want to disable the "----" lines from psql so that our diffs +-- look reasonable. +-- +-- The default is "aligned" if that ever changes then we need to change how we restore +-- it at the end of the test +\pset format +\pset format unaligned +-- +-- Check 2) Find gpmapreduce and print the help file +-- +-- start_ignore +select execute( '@gpwhich_gpmapreduce@ --help' ); +execute +--------------------- +@gpwhich_gpmapreduce@ - Cloudberry Map/Reduce Driver 1.00b2 + +Usage: + @gpwhich_gpmapreduce@ [options] -f file.yml [dbname [username]] + +General options: + -? | --help show this help, then exit + -V | --version show version information, then exit + -v | --verbose verbose output + -x | --explain do not run jobs, but produce explain plans + -X | --explain-analyze run jobs and produce explain-analyze plans + -k | --key = sets a yaml variable + +Connection options: + -h | --host database server host or socket directory + -p | --port database server port + -U | --username database user name + -W | --password prompt for password + +Debug options: + -D | --debug enable some debugging output + -P | --print print-only mode, do not run jobs + +(1 row) +-- end_ignore +-- +-- TEST 1) complain about missing file +-- +SELECT mapreduce('nosuchfile') ORDER BY 1; +mapreduce +--------------------- +STDERR> Error: Could not open file 'nosuchfile' +STDERR> +STDOUT> + +(3 rows) +-- +-- TEST 2) Tests reading and sorting an input file; +-- +SELECT mapreduce('@abs_srcdir@/yml/sort.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73685_run_1 +STDERR> +STDOUT> +value +------------------------- +alex, 30, (1.352,8.2) +belinda, 38, (8.9,1.7) +bertha, 88, (2.75,9.4) +carina, 58, (4.27,8.8) +carmen, 78, (3.8,8.2) +chris, 78, (9.78,2) +denise, 24, (3.78,87.90) +diane, 18, (5.912,5.3) +edna, 18, (1.53,3.5) +esther, 98, (5.36,7.6) +fanny, 08, (1.2,0.9) +gina, 18, (9.82,7.5) +jane, 58, (1.34,0.44) +jean, 28, (8.561,7.3) +jenifer, 38, (6.6,23.3) +joan, 18, (9.4,47.04) +joe, 20, (5.5,2.5) +juanita, 58, (4.57,35.8) +julie, 68, (3.6,7.2) +karen, 48, (8.73,0.0) +koko, 88, (1.7,5.5) +leah, 68, (0.6,3.37) +lita, 25, (1.3,8.7) +liza, 38, (9.76,6.90) +louise, 98, (5.0,8.7) +martie, 88, (8.358,.93) +mary, 08, (3.7,39.20) +melissa, 28, (3.089,087.23) +mike, 40, (3.1,6.2) +nan, 28, (6.35,0.43) +pamela, 48, (8.21,9.3) +pat, 18, (1.19,0.6) +paula, 68, (0.5,0.5) +rean, 48, (8.5,5.0) +sally, 34, (3.8,45.8) +sandra, 19, (9.345,09.6) +sandy, 38, (3.8,0.2) +sarah, 88, (8.4,2.3) +sharon, 78, (9.237,8.8) +sue, 50, (8.34,7.375) +sumi, 38, (1.15,0.6) +susan, 78, (6.579,3) +teresa, 38, (7.7,1.8) +trisha, 88, (1.29,2.2) +trudy, 88, (6.01,0.5) +velma, 68, (8.8,8.9) +vera, 78, (9.73,6.4) +wendy, 78, (2.62,03.3) +zena, 98, (0.35,0) +zola, 58, (2.56,4.3) +(50 rows) + + +(3 rows) +-- +-- TEST 3) Tests a basic map function and parameter passing +-- +SELECT mapreduce('@abs_srcdir@/yml/grep.yml', 'key=an') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73691_run_1 +STDERR> +STDOUT> +key|value +---+---------------------- +an |diane, 18, (5.912,5.3) +an |fanny, 08, (1.2,0.9) +an |jane, 58, (1.34,0.44) +an |jean, 28, (8.561,7.3) +an |joan, 18, (9.4,47.04) +an |juanita, 58, (4.57,35.8) +an |nan, 28, (6.35,0.43) +an |rean, 48, (8.5,5.0) +an |sandra, 19, (9.345,09.6) +an |sandy, 38, (3.8,0.2) +an |susan, 78, (6.579,3) +(11 rows) + + +(3 rows) +-- +-- Test 4) Tests producing multiple columns +-- +SELECT mapreduce('@abs_srcdir@/yml/grep2.yml', 'key=an') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73697_run_1 +STDERR> +STDOUT> +name |age|location +-------+---+------------ +diane | 18|(5.912,5.3) +fanny | 8|(1.2,0.9) +jane | 58|(1.34,0.44) +jean | 28|(8.561,7.3) +joan | 18|(9.4,47.04) +juanita| 58|(4.57,35.8) +nan | 28|(6.35,0.43) +rean | 48|(8.5,5.0) +sandra | 19|(9.345,09.6) +sandy | 38|(3.8,0.2) +susan | 78|(6.579,3) +(11 rows) + + +(3 rows) +-- +-- Test 5) Tests a basic reduce function and the builtin counterpart +-- +SELECT mapreduce('@abs_srcdir@/yml/agebracket.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73703_run_1 +STDERR> +STDOUT> +key |value +---------------+----- +0 => age < 10 | 2 +10 => age < 20 | 6 +20 => age < 30 | 6 +30 => age < 40 | 8 +40 => age < 50 | 4 +50 => age < 60 | 5 +60 => age < 70 | 4 +70 => age < 80 | 6 +80 => age < 90 | 6 +90 => age < 100| 3 +(10 rows) + + +(3 rows) +SELECT mapreduce('@abs_srcdir@/yml/agebracket_builtin.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73703_run_1 +STDERR> +STDOUT> +key |value +---------------+----- +0 => age < 10 | 2 +10 => age < 20 | 6 +20 => age < 30 | 6 +30 => age < 40 | 8 +40 => age < 50 | 4 +50 => age < 60 | 5 +60 => age < 70 | 4 +70 => age < 80 | 6 +80 => age < 90 | 6 +90 => age < 100| 3 +(10 rows) + + +(3 rows) +-- +-- Test 6) File Output tests +-- +SELECT execute('rm @abs_builddir@/results/fileout_*.out') ORDER BY 1; +execute +--------------------- + +(1 row) +SELECT mapreduce('@abs_srcdir@/yml/fileout.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73712_run_1 +STDERR> mapreduce_73712_run_2 +STDERR> mapreduce_73712_run_3 +STDERR> +STDOUT> + +(3 rows) +SELECT execute('cat @abs_builddir@/results/fileout_none.out') ORDER BY 1; +execute +--------------------- +row 1:data 1 +row 2:data 2 +row 3:data 3 + +(1 row) +SELECT execute('cat @abs_builddir@/results/fileout_replace.out') ORDER BY 1; +execute +--------------------- +row 1|data 1 +row 2|data 2 +row 3|data 3 + +(1 row) +SELECT execute('cat @abs_builddir@/results/fileout_append.out') ORDER BY 1; +execute +--------------------- +row 1,data 1 +row 2,data 2 +row 3,data 3 + +(1 row) +SELECT mapreduce('@abs_srcdir@/yml/fileout.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_73721_run_1 +STDERR> mapreduce_73721_run_2 +STDERR> mapreduce_73721_run_3 +STDERR> Error: OUTPUT 'out_3': file '@abs_builddir@/results/fileout_none.out' already exists, at line 27 +STDERR> Error: Object creation Failure +STDERR> +STDOUT> + +(3 rows) +SELECT execute('cat @abs_builddir@/results/fileout_none.out') ORDER BY 1; +execute +--------------------- +row 1:data 1 +row 2:data 2 +row 3:data 3 + +(1 row) +SELECT execute('cat @abs_builddir@/results/fileout_replace.out') ORDER BY 1; +execute +--------------------- +row 1|data 1 +row 2|data 2 +row 3|data 3 + +(1 row) +SELECT execute('cat @abs_builddir@/results/fileout_append.out') ORDER BY 1; +execute +--------------------- +row 1,data 1 +row 2,data 2 +row 3,data 3 +row 1,data 1 +row 2,data 2 +row 3,data 3 + +(1 row) +-- +-- Test 7) Syntax error lineno reporting +-- +SELECT mapreduce('@abs_srcdir@/yml/perlerror.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> ERROR: syntax error at line 18, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_1977_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 28, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_1992_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 37, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_2000_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 45, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_2011_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 53, near "[]" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_2020_grep_map" +STDERR> Error: Object creation Failure +STDERR> ERROR: syntax error at line 67, near "}GABLECK" +STDERR> Execution of aborted due to compilation errors. +STDERR> CONTEXT: compilation of PL/Perl function "mapreduce_52431_mymap" +STDERR> Error: Object creation Failure +STDERR> WARNING: unset parameter - myMap(key => NULL) +STDERR> mapreduce_PID_run_1 +STDERR> ERROR: }GABLECK!{ at line 85. +STDERR> CONTEXT: PL/Perl function "mapreduce_52435_mymap" +STDERR> Error: Execution Failure +STDERR> +STDOUT> + +(3 rows) +SELECT mapreduce('@abs_srcdir@/yml/yamlerror.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> Error: OUTPUT 'output_table_empty': Invalid TABLE, at line 8 +STDERR> Error: OUTPUT 'output_table_empty': Invalid TABLE, at line 8 +STDERR> Error: OUTPUT 'output_table_string': Invalid TABLE, at line 11 +STDERR> Error: OUTPUT 'output_table_string': Invalid TABLE, at line 11 +STDERR> Error: OUTPUT 'output_table_number': Invalid TABLE, at line 14 +STDERR> Error: OUTPUT 'output_table_number': Invalid TABLE, at line 14 +STDERR> Error: OUTPUT 'output_table_list': Invalid TABLE, at line 17 +STDERR> Error: OUTPUT 'output_table_list': Invalid TABLE, at line 17 +STDERR> Error: OUTPUT 'output_table_mapping': Invalid TABLE, at line 20 +STDERR> Error: OUTPUT 'output_table_mapping': Invalid TABLE, at line 20 +STDERR> Error: OUTPUT 'output_table_mapping': Missing FILE or TABLE, at line 21 +STDERR> Error: INPUT 'input_columns_mapping': Invalid COLUMNS, at line 33 +STDERR> Error: Unrecognized VERSION, at line 62 +STDERR> Error: DOCUMENT: Missing VERSION, at line 98 +STDERR> Error: INPUT must contain a YAML MAPPING, at line 138 +STDERR> Error: INPUT: Missing NAME, at line 138 +STDERR> Error: INPUT: Missing FILE, GPFDIST, TABLE, QUERY, or EXEC, at line 138 +STDERR> Error: INPUT 'mystart': Duplicate QUERY for INPUT, at line 144 +STDERR> Error: parse failure +STDERR> +STDOUT> + +(3 rows) +-- Test 8) C functions using LIBRARY +SELECT mapreduce('@abs_srcdir@/yml/c.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> +STDOUT> +key|value +---+----- + 0| 136 + 1| 401 + 3| 98 + 4| 1162 + 5| 257 + 6| 2073 + 7| 2138 + 8| 1082 + 9| 2241 + 10|11556 + 11| 8236 + 12| 473 + 13| 477 + 14| 0 + 15| 2287 + 16| 2772 + 17| 2359 + 18| 536 + 19|25936 + 20| 910 +(20 rows) + + +(3 rows) +-- Test 9) PageRank +SELECT mapreduce('@abs_srcdir@/yml/pagerank-init.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> DONE +STDERR> +STDOUT> + +(3 rows) +SELECT count(*) FROM pagerank_source; +count +20 +(1 row) +SELECT mapreduce('@abs_srcdir@/yml/pagerank-iter.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> DONE +STDERR> +STDOUT> + +(3 rows) +SELECT mapreduce('@abs_srcdir@/yml/pagerank-final.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> DONE +STDERR> +STDOUT> + +(3 rows) +CREATE OR REPLACE FUNCTION parse_pg_rank_kv(s text) returns text as $$ +i = s.index(",") +rank = s[:i] +neighbors = list(map(int, s[i+1:].strip().strip("()").split(","))) +neighbors.sort() +ns = "(" + ",".join(map(str, neighbors)) + ")" +return ", ".join([rank, ns]) +$$ LANGUAGE plpython3u NO SQL; +-- The result of the following test case is related to the underlying hash algorithm +-- because the intermediate result of computing is saved as table in gpdb and hash-distributed. +-- If you change the underlying hash algorithm, please remember to verify the answer and change +-- here also. +SELECT parse_pg_rank_kv(ps.value), parse_pg_rank_kv(pn.value) FROM pagerank_source ps JOIN pagerank_next pn USING (key) WHERE ps.key = '17'; +parse_pg_rank_kv|parse_pg_rank_kv +0.85, (1,258,259,260,261,262,263,264,265,266)|0.265867988394584, (1,258,259,260,261,262,263,264,265,266) +(1 row) +-- Test setup +CREATE TABLE simple (m int,n int) distributed randomly; +INSERT INTO simple VALUES (1,10), (2,20), (2,21), (2,22), (3,30), (4,40), (5,50), (5,50), (10,100), (2,21); +CREATE FUNCTION tran (state int, arg2 int) returns int language C AS '@abs_srcdir@/regress.so', 'tran'; +CREATE FUNCTION tran (state int) returns int language C AS '@abs_srcdir@/regress.so', 'tran'; +CREATE FUNCTION final (state int) returns int language C AS '@abs_srcdir@/regress.so', 'final'; +CREATE FUNCTION cons (state int, value int, out output int) returns int language C AS '@abs_srcdir@/regress.so', 'cons'; +CREATE FUNCTION cons_wrongtype (state int, value text) returns int language C AS '@abs_srcdir@/regress.so', 'cons'; +CREATE OR REPLACE FUNCTION retcomposite(int) + RETURNS SETOF __retcomposite + AS '@abs_srcdir@/regress.so', 'retcomposite' + LANGUAGE C IMMUTABLE STRICT; +NOTICE: type "__retcomposite" is not yet defined +DETAIL: Creating a shell type definition. +CREATE FUNCTION gpmapred_scube_accum8(numeric,numeric) RETURNS numeric +AS 'SELECT $1 + $2 * $2 * $2' +LANGUAGE SQL +IMMUTABLE +RETURNS NULL ON NULL INPUT; +CREATE AGGREGATE gpmapred_scube(numeric) ( + SFUNC = gpmapred_scube_accum8, + STYPE = numeric, + INITCOND = 0 ); +-- Test 10) Aggregate Function +SELECT mapreduce('@abs_srcdir@/yml/aggFunction.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> +STDOUT> + m| value +--+------- + 1| 1000 + 2| 37170 + 3| 27000 + 4| 64000 + 5| 250000 +10|1000000 +(6 rows) + + +(3 rows) +-- Test 11) Built-in Functions +SELECT mapreduce('@abs_srcdir@/yml/builtinfunction.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> mapreduce_PID_run_2 +STDERR> mapreduce_PID_run_3 +STDERR> mapreduce_PID_run_4 +STDERR> mapreduce_PID_run_5 +STDERR> +STDOUT> + m|value +--+----- + 1| 10 + 2| 20 + 3| 30 + 4| 40 + 5| 50 +10| 100 +(6 rows) + + m|value +--+----- + 1| 10 + 2| 22 + 3| 30 + 4| 40 + 5| 50 +10| 100 +(6 rows) + + m| value +--+-------------------- + 1| 10.0000000000000000 + 2| 21.0000000000000000 + 3| 30.0000000000000000 + 4| 40.0000000000000000 + 5| 50.0000000000000000 +10|100.0000000000000000 +(6 rows) + + m|value +--+----- + 1| 1 + 2| 4 + 3| 1 + 4| 1 + 5| 2 +10| 1 +(6 rows) + + m|value +--+----- + 1| 10 + 2| 20 + 2| 21 + 2| 21 + 2| 22 + 3| 30 + 4| 40 + 5| 50 + 5| 50 +10| 100 +(10 rows) + + +(3 rows) +-- Test 12) Ambigious function +SELECT mapreduce('@abs_srcdir@/yml/ambiguousfunction.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> Error: MAP 'tran': Ambiguous function, supply a function prototype for disambiguation, at line 11 +STDERR> Error: Object creation Failure +STDERR> +STDOUT> + +(3 rows) +-- Test 13) Parameter override +SELECT mapreduce('@abs_srcdir@/yml/paramoverride.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> mapreduce_PID_run_1 +STDERR> +STDOUT> + m|retval +--+------ + 1| 20 + 2| 168 + 3| 60 + 4| 80 + 5| 200 +10| 200 +(6 rows) + + +(3 rows) +-- Test 14) Incorrect parameter type +SELECT mapreduce('@abs_srcdir@/yml/paramdifftype.yml') ORDER BY 1; +mapreduce +--------------------- +STDERR> ERROR: function cons_wrongtype(integer, integer) does not exist +STDERR> Error: Object creation Failure +STDERR> +STDOUT> + +(3 rows) +-- +-- Cleanup) Restore normal formatting options +-- +\pset format aligned diff --git a/src/backend/cdb/cdbpath.c b/src/backend/cdb/cdbpath.c index 89d15437661..9e3697a3b03 100644 --- a/src/backend/cdb/cdbpath.c +++ b/src/backend/cdb/cdbpath.c @@ -2964,7 +2964,6 @@ can_elide_explicit_motion(PlannerInfo *root, Index rti, Path *subpath, * There was once an idea reseting path's parallel_works to avoid * Motion if inner and outer's parallel_workers doesn't match. * But there are a lot of issues we don't have a clear answer. - * See https://code.hashdata.xyz/cloudberry/cbdb-postgres-merge/-/issues/43. * * We couldn't expect the parallel_workers of outer or inner path. * Partial path may generate locus(parallel_workers=0) if needed, ex: diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index f795078e761..0bc2aaa9f6e 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2683,7 +2683,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index diff --git a/src/test/regress/expected/create_index_optimizer.out b/src/test/regress/expected/create_index_optimizer.out index f661e53f487..65f5f92b8bd 100644 --- a/src/test/regress/expected/create_index_optimizer.out +++ b/src/test/regress/expected/create_index_optimizer.out @@ -2723,7 +2723,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation ERROR: cannot reindex system catalogs concurrently REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql index 1b598d13b27..ac5526ea9b1 100644 --- a/src/test/regress/sql/create_index.sql +++ b/src/test/regress/sql/create_index.sql @@ -1109,7 +1109,6 @@ COMMIT; -- we keep the `CONCURRENTLY` to make the following commands fail, -- so these commands will not cause deadlock with test create_view, -- like `drop schema xxx cascade;`. --- See more details at https://code.hashdata.xyz/cloudberry/cbdb/-/issues/54 REINDEX TABLE CONCURRENTLY pg_class; -- no catalog relation REINDEX INDEX CONCURRENTLY pg_class_oid_index; -- no catalog index -- These are the toast table and index of pg_authid.