From f5e061df6f32fd7de74d2601ef9c8dd6f5fb2de3 Mon Sep 17 00:00:00 2001 From: Huansong Fu Date: Tue, 19 Jul 2022 07:13:55 -0700 Subject: [PATCH 1/8] ALTER TABLE SET ACCESS METHOD: AO->AOCO support Currently adding support for the following cases: ``` CREATE TABLE foo (appendonly=true); ALTER TABLE foo SET ACCESS METHOD ao_column; -- Or: ALTER TABLE foo SET WITH (appendonly=true, orientation=column); ``` Similar to other variations of ATSETAM commands, user can also specify reloptions in a WITH clause, such as: ``` ALTER TABLE foo SET ACCESS METHOD ao_column WITH (blocksize=65536); ``` If no reloptions are given, the new AOCO table will use the existing table-level options for its column encoding options. If any reloption is given in the WITH clause, it will be recorded in the catalog and then used for the column encoding option too. Note that there was once a thought to support specifying column-level encoding in the ATSETAM command, but was abandoned because there exists better alternatives. Discussions see https://groups.google.com/a/greenplum.org/g/gpdb-dev/c/NaNH6TssgA8 --- src/backend/catalog/pg_appendonly.c | 4 +- src/backend/commands/tablecmds.c | 35 ++++ src/backend/nodes/makefuncs.c | 1 + src/backend/parser/gram.y | 38 ++--- .../regress/expected/alter_table_set_am.out | 160 +++++++++++++++++- src/test/regress/sql/alter_table_set_am.sql | 82 ++++++++- 6 files changed, 293 insertions(+), 27 deletions(-) diff --git a/src/backend/catalog/pg_appendonly.c b/src/backend/catalog/pg_appendonly.c index 1f51d8a6256..c1c364cdbd4 100644 --- a/src/backend/catalog/pg_appendonly.c +++ b/src/backend/catalog/pg_appendonly.c @@ -578,9 +578,7 @@ ATAOEntries(Form_pg_class relform1, Form_pg_class relform2, swapAppendonlyEntriesUsingTAM(relform1, relform2, frozenXid, cutoffMulti); break; case AO_COLUMN_TABLE_AM_OID: - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("alter table does not support switch from AO to AOCO"))); + SwapAppendonlyEntries(relform1->oid, relform2->oid); break; default: ereport(ERROR, diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 6aacb41386b..f8f46934427 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -584,6 +584,7 @@ static bool prebuild_temp_table(Relation rel, RangeVar *tmpname, DistributedBy * static void checkATSetDistributedByStandalone(AlteredTableInfo *tab, Relation rel); +static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOptions); /* ---------------------------------------------------------------- @@ -4818,6 +4819,35 @@ AlterTable(AlterTableStmt *stmt, LOCKMODE lockmode, } } +/* + * Populate the column encoding option for each column in the relation. + */ +static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOptions) +{ + int attno; + List *colDefs = NIL; + TupleDesc tupdesc = RelationGetDescr(rel); + + /* Figure out the column definition list. */ + for (attno = 0; attno < tupdesc->natts; attno++) + { + Form_pg_attribute att = TupleDescAttr(tupdesc, attno); + ColumnDef *cd = makeColumnDef(NameStr(att->attname), + att->atttypid, + att->atttypmod, + 0); + colDefs = lappend(colDefs, cd); + } + + List *attr_encodings = transformColumnEncoding(rel, + colDefs /*column clauses*/, + stenc /*encoding clauses*/, + withOptions /*withOptions*/, + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE /*rootpartition*/, + false /*errorOnEncodingClause*/); + AddRelationAttributeEncodings(rel, attr_encodings); +} + /* * AlterTableInternal * @@ -6179,6 +6209,11 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, if (aoopt_changed) tab->rewrite |= AT_REWRITE_ALTER_RELOPTS; } + + /* If we are changing AM to AOCO, add pg_attribute_encoding entries for each column. */ + if (tab->newAccessMethod == AO_COLUMN_TABLE_AM_OID) + populate_rel_col_encodings(rel, NULL, (List*)cmd->def); + break; case AT_SetTableSpace: /* SET TABLESPACE */ diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c index 6ac95ee2b43..cbf64e2ad00 100644 --- a/src/backend/nodes/makefuncs.c +++ b/src/backend/nodes/makefuncs.c @@ -506,6 +506,7 @@ makeColumnDef(const char *colname, Oid typeOid, int32 typmod, Oid collOid) n->collClause = NULL; n->collOid = collOid; n->constraints = NIL; + n->encoding = NIL; n->fdwoptions = NIL; n->location = -1; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 8b8a3c28bd8..f300cdb8284 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -3809,19 +3809,11 @@ alter_table_cmd: n->newowner = $3; $$ = (Node *)n; } - /* ALTER TABLE SET ACCESS METHOD */ - | SET ACCESS METHOD name - { - AlterTableCmd *n = makeNode(AlterTableCmd); - n->subtype = AT_SetAccessMethod; - n->name = $4; - $$ = (Node *)n; - } /* ALTER TABLE SET ACCESS METHOD WITH () */ - | SET ACCESS METHOD name WITH definition + | SET ACCESS METHOD name OptWith { AlterTableCmd *n = makeNode(AlterTableCmd); - char *witham = greenplumLegacyAOoptions(n->name, &$6); + char *witham = greenplumLegacyAOoptions(n->name, &$5); n->subtype = AT_SetAccessMethod; n->name = $4; /* @@ -3829,18 +3821,24 @@ alter_table_cmd: * clause such as 'appendonly' or 'appendoptimized', it has * to match with the AM name. */ - if (witham && - (strlen(witham) != strlen(n->name) || - strncmp(n->name, witham, strlen(n->name) != 0))) + if (witham) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("ACCESS METHOD is specified as \"%s\" but " - "the WITH option indicates it to be \"%s\"", - n->name, witham), - parser_errposition(@5))); + if (strlen(witham) != strlen(n->name) || + strncmp(n->name, witham, strlen(n->name) != 0)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("ACCESS METHOD is specified as \"%s\" but " + "the WITH option indicates it to be \"%s\"", + n->name, witham), + parser_errposition(@5))); + else + ereport(NOTICE, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("Redundant clauses are used to indicate the access method."), + errhint("Only one of these is needed to indicate access method: the " + "SET ACCESS METHOD clause or the options in the WITH clause."))); } - n->def = (Node *) $6; + n->def = (Node *) $5; $$ = (Node *)n; } /* ALTER TABLE SET TABLESPACE */ diff --git a/src/test/regress/expected/alter_table_set_am.out b/src/test/regress/expected/alter_table_set_am.out index 9fe0e3d9686..58e0f934d4b 100644 --- a/src/test/regress/expected/alter_table_set_am.out +++ b/src/test/regress/expected/alter_table_set_am.out @@ -601,12 +601,168 @@ SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam DROP TABLE ataoset; DROP TABLE ataoset2; --- Final scenario: run the iterations of AT from "A" to "B" and back to "A", that includes: +-- Scenario 5: AO to AOCO +SET gp_default_storage_options = 'blocksize=65536, compresstype=zlib, compresslevel=5, checksum=true'; +CREATE TABLE ao2co(a int, b int) WITH (appendonly=true); +CREATE TABLE ao2co2(a int, b int) WITH (appendonly=true); +CREATE TABLE ao2co3(a int, b int) WITH (appendonly=true); +CREATE TABLE ao2co4(a int, b int) WITH (appendonly=true); +CREATE INDEX index_ao2co ON ao2co(b); +CREATE INDEX index_ao2co3 ON ao2co3(b); +INSERT INTO ao2co SELECT i,i FROM generate_series(1,5) i; +INSERT INTO ao2co2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO ao2co3 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO ao2co4 SELECT i,i FROM generate_series(1,5) i; +-- ERROR: conflicting storage option specified. +ALTER TABLE ao2co SET ACCESS METHOD ao_column WITH (appendoptimized=true, orientation=row); +ERROR: ACCESS METHOD is specified as "ao_column" but the WITH option indicates it to be "ao_row" +LINE 1: ALTER TABLE ao2co SET ACCESS METHOD ao_column WITH (appendop... + ^ +-- Use of *both* ACCESS METHOD and WITH clauses is allowed, but we'll print a hint to indicate the redundancy. +ALTER TABLE ao2co SET ACCESS METHOD ao_row WITH (appendoptimized=true, orientation=row); +HINT: Only one of these is needed to indicate access method: the SET ACCESS METHOD clause or the options in the WITH clause. +NOTICE: Redundant clauses are used to indicate the access method. +CREATE TEMP TABLE relfilebeforeao AS + SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'ao2co%' + UNION SELECT gp_segment_id segid, relname, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'ao2co%' ORDER BY segid; +-- Check once the reloptions +SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'ao2co%'; + relname | amname | reloptions +---------+--------+----------------------------------- + ao2co | ao_row | {blocksize=65536,compresslevel=5} + ao2co2 | ao_row | {blocksize=65536,compresslevel=5} + ao2co3 | ao_row | {blocksize=65536,compresslevel=5} + ao2co4 | ao_row | {blocksize=65536,compresslevel=5} +(4 rows) + +-- Altering AO to AOCO with various syntaxes, reloptions: +ALTER TABLE ao2co SET ACCESS METHOD ao_column; +ALTER TABLE ao2co2 SET WITH (appendoptimized=true, orientation=column); +ALTER TABLE ao2co3 SET ACCESS METHOD ao_column WITH (blocksize=32768, compresslevel=3); +ALTER TABLE ao2co4 SET WITH (appendoptimized=true, orientation=column, blocksize=32768, compresslevel=3); +-- The tables are rewritten +CREATE TEMP TABLE relfileafterao AS + SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'ao2co%' + UNION SELECT gp_segment_id segid, relname, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'ao2co%' ORDER BY segid; +SELECT * FROM relfilebeforeao INTERSECT SELECT * FROM relfileafterao; + segid | relname | relfilenode +-------+---------+------------- +(0 rows) + +DROP TABLE relfilebeforeao; +DROP TABLE relfileafterao; +-- Check data is intact +SELECT count(*) FROM ao2co; + count +------- + 5 +(1 row) + +SELECT count(*) FROM ao2co2; + count +------- + 5 +(1 row) + +SELECT count(*) FROM ao2co3; + count +------- + 5 +(1 row) + +SELECT count(*) FROM ao2co4; + count +------- + 5 +(1 row) + +-- Aux tables should have been deleted for the old AO table and recreated for the new AOCO table +-- Only tested for 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- No need to test the other ones created by the alternative syntax SET WITH(). +SELECT * FROM gp_toolkit.__gp_aoseg('ao2co'); +ERROR: 'ao2co' is not an append-only row relation +SELECT * FROM gp_toolkit.__gp_aovisimap('ao2co'); + tid | segno | row_num +-----+-------+--------- +(0 rows) + +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('ao2co'); + count +------- + 6 +(1 row) + +SELECT * FROM gp_toolkit.__gp_aoblkdir('ao2co'); + tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count +---------+-------+----------------+----------+--------------+-------------+----------- +(0 rows) + +SELECT * FROM gp_toolkit.__gp_aoseg('ao2co3'); +ERROR: 'ao2co3' is not an append-only row relation +SELECT * FROM gp_toolkit.__gp_aovisimap('ao2co3'); + tid | segno | row_num +-----+-------+--------- +(0 rows) + +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('ao2co3'); + count +------- + 6 +(1 row) + +SELECT * FROM gp_toolkit.__gp_aoblkdir('ao2co3'); + tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count +---------+-------+----------------+----------+--------------+-------------+----------- +(0 rows) + +-- pg_attribute_encoding should have columns for the AOCO table +SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid = c.oid AND c.relname LIKE 'ao2co%'; + relname | attnum | attoptions +---------+--------+----------------------------------------------------- + ao2co | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + ao2co | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + ao2co2 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + ao2co2 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + ao2co3 | 1 | {blocksize=32768,compresslevel=3,compresstype=zlib} + ao2co3 | 2 | {blocksize=32768,compresslevel=3,compresstype=zlib} + ao2co4 | 1 | {blocksize=32768,compresslevel=3,compresstype=zlib} + ao2co4 | 2 | {blocksize=32768,compresslevel=3,compresstype=zlib} +(8 rows) + +-- AM and reloptions changed accordingly +SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'ao2co%'; + relname | amname | reloptions +---------+-----------+----------------------------------- + ao2co | ao_column | {blocksize=65536,compresslevel=5} + ao2co2 | ao_column | {blocksize=65536,compresslevel=5} + ao2co3 | ao_column | {blocksize=32768,compresslevel=3} + ao2co4 | ao_column | {blocksize=32768,compresslevel=3} +(4 rows) + +-- pg_appendonly should reflect the changes in reloptions +SELECT c.relname,a.blocksize,a.compresslevel,a.checksum,a.compresstype,a.columnstore +FROM pg_appendonly a, pg_class c WHERE a.relid = c.oid AND relname like ('ao2co%'); + relname | blocksize | compresslevel | checksum | compresstype | columnstore +---------+-----------+---------------+----------+--------------+------------- + ao2co | 65536 | 5 | t | zlib | t + ao2co2 | 65536 | 5 | t | zlib | t + ao2co3 | 32768 | 3 | t | zlib | t + ao2co4 | 32768 | 3 | t | zlib | t +(4 rows) + +DROP TABLE ao2co; +DROP TABLE ao2co2; +DROP TABLE ao2co3; +DROP TABLE ao2co4; +-- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". +-- The following cases will cover all variations of such iterations: -- 1. Heap->AO->Heap->AO -- (TODO) 2. AO->AOCO->AO->AOCO -- (TODO) 3. Heap->AOCO->Heap->AOCO -- 1. Heap->AO->Heap->AO -CREATE TABLE heapao(a int, b int) WITH (appendonly=true); +CREATE TABLE heapao(a int, b int); CREATE INDEX heapaoindex ON heapao(b); INSERT INTO heapao SELECT i,i FROM generate_series(1,5) i; ALTER TABLE heapao SET ACCESS METHOD ao_row; diff --git a/src/test/regress/sql/alter_table_set_am.sql b/src/test/regress/sql/alter_table_set_am.sql index aebf63f9a0b..d2d65f65d3c 100644 --- a/src/test/regress/sql/alter_table_set_am.sql +++ b/src/test/regress/sql/alter_table_set_am.sql @@ -341,13 +341,90 @@ SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam DROP TABLE ataoset; DROP TABLE ataoset2; --- Final scenario: run the iterations of AT from "A" to "B" and back to "A", that includes: +-- Scenario 5: AO to AOCO +SET gp_default_storage_options = 'blocksize=65536, compresstype=zlib, compresslevel=5, checksum=true'; +CREATE TABLE ao2co(a int, b int) WITH (appendonly=true); +CREATE TABLE ao2co2(a int, b int) WITH (appendonly=true); +CREATE TABLE ao2co3(a int, b int) WITH (appendonly=true); +CREATE TABLE ao2co4(a int, b int) WITH (appendonly=true); +CREATE INDEX index_ao2co ON ao2co(b); +CREATE INDEX index_ao2co3 ON ao2co3(b); + +INSERT INTO ao2co SELECT i,i FROM generate_series(1,5) i; +INSERT INTO ao2co2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO ao2co3 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO ao2co4 SELECT i,i FROM generate_series(1,5) i; + +-- ERROR: conflicting storage option specified. +ALTER TABLE ao2co SET ACCESS METHOD ao_column WITH (appendoptimized=true, orientation=row); +-- Use of *both* ACCESS METHOD and WITH clauses is allowed, but we'll print a hint to indicate the redundancy. +ALTER TABLE ao2co SET ACCESS METHOD ao_row WITH (appendoptimized=true, orientation=row); + +CREATE TEMP TABLE relfilebeforeao AS + SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'ao2co%' + UNION SELECT gp_segment_id segid, relname, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'ao2co%' ORDER BY segid; + +-- Check once the reloptions +SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'ao2co%'; + +-- Altering AO to AOCO with various syntaxes, reloptions: +ALTER TABLE ao2co SET ACCESS METHOD ao_column; +ALTER TABLE ao2co2 SET WITH (appendoptimized=true, orientation=column); +ALTER TABLE ao2co3 SET ACCESS METHOD ao_column WITH (blocksize=32768, compresslevel=3); +ALTER TABLE ao2co4 SET WITH (appendoptimized=true, orientation=column, blocksize=32768, compresslevel=3); + +-- The tables are rewritten +CREATE TEMP TABLE relfileafterao AS + SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'ao2co%' + UNION SELECT gp_segment_id segid, relname, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'ao2co%' ORDER BY segid; + +SELECT * FROM relfilebeforeao INTERSECT SELECT * FROM relfileafterao; +DROP TABLE relfilebeforeao; +DROP TABLE relfileafterao; + +-- Check data is intact +SELECT count(*) FROM ao2co; +SELECT count(*) FROM ao2co2; +SELECT count(*) FROM ao2co3; +SELECT count(*) FROM ao2co4; + +-- Aux tables should have been deleted for the old AO table and recreated for the new AOCO table +-- Only tested for 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- No need to test the other ones created by the alternative syntax SET WITH(). +SELECT * FROM gp_toolkit.__gp_aoseg('ao2co'); +SELECT * FROM gp_toolkit.__gp_aovisimap('ao2co'); +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('ao2co'); +SELECT * FROM gp_toolkit.__gp_aoblkdir('ao2co'); +SELECT * FROM gp_toolkit.__gp_aoseg('ao2co3'); +SELECT * FROM gp_toolkit.__gp_aovisimap('ao2co3'); +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('ao2co3'); +SELECT * FROM gp_toolkit.__gp_aoblkdir('ao2co3'); + +-- pg_attribute_encoding should have columns for the AOCO table +SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid = c.oid AND c.relname LIKE 'ao2co%'; + +-- AM and reloptions changed accordingly +SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'ao2co%'; + +-- pg_appendonly should reflect the changes in reloptions +SELECT c.relname,a.blocksize,a.compresslevel,a.checksum,a.compresstype,a.columnstore +FROM pg_appendonly a, pg_class c WHERE a.relid = c.oid AND relname like ('ao2co%'); + +DROP TABLE ao2co; +DROP TABLE ao2co2; +DROP TABLE ao2co3; +DROP TABLE ao2co4; + +-- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". +-- The following cases will cover all variations of such iterations: -- 1. Heap->AO->Heap->AO -- (TODO) 2. AO->AOCO->AO->AOCO -- (TODO) 3. Heap->AOCO->Heap->AOCO -- 1. Heap->AO->Heap->AO -CREATE TABLE heapao(a int, b int) WITH (appendonly=true); +CREATE TABLE heapao(a int, b int); CREATE INDEX heapaoindex ON heapao(b); INSERT INTO heapao SELECT i,i FROM generate_series(1,5) i; @@ -358,3 +435,4 @@ ALTER TABLE heapao SET ACCESS METHOD ao_row; -- Just checking data is intact. SELECT count(*) FROM heapao; DROP TABLE heapao; + From 23d630ad3e20ad323f77bff08fdbdfaebb57d4ba Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 22 Jan 2025 06:24:27 +0000 Subject: [PATCH 2/8] Fix populate_rel_col_encodings --- src/backend/commands/tablecmds.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index f8f46934427..57b0b8129e2 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -4826,6 +4826,7 @@ static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOpti { int attno; List *colDefs = NIL; + const TableAmRoutine *tam; TupleDesc tupdesc = RelationGetDescr(rel); /* Figure out the column definition list. */ @@ -4839,12 +4840,16 @@ static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOpti colDefs = lappend(colDefs, cd); } - List *attr_encodings = transformColumnEncoding(rel, + tam = GetTableAmRoutineByAmId(rel->rd_rel->relam); + + List *attr_encodings = transformColumnEncoding(tam /* TableAmRoutine */, rel, colDefs /*column clauses*/, stenc /*encoding clauses*/, withOptions /*withOptions*/, - rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE /*rootpartition*/, - false /*errorOnEncodingClause*/); + NULL, + false, + RelationIsAoCols(rel)); + AddRelationAttributeEncodings(rel, attr_encodings); } From 96b41edc6da706c0ce885d0f246aa0142f359f02 Mon Sep 17 00:00:00 2001 From: Huansong Fu Date: Mon, 15 Aug 2022 11:31:58 -0700 Subject: [PATCH 3/8] Remove existing reloptions when AM is changed Previously, we only remove the existing reloptions when the AM is changing between Heap and AO/AOCO, since the reloptions are mutually exclusive between them. Now we do the same when changing AM between AO and AOCO too, for the following reasons: 1. Keep behaviors consistent. 2. Even between AO and AOCO there's certain reloption that's not applicable to one of them: e.g. 'compresslevel=rle_type' only applies to AOCO but not AO. 3. If customer specifies reloptions themselves within the ATSETAM statement, we then need to pick and remove those conflicting with the new reloptions. Too much complicities will be introduced. With this change, we also refactored ATExecSetRelOptions a little bit since now we don't rely on it to remove reloptions. And it becomes more readable too. --- src/backend/catalog/pg_appendonly.c | 1 + src/backend/commands/tablecmds.c | 53 +++++- .../regress/expected/alter_table_set_am.out | 10 +- src/test/regress/sql/alter_table_set_am.sql | 161 ++++++++++++++++++ 4 files changed, 212 insertions(+), 13 deletions(-) diff --git a/src/backend/catalog/pg_appendonly.c b/src/backend/catalog/pg_appendonly.c index c1c364cdbd4..8ec58f74f34 100644 --- a/src/backend/catalog/pg_appendonly.c +++ b/src/backend/catalog/pg_appendonly.c @@ -21,6 +21,7 @@ #include "catalog/pg_am_d.h" #include "catalog/pg_appendonly.h" +#include "catalog/pg_attribute_encoding.h" #include "catalog/pg_type.h" #include "catalog/pg_proc.h" #include "catalog/gp_fastsequence.h" diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 57b0b8129e2..901a471c7c2 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -585,6 +585,7 @@ static bool prebuild_temp_table(Relation rel, RangeVar *tmpname, DistributedBy * static void checkATSetDistributedByStandalone(AlteredTableInfo *tab, Relation rel); static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOptions); +static void remove_rel_opts(Relation rel); /* ---------------------------------------------------------------- @@ -6200,7 +6201,6 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, break; case AT_SetAccessMethod: /* SET ACCESS METHOD */ /* Set reloptions if specified any. Otherwise handled specially in Phase 3. */ - if (cmd->def) { bool aoopt_changed = false; @@ -16125,7 +16125,14 @@ ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, if (defList == NIL && operation != AT_ReplaceRelOptions) return; /* nothing to do */ - newAM = OidIsValid(newAccessMethod) ? GetTableAmRoutineByAmId(newAccessMethod) : NULL; + + /* If we are changing access method, simply remove all the existing ones. */ + if (OidIsValid(newAccessMethod)) + { + newAM = GetTableAmRoutineByAmId(newAccessMethod); + remove_rel_opts(rel); + } else + newAM = NULL; pgclass = table_open(RelationRelationId, RowExclusiveLock); @@ -16135,14 +16142,11 @@ ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for relation %u", relid); - if (operation == AT_ReplaceRelOptions || - (newAccessMethod != InvalidOid && IsAccessMethodAO(rel->rd_rel->relam) != IsAccessMethodAO(newAccessMethod))) + if (operation == AT_ReplaceRelOptions) { /* - * If we're supposed to replace the reloptions list, or if we're - * changing AM between heap and AO/CO so the old reloptions won't - * apply to the new table anymore, we just pretend there were - * none before. + * If we're supposed to replace the reloptions list, we just + * pretend there were none before. */ datum = (Datum) 0; isnull = true; @@ -17997,6 +18001,39 @@ get_rel_opts(Relation rel) return newOptions; } +/* + * GPDB: Convenience function to remove the pg_class.reloptions field for a given relation. + */ +static void +remove_rel_opts(Relation rel) +{ + Datum val[Natts_pg_class] = {0}; + bool null[Natts_pg_class] = {0}; + bool repl[Natts_pg_class] = {0}; + Relation classrel; + HeapTuple tup; + + classrel = table_open(RelationRelationId, RowExclusiveLock); + + tup = SearchSysCacheCopy1(RELOID, RelationGetRelid(rel)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for relation %u", RelationGetRelid(rel)); + + val[Anum_pg_class_reloptions - 1] = (Datum) 0; + null[Anum_pg_class_reloptions - 1] = true; + repl[Anum_pg_class_reloptions - 1] = true; + + tup = heap_modify_tuple(tup, RelationGetDescr(classrel), + val, null, repl); + CatalogTupleUpdate(classrel, &tup->t_self, tup); + + heap_freetuple(tup); + + table_close(classrel, RowExclusiveLock); + + CommandCounterIncrement(); +} + static RangeVar * make_temp_table_name(Relation rel, BackendId id) { diff --git a/src/test/regress/expected/alter_table_set_am.out b/src/test/regress/expected/alter_table_set_am.out index 58e0f934d4b..701496a0f3d 100644 --- a/src/test/regress/expected/alter_table_set_am.out +++ b/src/test/regress/expected/alter_table_set_am.out @@ -714,7 +714,7 @@ SELECT count(*) FROM gp_toolkit.__gp_aocsseg('ao2co3'); SELECT * FROM gp_toolkit.__gp_aoblkdir('ao2co3'); tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count ----------+-------+----------------+----------+--------------+-------------+----------- + (0 rows) -- pg_attribute_encoding should have columns for the AOCO table @@ -735,8 +735,8 @@ SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'ao2co%'; relname | amname | reloptions ---------+-----------+----------------------------------- - ao2co | ao_column | {blocksize=65536,compresslevel=5} - ao2co2 | ao_column | {blocksize=65536,compresslevel=5} + ao2co | ao_column | + ao2co2 | ao_column | ao2co3 | ao_column | {blocksize=32768,compresslevel=3} ao2co4 | ao_column | {blocksize=32768,compresslevel=3} (4 rows) @@ -746,8 +746,8 @@ SELECT c.relname,a.blocksize,a.compresslevel,a.checksum,a.compresstype,a.columns FROM pg_appendonly a, pg_class c WHERE a.relid = c.oid AND relname like ('ao2co%'); relname | blocksize | compresslevel | checksum | compresstype | columnstore ---------+-----------+---------------+----------+--------------+------------- - ao2co | 65536 | 5 | t | zlib | t - ao2co2 | 65536 | 5 | t | zlib | t + ao2co | 32768 | 0 | t | | t + ao2co2 | 32768 | 0 | t | | t ao2co3 | 32768 | 3 | t | zlib | t ao2co4 | 32768 | 3 | t | zlib | t (4 rows) diff --git a/src/test/regress/sql/alter_table_set_am.sql b/src/test/regress/sql/alter_table_set_am.sql index d2d65f65d3c..11a0b90d1c0 100644 --- a/src/test/regress/sql/alter_table_set_am.sql +++ b/src/test/regress/sql/alter_table_set_am.sql @@ -417,6 +417,167 @@ DROP TABLE ao2co2; DROP TABLE ao2co3; DROP TABLE ao2co4; +-- Scenario 6: AOCO to Heap +SET gp_default_storage_options = 'blocksize=65536, compresstype=zlib, compresslevel=5, checksum=true'; + +CREATE TABLE co2heap(a int, b int) WITH (appendonly=true, orientation=column); +CREATE TABLE co2heap2(a int, b int) WITH (appendonly=true, orientation=column); +CREATE TABLE co2heap3(a int, b int) WITH (appendonly=true, orientation=column); +CREATE TABLE co2heap4(a int, b int) WITH (appendonly=true, orientation=column); +CREATE INDEX aoi ON co2heap(b); + +INSERT INTO co2heap SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2heap2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2heap3 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2heap4 SELECT i,i FROM generate_series(1,5) i; + +-- Prior-ATSETAM checks: +-- Check once that the AO tables have the custom reloptions +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2heap%'; +-- Check once that the AO tables have relfrozenxid = 0 +SELECT relname, relfrozenxid FROM pg_class WHERE relname LIKE 'co2heap%'; +-- Check once that the pg_attribute_encoding has entries for the AOCO tables. +SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2heap%'; + +CREATE TEMP TABLE relfilebeforeco2heap AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2heap%' + UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'co2heap%' ORDER BY segid; + +-- Various cases of altering AOCO to AO: +-- 1. Basic ATSETAMs: +ALTER TABLE co2heap SET ACCESS METHOD heap; +ALTER TABLE co2heap2 SET WITH (appendoptimized=false); +-- 2. ATSETAM with reloptions: +ALTER TABLE co2heap3 SET ACCESS METHOD heap WITH (fillfactor=70); +ALTER TABLE co2heap4 SET WITH (appendoptimized=false, fillfactor=70); + +-- The tables and indexes should have been rewritten (should have different relfilenodes) +CREATE TEMP TABLE relfileafterco2heap AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2heap%' + UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'co2heap%' ORDER BY segid; + +SELECT * FROM relfilebeforeco2heap INTERSECT SELECT * FROM relfileafterco2heap; + +-- Check data is intact +SELECT count(*) FROM co2heap; +SELECT count(*) FROM co2heap2; +SELECT count(*) FROM co2heap3; +SELECT count(*) FROM co2heap4; + +-- No AO aux tables should be left. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- No need to test the other ones created by the alternative syntax SET WITH(). +SELECT * FROM gp_toolkit.__gp_aoseg('co2heap'); +SELECT * FROM gp_toolkit.__gp_aovisimap('co2heap'); +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('co2heap'); +SELECT * FROM gp_toolkit.__gp_aoblkdir('co2heap'); +SELECT * FROM gp_toolkit.__gp_aoseg('co2heap3'); +SELECT * FROM gp_toolkit.__gp_aovisimap('co2heap3'); +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('co2heap3'); +SELECT * FROM gp_toolkit.__gp_aoblkdir('co2heap3'); + +-- No pg_appendonly entries should be left too +SELECT c.relname FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2heap%' AND c.oid = p.relid; + +-- The altered tables should have heap AM. +SELECT c.relname, a.amname FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'co2heap%'; + +-- The new heap tables shouldn't have the old AO table's reloptions +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2heap%'; + +-- The new heap tables should have a valid relfrozenxid +SELECT relname, relfrozenxid <> '0' FROM pg_class WHERE relname LIKE 'co2heap%'; + +-- The pg_attribute_encoding entries for the altered tables should have all gone. +SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2heap%'; + +DROP TABLE co2heap; +DROP TABLE co2heap2; +DROP TABLE co2heap3; +DROP TABLE co2heap4; + +-- Scenario 7: AOCO to AO +CREATE TABLE co2ao(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE TABLE co2ao2(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE TABLE co2ao3(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE TABLE co2ao4(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE INDEX aoi ON co2ao(b); +CREATE INDEX aoi2 ON co2ao3(b); + +INSERT INTO co2ao SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2ao2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2ao3 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2ao4 SELECT i,i FROM generate_series(1,5) i; + +-- Prior-ATSETAM checks: +-- Check once that the AOCO tables have the custom reloptions +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; +-- Check once that pg_appendonly has expected entries. +SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; +-- Check once that the pg_attribute_encoding has entries for the AOCO tables. +SELECT c.relname, a.* FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; + +CREATE TEMP TABLE relfilebeforeco2ao AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2ao%' + UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'co2ao%' ORDER BY segid; + +-- Various cases of altering AOCO to AO: +-- 1. Basic ATSETAMs: +ALTER TABLE co2ao SET ACCESS METHOD ao_row; +ALTER TABLE co2ao2 SET WITH (appendoptimized=true); +-- 2. ATSETAM with reloptions: +ALTER TABLE co2ao3 SET ACCESS METHOD ao_row WITH (compresstype=zlib, compresslevel=7); +ALTER TABLE co2ao4 SET WITH (appendoptimized=true, compresstype=zlib, compresslevel=7); + +-- The tables and indexes should have been rewritten (should have different relfilenodes) +CREATE TEMP TABLE relfileafterco2ao AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2ao%' + UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'co2ao%' ORDER BY segid; + +SELECT * FROM relfilebeforeco2ao INTERSECT SELECT * FROM relfileafterco2ao; + +DROP TABLE relfilebeforeco2ao; +DROP TABLE relfileafterco2ao; + +-- Check data is intact +SELECT count(*) FROM co2ao; +SELECT count(*) FROM co2ao2; +SELECT count(*) FROM co2ao3; +SELECT count(*) FROM co2ao4; + +-- AO aux tables should still be there. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- No need to test the other ones created by the alternative syntax SET WITH(). +SELECT * FROM gp_toolkit.__gp_aoseg('co2ao'); +SELECT * FROM gp_toolkit.__gp_aovisimap('co2ao'); +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('co2ao'); +SELECT * FROM gp_toolkit.__gp_aoblkdir('co2ao'); +SELECT * FROM gp_toolkit.__gp_aoseg('co2ao3'); +SELECT * FROM gp_toolkit.__gp_aovisimap('co2ao3'); +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('co2ao3'); +SELECT * FROM gp_toolkit.__gp_aoblkdir('co2ao3'); + +-- pg_appendonly entries should be still be there, but options has changed accordingly. +SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; + +-- The altered tables should show AOCO AM. +SELECT c.relname, a.amname FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'co2ao%'; + +-- The new tables should have new reloptions. +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; + +-- The pg_attribute_encoding entries for the altered tables should have all gone. +SELECT c.relname, a.* FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; + +DROP TABLE co2ao; +DROP TABLE co2ao2; +DROP TABLE co2ao3; +DROP TABLE co2ao4; + -- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". -- The following cases will cover all variations of such iterations: -- 1. Heap->AO->Heap->AO From ef28ad5928dc35b4321c5a68b01acd2707044a23 Mon Sep 17 00:00:00 2001 From: Huansong Fu Date: Thu, 11 Aug 2022 14:14:42 -0700 Subject: [PATCH 4/8] ALTER TABLE SET ACCESS METHOD: AOCO->AO support As part of the ATSETAM support, this commit adds support for changing AM of a table from AOCO to AO. E.g.: ``` CREATE TABLE foo (appendonly=true, orientation=column); ALTER TABLE foo SET ACCESS METHOD ao_row; -- Or: ALTER TABLE foo SET WITH (appendoptimized=true); ``` Optionally, user can specify reloptions in a WITH clause too, e.g.: ``` ALTER TABLE foo SET ACCESS METHOD ao_row WITH (compresslevel=7); ``` Additionally for the regress test: 1. Add more cases for altering with the same existing AM; 2. Add more cases for the "Final scenario" in the test. --- src/backend/catalog/pg_appendonly.c | 7 +- src/backend/commands/tablecmds.c | 10 +- .../regress/expected/alter_table_set_am.out | 438 +++++++++++++++++- src/test/regress/sql/alter_table_set_am.sql | 82 ++-- 4 files changed, 480 insertions(+), 57 deletions(-) diff --git a/src/backend/catalog/pg_appendonly.c b/src/backend/catalog/pg_appendonly.c index 8ec58f74f34..7dd08ae5cfb 100644 --- a/src/backend/catalog/pg_appendonly.c +++ b/src/backend/catalog/pg_appendonly.c @@ -596,9 +596,10 @@ ATAOEntries(Form_pg_class relform1, Form_pg_class relform2, errmsg("alter table does not support switch from AOCO to Heap"))); break; case AO_ROW_TABLE_AM_OID: - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("alter table does not support switch from AOCO to AO"))); + /* For pg_appendonly entries, it's same as AO->AO/CO. */ + SwapAppendonlyEntries(relform1->oid, relform2->oid); + /* For pg_attribute_encoding entries, it's same as AOCO->heap.*/ + RemoveAttributeEncodingsByRelid(relform1->oid); break; case AO_COLUMN_TABLE_AM_OID: swapAppendonlyEntriesUsingTAM(relform1, relform2, frozenXid, cutoffMulti); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 901a471c7c2..1cbac7fea1c 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -585,7 +585,7 @@ static bool prebuild_temp_table(Relation rel, RangeVar *tmpname, DistributedBy * static void checkATSetDistributedByStandalone(AlteredTableInfo *tab, Relation rel); static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOptions); -static void remove_rel_opts(Relation rel); +static void clear_rel_opts(Relation rel); /* ---------------------------------------------------------------- @@ -16103,7 +16103,7 @@ ATPrepSetTableSpace(AlteredTableInfo *tab, Relation rel, const char *tablespacen * * GPDB specific arguments: * aoopt_changed: whether any AO storage options have been changed in this function. - * am_change_heap_ao: whether we are changing the AM from heap->AO/CO or vice-versa. + * valid_as_ao: whether we validate teh reloptions as AO tables. */ static void ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, @@ -16130,7 +16130,7 @@ ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, if (OidIsValid(newAccessMethod)) { newAM = GetTableAmRoutineByAmId(newAccessMethod); - remove_rel_opts(rel); + clear_rel_opts(rel); } else newAM = NULL; @@ -18002,10 +18002,10 @@ get_rel_opts(Relation rel) } /* - * GPDB: Convenience function to remove the pg_class.reloptions field for a given relation. + * GPDB: Convenience function to clear the pg_class.reloptions field for a given relation. */ static void -remove_rel_opts(Relation rel) +clear_rel_opts(Relation rel) { Datum val[Natts_pg_class] = {0}; bool null[Natts_pg_class] = {0}; diff --git a/src/test/regress/expected/alter_table_set_am.out b/src/test/regress/expected/alter_table_set_am.out index 701496a0f3d..a8da1c39f72 100644 --- a/src/test/regress/expected/alter_table_set_am.out +++ b/src/test/regress/expected/alter_table_set_am.out @@ -1,31 +1,51 @@ -- Check changing table access method --- Scenario 1: Heap to Heap -CREATE TABLE heap2heap(a int, b int) DISTRIBUTED BY (a); -CREATE TABLE heap2heap2(a int, b int) DISTRIBUTED BY (a); -INSERT INTO heap2heap SELECT i,i FROM generate_series(1,5) i; -INSERT INTO heap2heap2 SELECT i,i FROM generate_series(1,5) i; -CREATE TEMP TABLE relfilebeforeheap AS - SELECT -1 segid, relfilenode FROM pg_class WHERE relname in ('heap2heap', 'heap2heap2') +-- Scenario 1: Changing to the same AM: it should have no effect but +-- make sure it doesn't rewrite table or blow up existing reloptions: +CREATE TABLE sameam_heap(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); +CREATE TABLE sameam_heap2(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); +CREATE TABLE sameam_ao(a int, b int) WITH (appendoptimized=true, orientation=row, compresstype=zlib, compresslevel=3); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE sameam_co(a int, b int) WITH (appendoptimized=true, orientation=column, compresstype=rle_type, compresslevel=3); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO sameam_heap SELECT i,i FROM generate_series(1,5) i; +INSERT INTO sameam_heap2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO sameam_ao SELECT i,i FROM generate_series(1,5) i; +INSERT INTO sameam_co SELECT i,i FROM generate_series(1,5) i; +CREATE TEMP TABLE relfilebeforesameam AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'sameam_%' UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') - WHERE relname in ('heap2heap', 'heap2heap2') ORDER BY segid; + WHERE relname LIKE 'sameam_%' ORDER BY segid; NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'segid' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -- changing to the same access method shouldn't rewrite the table -- (i.e. the relfilenodes shouldn't change) -ALTER TABLE heap2heap SET ACCESS METHOD heap; -ALTER TABLE heap2heap2 SET WITH (appendoptimized=false); -CREATE TEMP TABLE relfileafterheap AS - SELECT -1 segid, relfilenode FROM pg_class WHERE relname in ('heap2heap', 'heap2heap2') +ALTER TABLE sameam_heap SET ACCESS METHOD heap; +ALTER TABLE sameam_heap2 SET WITH (appendoptimized=false); -- Alternative syntax of ATSETAM +ALTER TABLE sameam_ao SET ACCESS METHOD ao_row; +ALTER TABLE sameam_co SET ACCESS METHOD ao_column; +CREATE TEMP TABLE relfileaftersameam AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'sameam_%' UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') - WHERE relname in ('heap2heap', 'heap2heap2') ORDER BY segid; + WHERE relname LIKE 'sameam_%' ORDER BY segid; NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'segid' as the Greenplum Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -- relfilenodes shouldn't change -SELECT count(*) FROM (SELECT * FROM relfilebeforeheap UNION SELECT * FROM relfileafterheap)a; - count -------- - 8 -(1 row) +SELECT * FROM relfilebeforesameam EXCEPT SELECT * FROM relfileaftersameam; + segid | relfilenode +-------+------------- +(0 rows) + +-- reloptions should remain the same +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'sameam_%'; + relname | reloptions +--------------+----------------------------------------- + sameam_ao | {compresstype=zlib,compresslevel=3} + sameam_co | {compresstype=rle_type,compresslevel=3} + sameam_heap | {fillfactor=70} + sameam_heap2 | {fillfactor=70} +(4 rows) -- Scenario 2: Heap to AO CREATE TABLE heap2ao(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); @@ -714,7 +734,7 @@ SELECT count(*) FROM gp_toolkit.__gp_aocsseg('ao2co3'); SELECT * FROM gp_toolkit.__gp_aoblkdir('ao2co3'); tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count - +---------+-------+----------------+----------+--------------+-------------+----------- (0 rows) -- pg_attribute_encoding should have columns for the AOCO table @@ -756,10 +776,373 @@ DROP TABLE ao2co; DROP TABLE ao2co2; DROP TABLE ao2co3; DROP TABLE ao2co4; +-- Scenario 6: AOCO to Heap +SET gp_default_storage_options = 'blocksize=65536, compresstype=zlib, compresslevel=5, checksum=true'; +CREATE TABLE co2heap(a int, b int) WITH (appendonly=true, orientation=column); +CREATE TABLE co2heap2(a int, b int) WITH (appendonly=true, orientation=column); +CREATE TABLE co2heap3(a int, b int) WITH (appendonly=true, orientation=column); +CREATE TABLE co2heap4(a int, b int) WITH (appendonly=true, orientation=column); +CREATE INDEX aoi ON co2heap(b); +INSERT INTO co2heap SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2heap2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2heap3 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2heap4 SELECT i,i FROM generate_series(1,5) i; +-- Prior-ATSETAM checks: +-- Check once that the AO tables have the custom reloptions +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2heap%'; + relname | reloptions +----------+----------------------------------- + co2heap | {blocksize=65536,compresslevel=5} + co2heap2 | {blocksize=65536,compresslevel=5} + co2heap3 | {blocksize=65536,compresslevel=5} + co2heap4 | {blocksize=65536,compresslevel=5} +(4 rows) + +-- Check once that the AO tables have relfrozenxid = 0 +SELECT relname, relfrozenxid FROM pg_class WHERE relname LIKE 'co2heap%'; + relname | relfrozenxid +----------+-------------- + co2heap | 0 + co2heap2 | 0 + co2heap3 | 0 + co2heap4 | 0 +(4 rows) + +-- Check once that the pg_attribute_encoding has entries for the AOCO tables. +SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2heap%'; + relname | attnum | attoptions +----------+--------+----------------------------------------------------- + co2heap | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + co2heap | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + co2heap2 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + co2heap2 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + co2heap3 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + co2heap3 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + co2heap4 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + co2heap4 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} +(8 rows) + +CREATE TEMP TABLE relfilebeforeco2heap AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2heap%' + UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'co2heap%' ORDER BY segid; +-- Various cases of altering AOCO to AO: +-- 1. Basic ATSETAMs: +ALTER TABLE co2heap SET ACCESS METHOD heap; +ALTER TABLE co2heap2 SET WITH (appendoptimized=false); +-- 2. ATSETAM with reloptions: +ALTER TABLE co2heap3 SET ACCESS METHOD heap WITH (fillfactor=70); +ALTER TABLE co2heap4 SET WITH (appendoptimized=false, fillfactor=70); +-- The tables and indexes should have been rewritten (should have different relfilenodes) +CREATE TEMP TABLE relfileafterco2heap AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2heap%' + UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'co2heap%' ORDER BY segid; +SELECT * FROM relfilebeforeco2heap INTERSECT SELECT * FROM relfileafterco2heap; + segid | relfilenode +-------+------------- +(0 rows) + +-- Check data is intact +SELECT count(*) FROM co2heap; + count +------- + 5 +(1 row) + +SELECT count(*) FROM co2heap2; + count +------- + 5 +(1 row) + +SELECT count(*) FROM co2heap3; + count +------- + 5 +(1 row) + +SELECT count(*) FROM co2heap4; + count +------- + 5 +(1 row) + +-- No AO aux tables should be left. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- No need to test the other ones created by the alternative syntax SET WITH(). +SELECT * FROM gp_toolkit.__gp_aoseg('co2heap'); +ERROR: 'co2heap' is not an append-only row relation +SELECT * FROM gp_toolkit.__gp_aovisimap('co2heap'); +ERROR: function not supported on relation +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('co2heap'); +ERROR: 'co2heap' is not an append-only columnar relation +SELECT * FROM gp_toolkit.__gp_aoblkdir('co2heap'); +ERROR: function not supported on non append-optimized relation +SELECT * FROM gp_toolkit.__gp_aoseg('co2heap3'); +ERROR: 'co2heap3' is not an append-only row relation +SELECT * FROM gp_toolkit.__gp_aovisimap('co2heap3'); +ERROR: function not supported on relation +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('co2heap3'); +ERROR: 'co2heap3' is not an append-only columnar relation +SELECT * FROM gp_toolkit.__gp_aoblkdir('co2heap3'); +ERROR: function not supported on non append-optimized relation +-- No pg_appendonly entries should be left too +SELECT c.relname FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2heap%' AND c.oid = p.relid; + relname +--------- +(0 rows) + +-- The altered tables should have heap AM. +SELECT c.relname, a.amname FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'co2heap%'; + relname | amname +----------+-------- + co2heap | heap + co2heap2 | heap + co2heap3 | heap + co2heap4 | heap +(4 rows) + +-- The new heap tables shouldn't have the old AO table's reloptions +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2heap%'; + relname | reloptions +----------+----------------- + co2heap | + co2heap2 | + co2heap3 | {fillfactor=70} + co2heap4 | {fillfactor=70} +(4 rows) + +-- The new heap tables should have a valid relfrozenxid +SELECT relname, relfrozenxid <> '0' FROM pg_class WHERE relname LIKE 'co2heap%'; + relname | ?column? +----------+---------- + co2heap | t + co2heap2 | t + co2heap3 | t + co2heap4 | t +(4 rows) + +-- The pg_attribute_encoding entries for the altered tables should have all gone. +SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2heap%'; + relname | attnum | attoptions +---------+--------+------------ +(0 rows) + +DROP TABLE co2heap; +DROP TABLE co2heap2; +DROP TABLE co2heap3; +DROP TABLE co2heap4; +-- Scenario 7: AOCO to AO +CREATE TABLE co2ao(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE TABLE co2ao2(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE TABLE co2ao3(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE TABLE co2ao4(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE INDEX aoi ON co2ao(b); +CREATE INDEX aoi2 ON co2ao3(b); +INSERT INTO co2ao SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2ao2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2ao3 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO co2ao4 SELECT i,i FROM generate_series(1,5) i; +-- Prior-ATSETAM checks: +-- Check once that the AOCO tables have the custom reloptions +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; + relname | reloptions +---------+--------------------------------------------------------- + co2ao | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao3 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao4 | {compresstype=rle_type,compresslevel=3,blocksize=65536} +(4 rows) + +-- Check once that pg_appendonly has expected entries. +SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; + relname | compresstype | compresslevel | blocksize +---------+--------------+---------------+----------- + co2ao | rle_type | 3 | 65536 + co2ao2 | rle_type | 3 | 65536 + co2ao3 | rle_type | 3 | 65536 + co2ao4 | rle_type | 3 | 65536 +(4 rows) + +-- Check once that the pg_attribute_encoding has entries for the AOCO tables. +SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; + relname | attnum | attoptions +---------+--------+--------------------------------------------------------- + co2ao | 1 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao | 2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao2 | 1 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao2 | 2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao3 | 1 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao3 | 2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao4 | 1 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao4 | 2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} +(8 rows) + +-- Check once on the aoblkdirs +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count +---------------+---------+-------+----------------+----------+--------------+-------------+----------- + 0 | (0,1) | 1 | 0 | 0 | 1 | 0 | 3 + 0 | (0,2) | 1 | 1 | 0 | 1 | 0 | 3 + 1 | (0,1) | 1 | 0 | 0 | 1 | 0 | 1 + 1 | (0,2) | 1 | 1 | 0 | 1 | 0 | 1 + 2 | (0,1) | 1 | 0 | 0 | 1 | 0 | 1 + 2 | (0,2) | 1 | 1 | 0 | 1 | 0 | 1 +(6 rows) + +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao3')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count +---------------+---------+-------+----------------+----------+--------------+-------------+----------- + 0 | (0,1) | 1 | 0 | 0 | 1 | 0 | 3 + 0 | (0,2) | 1 | 1 | 0 | 1 | 0 | 3 + 1 | (0,1) | 1 | 0 | 0 | 1 | 0 | 1 + 1 | (0,2) | 1 | 1 | 0 | 1 | 0 | 1 + 2 | (0,1) | 1 | 0 | 0 | 1 | 0 | 1 + 2 | (0,2) | 1 | 1 | 0 | 1 | 0 | 1 +(6 rows) + +CREATE TEMP TABLE relfilebeforeco2ao AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2ao%' + UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'co2ao%' ORDER BY segid; +-- Various cases of altering AOCO to AO: +-- 1. Basic ATSETAMs: +ALTER TABLE co2ao SET ACCESS METHOD ao_row; +ALTER TABLE co2ao2 SET WITH (appendoptimized=true); +-- 2. ATSETAM with reloptions: +ALTER TABLE co2ao3 SET ACCESS METHOD ao_row WITH (compresstype=zlib, compresslevel=7); +ALTER TABLE co2ao4 SET WITH (appendoptimized=true, compresstype=zlib, compresslevel=7); +-- The tables and indexes should have been rewritten (should have different relfilenodes) +CREATE TEMP TABLE relfileafterco2ao AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2ao%' + UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') + WHERE relname LIKE 'co2ao%' ORDER BY segid; +SELECT * FROM relfilebeforeco2ao INTERSECT SELECT * FROM relfileafterco2ao; + segid | relfilenode +-------+------------- +(0 rows) + +DROP TABLE relfilebeforeco2ao; +DROP TABLE relfileafterco2ao; +-- Check data is intact +SELECT count(*) FROM co2ao; + count +------- + 5 +(1 row) + +SELECT count(*) FROM co2ao2; + count +------- + 5 +(1 row) + +SELECT count(*) FROM co2ao3; + count +------- + 5 +(1 row) + +SELECT count(*) FROM co2ao4; + count +------- + 5 +(1 row) + +-- AO aux tables should still be there, but AOCO seg tables are not. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- No need to test the other ones created by the alternative syntax SET WITH(). +SELECT * FROM gp_toolkit.__gp_aoseg('co2ao'); + segment_id | segno | eof | tupcount | varblockcount | eof_uncompressed | modcount | formatversion | state +------------+-------+-----+----------+---------------+------------------+----------+---------------+------- + 0 | 0 | 88 | 3 | 1 | 88 | 1 | 3 | 1 + 1 | 0 | 40 | 1 | 1 | 40 | 1 | 3 | 1 + 2 | 0 | 40 | 1 | 1 | 40 | 1 | 3 | 1 +(3 rows) + +SELECT * FROM gp_toolkit.__gp_aocsseg('co2ao'); +ERROR: 'co2ao' is not an append-only columnar relation +SELECT gp_segment_id, (gp_toolkit.__gp_aovisimap('co2ao')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tid | segno | row_num +---------------+-----+-------+--------- +(0 rows) + +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count +---------------+---------+-------+----------------+----------+--------------+-------------+----------- + 0 | (0,1) | 0 | 0 | 0 | 1 | 0 | 3 + 1 | (0,1) | 0 | 0 | 0 | 1 | 0 | 1 + 2 | (0,1) | 0 | 0 | 0 | 1 | 0 | 1 +(3 rows) + +SELECT * FROM gp_toolkit.__gp_aoseg('co2ao3'); + segment_id | segno | eof | tupcount | varblockcount | eof_uncompressed | modcount | formatversion | state +------------+-------+-----+----------+---------------+------------------+----------+---------------+------- + 0 | 0 | 72 | 3 | 1 | 88 | 1 | 3 | 1 + 1 | 0 | 40 | 1 | 1 | 40 | 1 | 3 | 1 + 2 | 0 | 40 | 1 | 1 | 40 | 1 | 3 | 1 +(3 rows) + +SELECT * FROM gp_toolkit.__gp_aocsseg('co2ao3'); +ERROR: 'co2ao3' is not an append-only columnar relation +SELECT gp_segment_id, (gp_toolkit.__gp_aovisimap('co2ao3')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tid | segno | row_num +---------------+-----+-------+--------- +(0 rows) + +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao3')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count +---------------+---------+-------+----------------+----------+--------------+-------------+----------- + 0 | (0,1) | 0 | 0 | 0 | 1 | 0 | 3 + 1 | (0,1) | 0 | 0 | 0 | 1 | 0 | 1 + 2 | (0,1) | 0 | 0 | 0 | 1 | 0 | 1 +(3 rows) + +-- pg_appendonly entries should be still be there, but options has changed accordingly. +SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; + relname | compresstype | compresslevel | blocksize +---------+--------------+---------------+----------- + co2ao | | 0 | 32768 + co2ao2 | | 0 | 32768 + co2ao3 | zlib | 7 | 32768 + co2ao4 | zlib | 7 | 32768 +(4 rows) + +-- The altered tables should show AO AM. +SELECT c.relname, a.amname FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'co2ao%'; + relname | amname +---------+-------- + co2ao | ao_row + co2ao2 | ao_row + co2ao3 | ao_row + co2ao4 | ao_row +(4 rows) + +-- Only the new tables altered w/ reloptions supplies should have reloptions. +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; + relname | reloptions +---------+------------------------------------- + co2ao | + co2ao2 | + co2ao3 | {compresstype=zlib,compresslevel=7} + co2ao4 | {compresstype=zlib,compresslevel=7} +(4 rows) + +-- The pg_attribute_encoding entries for the altered tables should have all gone. +SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; + relname | attnum | attoptions +---------+--------+------------ +(0 rows) + +DROP TABLE co2ao; +DROP TABLE co2ao2; +DROP TABLE co2ao3; +DROP TABLE co2ao4; -- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". -- The following cases will cover all variations of such iterations: -- 1. Heap->AO->Heap->AO --- (TODO) 2. AO->AOCO->AO->AOCO +-- 2. AO->AOCO->AO->AOCO -- (TODO) 3. Heap->AOCO->Heap->AOCO -- 1. Heap->AO->Heap->AO CREATE TABLE heapao(a int, b int); @@ -776,3 +1159,18 @@ SELECT count(*) FROM heapao; (1 row) DROP TABLE heapao; +-- 2. AO->AOCO->AO->AOCO +CREATE TABLE aoco(a int, b int) with (appendoptimized=true); +CREATE INDEX aocoindex ON aoco(b); +INSERT INTO aoco SELECT i,i FROM generate_series(1,5) i; +ALTER TABLE aoco SET ACCESS METHOD ao_column; +ALTER TABLE aoco SET ACCESS METHOD ao_row; +ALTER TABLE aoco SET ACCESS METHOD ao_column; +-- Just checking data is intact. +SELECT count(*) FROM aoco; + count +------- + 5 +(1 row) + +DROP TABLE aoco; diff --git a/src/test/regress/sql/alter_table_set_am.sql b/src/test/regress/sql/alter_table_set_am.sql index 11a0b90d1c0..e97903b2ed8 100644 --- a/src/test/regress/sql/alter_table_set_am.sql +++ b/src/test/regress/sql/alter_table_set_am.sql @@ -1,30 +1,38 @@ -- Check changing table access method --- Scenario 1: Heap to Heap -CREATE TABLE heap2heap(a int, b int) DISTRIBUTED BY (a); -CREATE TABLE heap2heap2(a int, b int) DISTRIBUTED BY (a); - -INSERT INTO heap2heap SELECT i,i FROM generate_series(1,5) i; -INSERT INTO heap2heap2 SELECT i,i FROM generate_series(1,5) i; - -CREATE TEMP TABLE relfilebeforeheap AS - SELECT -1 segid, relfilenode FROM pg_class WHERE relname in ('heap2heap', 'heap2heap2') +-- Scenario 1: Changing to the same AM: it should have no effect but +-- make sure it doesn't rewrite table or blow up existing reloptions: +CREATE TABLE sameam_heap(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); +CREATE TABLE sameam_heap2(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); +CREATE TABLE sameam_ao(a int, b int) WITH (appendoptimized=true, orientation=row, compresstype=zlib, compresslevel=3); +CREATE TABLE sameam_co(a int, b int) WITH (appendoptimized=true, orientation=column, compresstype=rle_type, compresslevel=3); + +INSERT INTO sameam_heap SELECT i,i FROM generate_series(1,5) i; +INSERT INTO sameam_heap2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO sameam_ao SELECT i,i FROM generate_series(1,5) i; +INSERT INTO sameam_co SELECT i,i FROM generate_series(1,5) i; + +CREATE TEMP TABLE relfilebeforesameam AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'sameam_%' UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') - WHERE relname in ('heap2heap', 'heap2heap2') ORDER BY segid; + WHERE relname LIKE 'sameam_%' ORDER BY segid; -- changing to the same access method shouldn't rewrite the table -- (i.e. the relfilenodes shouldn't change) -ALTER TABLE heap2heap SET ACCESS METHOD heap; -ALTER TABLE heap2heap2 SET WITH (appendoptimized=false); +ALTER TABLE sameam_heap SET ACCESS METHOD heap; +ALTER TABLE sameam_heap2 SET WITH (appendoptimized=false); -- Alternative syntax of ATSETAM +ALTER TABLE sameam_ao SET ACCESS METHOD ao_row; +ALTER TABLE sameam_co SET ACCESS METHOD ao_column; -CREATE TEMP TABLE relfileafterheap AS - SELECT -1 segid, relfilenode FROM pg_class WHERE relname in ('heap2heap', 'heap2heap2') +CREATE TEMP TABLE relfileaftersameam AS + SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'sameam_%' UNION SELECT gp_segment_id segid, relfilenode FROM gp_dist_random('pg_class') - WHERE relname in ('heap2heap', 'heap2heap2') ORDER BY segid; + WHERE relname LIKE 'sameam_%' ORDER BY segid; -- relfilenodes shouldn't change -SELECT count(*) FROM (SELECT * FROM relfilebeforeheap UNION SELECT * FROM relfileafterheap)a; - +SELECT * FROM relfilebeforesameam EXCEPT SELECT * FROM relfileaftersameam; +-- reloptions should remain the same +SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'sameam_%'; -- Scenario 2: Heap to AO CREATE TABLE heap2ao(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); @@ -517,7 +525,10 @@ SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; -- Check once that pg_appendonly has expected entries. SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; -- Check once that the pg_attribute_encoding has entries for the AOCO tables. -SELECT c.relname, a.* FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; +SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; +-- Check once on the aoblkdirs +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao')).* FROM gp_dist_random('gp_id'); +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao3')).* FROM gp_dist_random('gp_id'); CREATE TEMP TABLE relfilebeforeco2ao AS SELECT -1 segid, relfilenode FROM pg_class WHERE relname LIKE 'co2ao%' @@ -549,29 +560,29 @@ SELECT count(*) FROM co2ao2; SELECT count(*) FROM co2ao3; SELECT count(*) FROM co2ao4; --- AO aux tables should still be there. +-- AO aux tables should still be there, but AOCO seg tables are not. -- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. -- No need to test the other ones created by the alternative syntax SET WITH(). SELECT * FROM gp_toolkit.__gp_aoseg('co2ao'); -SELECT * FROM gp_toolkit.__gp_aovisimap('co2ao'); -SELECT count(*) FROM gp_toolkit.__gp_aocsseg('co2ao'); -SELECT * FROM gp_toolkit.__gp_aoblkdir('co2ao'); +SELECT * FROM gp_toolkit.__gp_aocsseg('co2ao'); +SELECT gp_segment_id, (gp_toolkit.__gp_aovisimap('co2ao')).* FROM gp_dist_random('gp_id'); +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao')).* FROM gp_dist_random('gp_id'); SELECT * FROM gp_toolkit.__gp_aoseg('co2ao3'); -SELECT * FROM gp_toolkit.__gp_aovisimap('co2ao3'); -SELECT count(*) FROM gp_toolkit.__gp_aocsseg('co2ao3'); -SELECT * FROM gp_toolkit.__gp_aoblkdir('co2ao3'); +SELECT * FROM gp_toolkit.__gp_aocsseg('co2ao3'); +SELECT gp_segment_id, (gp_toolkit.__gp_aovisimap('co2ao3')).* FROM gp_dist_random('gp_id'); +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao3')).* FROM gp_dist_random('gp_id'); -- pg_appendonly entries should be still be there, but options has changed accordingly. SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; --- The altered tables should show AOCO AM. +-- The altered tables should show AO AM. SELECT c.relname, a.amname FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'co2ao%'; --- The new tables should have new reloptions. +-- Only the new tables altered w/ reloptions supplies should have reloptions. SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; -- The pg_attribute_encoding entries for the altered tables should have all gone. -SELECT c.relname, a.* FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; +SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; DROP TABLE co2ao; DROP TABLE co2ao2; @@ -581,7 +592,7 @@ DROP TABLE co2ao4; -- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". -- The following cases will cover all variations of such iterations: -- 1. Heap->AO->Heap->AO --- (TODO) 2. AO->AOCO->AO->AOCO +-- 2. AO->AOCO->AO->AOCO -- (TODO) 3. Heap->AOCO->Heap->AOCO -- 1. Heap->AO->Heap->AO @@ -597,3 +608,16 @@ ALTER TABLE heapao SET ACCESS METHOD ao_row; SELECT count(*) FROM heapao; DROP TABLE heapao; +-- 2. AO->AOCO->AO->AOCO +CREATE TABLE aoco(a int, b int) with (appendoptimized=true); +CREATE INDEX aocoindex ON aoco(b); +INSERT INTO aoco SELECT i,i FROM generate_series(1,5) i; + +ALTER TABLE aoco SET ACCESS METHOD ao_column; +ALTER TABLE aoco SET ACCESS METHOD ao_row; +ALTER TABLE aoco SET ACCESS METHOD ao_column; + +-- Just checking data is intact. +SELECT count(*) FROM aoco; +DROP TABLE aoco; + From ffc84ac8bbcea712f27087870b213ee4df893d27 Mon Sep 17 00:00:00 2001 From: Huansong Fu Date: Wed, 17 Aug 2022 14:57:57 -0700 Subject: [PATCH 5/8] Fix an issue with rle_type when changing table from AO to AOCO It is found that when changing from AO to AOCO we cannot specify `compresstype=rle_type` which would complain that it is not applicable to AO table, despite that it should be a valid usage. This is because during setting new reloptions, we miss the information about the new AM being AOCO, and the code behavior is to validate the reloptions as AO tables. Fixing it by passing the new AM into ATExecSetRelOptions() based on which we can validate the reloptions. If there is no AM change, an InvalidOid will be passed in and we will use the current table AM to validate the reloptions. --- src/backend/commands/tablecmds.c | 6 +++- .../regress/expected/alter_table_set_am.out | 28 +++++++++---------- src/test/regress/sql/alter_table_set_am.sql | 4 +-- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 1cbac7fea1c..fdc30e59200 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -16103,7 +16103,7 @@ ATPrepSetTableSpace(AlteredTableInfo *tab, Relation rel, const char *tablespacen * * GPDB specific arguments: * aoopt_changed: whether any AO storage options have been changed in this function. - * valid_as_ao: whether we validate teh reloptions as AO tables. + * newam: the new AM if we will change the table AM. It's InvalidOid if no change is needed. */ static void ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, @@ -16121,6 +16121,10 @@ ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, bool repl_repl[Natts_pg_class]; const TableAmRoutine * newAM; static char *validnsps[] = HEAP_RELOPT_NAMESPACES; + Oid tableam; + + /* Get the new table AM if applicable. Otherwise get the one from the reltion. */ + tableam = (newam != InvalidOid) ? newam : rel->rd_rel->relam; if (defList == NIL && operation != AT_ReplaceRelOptions) return; /* nothing to do */ diff --git a/src/test/regress/expected/alter_table_set_am.out b/src/test/regress/expected/alter_table_set_am.out index a8da1c39f72..9fc7ee3dab2 100644 --- a/src/test/regress/expected/alter_table_set_am.out +++ b/src/test/regress/expected/alter_table_set_am.out @@ -659,8 +659,8 @@ SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam -- Altering AO to AOCO with various syntaxes, reloptions: ALTER TABLE ao2co SET ACCESS METHOD ao_column; ALTER TABLE ao2co2 SET WITH (appendoptimized=true, orientation=column); -ALTER TABLE ao2co3 SET ACCESS METHOD ao_column WITH (blocksize=32768, compresslevel=3); -ALTER TABLE ao2co4 SET WITH (appendoptimized=true, orientation=column, blocksize=32768, compresslevel=3); +ALTER TABLE ao2co3 SET ACCESS METHOD ao_column WITH (blocksize=32768, compresstype=rle_type, compresslevel=3); +ALTER TABLE ao2co4 SET WITH (appendoptimized=true, orientation=column, blocksize=32768, compresstype=rle_type, compresslevel=3); -- The tables are rewritten CREATE TEMP TABLE relfileafterao AS SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'ao2co%' @@ -739,26 +739,26 @@ SELECT * FROM gp_toolkit.__gp_aoblkdir('ao2co3'); -- pg_attribute_encoding should have columns for the AOCO table SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid = c.oid AND c.relname LIKE 'ao2co%'; - relname | attnum | attoptions ----------+--------+----------------------------------------------------- + relname | attnum | attoptions +---------+--------+--------------------------------------------------------- ao2co | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} ao2co | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} ao2co2 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} ao2co2 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} - ao2co3 | 1 | {blocksize=32768,compresslevel=3,compresstype=zlib} - ao2co3 | 2 | {blocksize=32768,compresslevel=3,compresstype=zlib} - ao2co4 | 1 | {blocksize=32768,compresslevel=3,compresstype=zlib} - ao2co4 | 2 | {blocksize=32768,compresslevel=3,compresstype=zlib} + ao2co3 | 1 | {blocksize=32768,compresstype=rle_type,compresslevel=3} + ao2co3 | 2 | {blocksize=32768,compresstype=rle_type,compresslevel=3} + ao2co4 | 1 | {blocksize=32768,compresstype=rle_type,compresslevel=3} + ao2co4 | 2 | {blocksize=32768,compresstype=rle_type,compresslevel=3} (8 rows) -- AM and reloptions changed accordingly SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'ao2co%'; - relname | amname | reloptions ----------+-----------+----------------------------------- + relname | amname | reloptions +---------+-----------+--------------------------------------------------------- ao2co | ao_column | ao2co2 | ao_column | - ao2co3 | ao_column | {blocksize=32768,compresslevel=3} - ao2co4 | ao_column | {blocksize=32768,compresslevel=3} + ao2co3 | ao_column | {blocksize=32768,compresstype=rle_type,compresslevel=3} + ao2co4 | ao_column | {blocksize=32768,compresstype=rle_type,compresslevel=3} (4 rows) -- pg_appendonly should reflect the changes in reloptions @@ -768,8 +768,8 @@ FROM pg_appendonly a, pg_class c WHERE a.relid = c.oid AND relname like ('ao2co% ---------+-----------+---------------+----------+--------------+------------- ao2co | 32768 | 0 | t | | t ao2co2 | 32768 | 0 | t | | t - ao2co3 | 32768 | 3 | t | zlib | t - ao2co4 | 32768 | 3 | t | zlib | t + ao2co3 | 32768 | 3 | t | rle_type | t + ao2co4 | 32768 | 3 | t | rle_type | t (4 rows) DROP TABLE ao2co; diff --git a/src/test/regress/sql/alter_table_set_am.sql b/src/test/regress/sql/alter_table_set_am.sql index e97903b2ed8..52e3b127c7a 100644 --- a/src/test/regress/sql/alter_table_set_am.sql +++ b/src/test/regress/sql/alter_table_set_am.sql @@ -379,8 +379,8 @@ SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam -- Altering AO to AOCO with various syntaxes, reloptions: ALTER TABLE ao2co SET ACCESS METHOD ao_column; ALTER TABLE ao2co2 SET WITH (appendoptimized=true, orientation=column); -ALTER TABLE ao2co3 SET ACCESS METHOD ao_column WITH (blocksize=32768, compresslevel=3); -ALTER TABLE ao2co4 SET WITH (appendoptimized=true, orientation=column, blocksize=32768, compresslevel=3); +ALTER TABLE ao2co3 SET ACCESS METHOD ao_column WITH (blocksize=32768, compresstype=rle_type, compresslevel=3); +ALTER TABLE ao2co4 SET WITH (appendoptimized=true, orientation=column, blocksize=32768, compresstype=rle_type, compresslevel=3); -- The tables are rewritten CREATE TEMP TABLE relfileafterao AS From 43a0aa2e7921477eda676c7dbcc164a11bdd924e Mon Sep 17 00:00:00 2001 From: Divyesh Vanjare Date: Mon, 15 Aug 2022 16:01:41 -0700 Subject: [PATCH 6/8] ALTER TABLE SET ACCESS METHOD: Heap->AOCO support Currently adding support for the following cases: ``` CREATE TABLE foo; ALTER TABLE foo SET ACCESS METHOD ao_column; -- Or: ALTER TABLE foo SET WITH (appendonly=true, orientation=column); ``` Similar to other variations of ATSETAM commands, user can also specify reloptions in a WITH clause, such as: ``` ALTER TABLE foo SET ACCESS METHOD ao_column WITH (blocksize=65536); ``` If no reloptions are given, the new AOCO table will use the default reloptions for its column encoding options. If any reloption is given in the WITH clause, it will be recorded in the catalog and then used for the column encoding option too. --- src/backend/catalog/pg_appendonly.c | 4 +- src/backend/commands/cluster.c | 2 +- .../regress/expected/alter_table_set_am.out | 205 +++++++++++++++++- src/test/regress/sql/alter_table_set_am.sql | 101 ++++++++- 4 files changed, 292 insertions(+), 20 deletions(-) diff --git a/src/backend/catalog/pg_appendonly.c b/src/backend/catalog/pg_appendonly.c index 7dd08ae5cfb..5b17075eeff 100644 --- a/src/backend/catalog/pg_appendonly.c +++ b/src/backend/catalog/pg_appendonly.c @@ -548,9 +548,7 @@ ATAOEntries(Form_pg_class relform1, Form_pg_class relform2, TransferAppendonlyEntries(relform2->oid, relform1->oid); break; case AO_COLUMN_TABLE_AM_OID: - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("alter table does not support switch from Heap to AOCO"))); + TransferAppendonlyEntries(relform2->oid, relform1->oid); break; case HEAP_TABLE_AM_OID: default: diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index aaf85df96c6..80554d50289 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -908,7 +908,7 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, Oid NewAccessMethod, ReleaseSysCache(tuple); } - if (RelationIsAppendOptimized(OldHeap) || NewAccessMethod == AO_ROW_TABLE_AM_OID) + if (IsAccessMethodAO(NewAccessMethod)) NewRelationCreateAOAuxTables(OIDNewHeap, createAoBlockDirectory); CacheInvalidateRelcacheByRelid(OIDNewHeap); diff --git a/src/test/regress/expected/alter_table_set_am.out b/src/test/regress/expected/alter_table_set_am.out index 9fc7ee3dab2..15da82171cb 100644 --- a/src/test/regress/expected/alter_table_set_am.out +++ b/src/test/regress/expected/alter_table_set_am.out @@ -1,5 +1,5 @@ -- Check changing table access method --- Scenario 1: Changing to the same AM: it should have no effect but +-- Scenario 1: Changing to the same AM: it should have no effect but -- make sure it doesn't rewrite table or blow up existing reloptions: CREATE TABLE sameam_heap(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); CREATE TABLE sameam_heap2(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); @@ -788,7 +788,7 @@ INSERT INTO co2heap2 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2heap3 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2heap4 SELECT i,i FROM generate_series(1,5) i; -- Prior-ATSETAM checks: --- Check once that the AO tables have the custom reloptions +-- Check once that the AO tables have the custom reloptions SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2heap%'; relname | reloptions ----------+----------------------------------- @@ -869,7 +869,7 @@ SELECT count(*) FROM co2heap4; (1 row) -- No AO aux tables should be left. --- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. -- No need to test the other ones created by the alternative syntax SET WITH(). SELECT * FROM gp_toolkit.__gp_aoseg('co2heap'); ERROR: 'co2heap' is not an append-only row relation @@ -945,7 +945,7 @@ INSERT INTO co2ao2 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2ao3 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2ao4 SELECT i,i FROM generate_series(1,5) i; -- Prior-ATSETAM checks: --- Check once that the AOCO tables have the custom reloptions +-- Check once that the AOCO tables have the custom reloptions SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; relname | reloptions ---------+--------------------------------------------------------- @@ -1051,7 +1051,7 @@ SELECT count(*) FROM co2ao4; (1 row) -- AO aux tables should still be there, but AOCO seg tables are not. --- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. -- No need to test the other ones created by the alternative syntax SET WITH(). SELECT * FROM gp_toolkit.__gp_aoseg('co2ao'); segment_id | segno | eof | tupcount | varblockcount | eof_uncompressed | modcount | formatversion | state @@ -1139,11 +1139,185 @@ DROP TABLE co2ao; DROP TABLE co2ao2; DROP TABLE co2ao3; DROP TABLE co2ao4; --- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". +-- Scenario 8: Heap to AOCO +SET gp_default_storage_options = 'blocksize=65536, compresstype=zlib, compresslevel=5, checksum=true'; +CREATE TABLE heap2co(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE heap2co2(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE heap2co3(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE heap2co4(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE INDEX index_heap2co ON heap2co(b); +CREATE INDEX index_heap2co3 ON heap2co3(b); +INSERT INTO heap2co SELECT i,i FROM generate_series(1,5) i; +INSERT INTO heap2co2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO heap2co3 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO heap2co4 SELECT i,i FROM generate_series(1,5) i; +CREATE TEMP TABLE relfilebeforeaoco AS +SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'heap2co%' +UNION SELECT gp_segment_id segid, relname, relfilenode FROM gp_dist_random('pg_class') +WHERE relname LIKE 'heap2co%' ORDER BY segid; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'segid' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +-- ERROR: conflicting storage option specified. +ALTER TABLE heap2co SET ACCESS METHOD ao_column WITH (appendoptimized=false); +ERROR: ACCESS METHOD is specified as "ao_column" but the WITH option indicates it to be "heap" +LINE 1: ALTER TABLE heap2co SET ACCESS METHOD ao_column WITH (append... + ^ +-- Use of *both* ACCESS METHOD and WITH clauses is allowed, but we'll print a hint to indicate the redundancy. +ALTER TABLE heap2co SET ACCESS METHOD ao_column WITH (appendoptimized=true, orientation=column); +NOTICE: Redundant clauses are used to indicate the access method. +HINT: Only one of these is needed to indicate access method: the SET ACCESS METHOD clause or the options in the WITH clause. +-- Check once the reloptions +SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'heap2co%'; + relname | amname | reloptions +----------+-----------+----------------------------------- + heap2co | ao_column | {blocksize=65536,compresslevel=5} + heap2co2 | heap | + heap2co3 | heap | + heap2co4 | heap | +(4 rows) + +-- Altering AO to AOCO with various syntaxes, reloptions: +ALTER TABLE heap2co SET ACCESS METHOD ao_column; +ALTER TABLE heap2co2 SET WITH (appendoptimized=true, orientation=column); +ALTER TABLE heap2co3 SET ACCESS METHOD ao_column WITH (blocksize=32768, compresslevel=3); +ALTER TABLE heap2co4 SET WITH (appendoptimized=true, orientation=column, blocksize=32768, compresslevel=3); +-- The tables are rewritten +CREATE TEMP TABLE relfileafteraoco AS +SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'heap2co%' +UNION SELECT gp_segment_id segid, relname, relfilenode FROM gp_dist_random('pg_class') +WHERE relname LIKE 'heap2co%' ORDER BY segid; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'segid' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +SELECT * FROM relfilebeforeaoco INTERSECT SELECT * FROM relfileafteraoco; + segid | relname | relfilenode +-------+---------+------------- +(0 rows) + +DROP TABLE relfilebeforeaoco; +DROP TABLE relfileafteraoco; +-- Check data is intact +SELECT count(*) FROM heap2co; + count +------- + 5 +(1 row) + +SELECT count(*) FROM heap2co2; + count +------- + 5 +(1 row) + +SELECT count(*) FROM heap2co3; + count +------- + 5 +(1 row) + +SELECT count(*) FROM heap2co4; + count +------- + 5 +(1 row) + +-- Aux tables should have been created for the new AOCO table +-- Only tested for 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +SELECT gp_segment_id, (gp_toolkit.__gp_aovisimap('heap2co')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tid | segno | row_num +---------------+-----+-------+--------- +(0 rows) + +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('heap2co')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count +---------------+---------+-------+----------------+----------+--------------+-------------+----------- + 2 | (0,1) | 0 | 0 | 0 | 1 | 0 | 1 + 2 | (0,2) | 0 | 1 | 0 | 1 | 0 | 1 + 0 | (0,1) | 0 | 0 | 0 | 1 | 0 | 3 + 0 | (0,2) | 0 | 1 | 0 | 1 | 0 | 3 + 1 | (0,1) | 0 | 0 | 0 | 1 | 0 | 1 + 1 | (0,2) | 0 | 1 | 0 | 1 | 0 | 1 +(6 rows) + +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('heap2co'); + count +------- + 6 +(1 row) + +SELECT gp_segment_id, (gp_toolkit.__gp_aovisimap('heap2co3')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tid | segno | row_num +---------------+-----+-------+--------- +(0 rows) + +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('heap2co3')).* FROM gp_dist_random('gp_id'); + gp_segment_id | tupleid | segno | columngroup_no | entry_no | first_row_no | file_offset | row_count +---------------+---------+-------+----------------+----------+--------------+-------------+----------- + 1 | (0,1) | 0 | 0 | 0 | 1 | 0 | 1 + 1 | (0,2) | 0 | 1 | 0 | 1 | 0 | 1 + 2 | (0,1) | 0 | 0 | 0 | 1 | 0 | 1 + 2 | (0,2) | 0 | 1 | 0 | 1 | 0 | 1 + 0 | (0,1) | 0 | 0 | 0 | 1 | 0 | 3 + 0 | (0,2) | 0 | 1 | 0 | 1 | 0 | 3 +(6 rows) + +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('heap2co3'); + count +------- + 6 +(1 row) + +-- pg_attribute_encoding should have columns for the AOCO table +SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid = c.oid AND c.relname LIKE 'heap2co%'; + relname | attnum | attoptions +----------+--------+----------------------------------------------------- + heap2co | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + heap2co | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + heap2co2 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + heap2co2 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + heap2co3 | 2 | {blocksize=32768,compresslevel=3,compresstype=zlib} + heap2co3 | 1 | {blocksize=32768,compresslevel=3,compresstype=zlib} + heap2co4 | 2 | {blocksize=32768,compresslevel=3,compresstype=zlib} + heap2co4 | 1 | {blocksize=32768,compresslevel=3,compresstype=zlib} +(8 rows) + +-- AM and reloptions changed accordingly +SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'heap2co%'; + relname | amname | reloptions +----------+-----------+----------------------------------- + heap2co | ao_column | {blocksize=65536,compresslevel=5} + heap2co2 | ao_column | {blocksize=65536,compresslevel=5} + heap2co3 | ao_column | {blocksize=32768,compresslevel=3} + heap2co4 | ao_column | {blocksize=32768,compresslevel=3} +(4 rows) + +-- pg_appendonly should reflect the changes in reloptions +SELECT c.relname,a.blocksize,a.compresslevel,a.checksum,a.compresstype,a.columnstore +FROM pg_appendonly a, pg_class c WHERE a.relid = c.oid AND relname like ('heap2co%'); + relname | blocksize | compresslevel | checksum | compresstype | columnstore +----------+-----------+---------------+----------+--------------+------------- + heap2co | 65536 | 5 | t | zlib | t + heap2co2 | 65536 | 5 | t | zlib | t + heap2co3 | 32768 | 3 | t | zlib | t + heap2co4 | 32768 | 3 | t | zlib | t +(4 rows) + +DROP TABLE heap2co; +DROP TABLE heap2co2; +DROP TABLE heap2co3; +DROP TABLE heap2co4; +-- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". -- The following cases will cover all variations of such iterations: -- 1. Heap->AO->Heap->AO -- 2. AO->AOCO->AO->AOCO --- (TODO) 3. Heap->AOCO->Heap->AOCO +-- 3. Heap->AOCO->Heap->AOCO -- 1. Heap->AO->Heap->AO CREATE TABLE heapao(a int, b int); CREATE INDEX heapaoindex ON heapao(b); @@ -1166,7 +1340,7 @@ INSERT INTO aoco SELECT i,i FROM generate_series(1,5) i; ALTER TABLE aoco SET ACCESS METHOD ao_column; ALTER TABLE aoco SET ACCESS METHOD ao_row; ALTER TABLE aoco SET ACCESS METHOD ao_column; --- Just checking data is intact. +-- Just checking data is intact. SELECT count(*) FROM aoco; count ------- @@ -1174,3 +1348,18 @@ SELECT count(*) FROM aoco; (1 row) DROP TABLE aoco; +-- 3. Heap->AOCO->Heap->AOCO +CREATE TABLE heapco(a int, b int); +CREATE INDEX heapcoindex ON heapco(b); +INSERT INTO heapco SELECT i,i FROM generate_series(1,5) i; +ALTER TABLE heapco SET ACCESS METHOD ao_column; +ALTER TABLE heapco SET ACCESS METHOD heap; +ALTER TABLE heapco SET ACCESS METHOD ao_column; +-- Just checking data is intact. +SELECT count(*) FROM heapco; + count +------- + 5 +(1 row) + +DROP TABLE heapco; diff --git a/src/test/regress/sql/alter_table_set_am.sql b/src/test/regress/sql/alter_table_set_am.sql index 52e3b127c7a..9520360981d 100644 --- a/src/test/regress/sql/alter_table_set_am.sql +++ b/src/test/regress/sql/alter_table_set_am.sql @@ -1,6 +1,6 @@ -- Check changing table access method --- Scenario 1: Changing to the same AM: it should have no effect but +-- Scenario 1: Changing to the same AM: it should have no effect but -- make sure it doesn't rewrite table or blow up existing reloptions: CREATE TABLE sameam_heap(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); CREATE TABLE sameam_heap2(a int, b int) WITH (fillfactor=70) DISTRIBUTED BY (a); @@ -440,7 +440,7 @@ INSERT INTO co2heap3 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2heap4 SELECT i,i FROM generate_series(1,5) i; -- Prior-ATSETAM checks: --- Check once that the AO tables have the custom reloptions +-- Check once that the AO tables have the custom reloptions SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2heap%'; -- Check once that the AO tables have relfrozenxid = 0 SELECT relname, relfrozenxid FROM pg_class WHERE relname LIKE 'co2heap%'; @@ -475,7 +475,7 @@ SELECT count(*) FROM co2heap3; SELECT count(*) FROM co2heap4; -- No AO aux tables should be left. --- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. -- No need to test the other ones created by the alternative syntax SET WITH(). SELECT * FROM gp_toolkit.__gp_aoseg('co2heap'); SELECT * FROM gp_toolkit.__gp_aovisimap('co2heap'); @@ -520,7 +520,7 @@ INSERT INTO co2ao3 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2ao4 SELECT i,i FROM generate_series(1,5) i; -- Prior-ATSETAM checks: --- Check once that the AOCO tables have the custom reloptions +-- Check once that the AOCO tables have the custom reloptions SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; -- Check once that pg_appendonly has expected entries. SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; @@ -561,7 +561,7 @@ SELECT count(*) FROM co2ao3; SELECT count(*) FROM co2ao4; -- AO aux tables should still be there, but AOCO seg tables are not. --- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. -- No need to test the other ones created by the alternative syntax SET WITH(). SELECT * FROM gp_toolkit.__gp_aoseg('co2ao'); SELECT * FROM gp_toolkit.__gp_aocsseg('co2ao'); @@ -589,11 +589,84 @@ DROP TABLE co2ao2; DROP TABLE co2ao3; DROP TABLE co2ao4; --- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". +-- Scenario 8: Heap to AOCO +SET gp_default_storage_options = 'blocksize=65536, compresstype=zlib, compresslevel=5, checksum=true'; +CREATE TABLE heap2co(a int, b int); +CREATE TABLE heap2co2(a int, b int); +CREATE TABLE heap2co3(a int, b int); +CREATE TABLE heap2co4(a int, b int); +CREATE INDEX index_heap2co ON heap2co(b); +CREATE INDEX index_heap2co3 ON heap2co3(b); + +INSERT INTO heap2co SELECT i,i FROM generate_series(1,5) i; +INSERT INTO heap2co2 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO heap2co3 SELECT i,i FROM generate_series(1,5) i; +INSERT INTO heap2co4 SELECT i,i FROM generate_series(1,5) i; + +CREATE TEMP TABLE relfilebeforeaoco AS +SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'heap2co%' +UNION SELECT gp_segment_id segid, relname, relfilenode FROM gp_dist_random('pg_class') +WHERE relname LIKE 'heap2co%' ORDER BY segid; + +-- ERROR: conflicting storage option specified. +ALTER TABLE heap2co SET ACCESS METHOD ao_column WITH (appendoptimized=false); +-- Use of *both* ACCESS METHOD and WITH clauses is allowed, but we'll print a hint to indicate the redundancy. +ALTER TABLE heap2co SET ACCESS METHOD ao_column WITH (appendoptimized=true, orientation=column); + +-- Check once the reloptions +SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'heap2co%'; + +-- Altering AO to AOCO with various syntaxes, reloptions: +ALTER TABLE heap2co SET ACCESS METHOD ao_column; +ALTER TABLE heap2co2 SET WITH (appendoptimized=true, orientation=column); +ALTER TABLE heap2co3 SET ACCESS METHOD ao_column WITH (blocksize=32768, compresslevel=3); +ALTER TABLE heap2co4 SET WITH (appendoptimized=true, orientation=column, blocksize=32768, compresslevel=3); + +-- The tables are rewritten +CREATE TEMP TABLE relfileafteraoco AS +SELECT -1 segid, relname, relfilenode FROM pg_class WHERE relname LIKE 'heap2co%' +UNION SELECT gp_segment_id segid, relname, relfilenode FROM gp_dist_random('pg_class') +WHERE relname LIKE 'heap2co%' ORDER BY segid; + +SELECT * FROM relfilebeforeaoco INTERSECT SELECT * FROM relfileafteraoco; +DROP TABLE relfilebeforeaoco; +DROP TABLE relfileafteraoco; + +-- Check data is intact +SELECT count(*) FROM heap2co; +SELECT count(*) FROM heap2co2; +SELECT count(*) FROM heap2co3; +SELECT count(*) FROM heap2co4; + +-- Aux tables should have been created for the new AOCO table +-- Only tested for 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +SELECT gp_segment_id, (gp_toolkit.__gp_aovisimap('heap2co')).* FROM gp_dist_random('gp_id'); +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('heap2co')).* FROM gp_dist_random('gp_id'); +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('heap2co'); +SELECT gp_segment_id, (gp_toolkit.__gp_aovisimap('heap2co3')).* FROM gp_dist_random('gp_id'); +SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('heap2co3')).* FROM gp_dist_random('gp_id'); +SELECT count(*) FROM gp_toolkit.__gp_aocsseg('heap2co3'); + +-- pg_attribute_encoding should have columns for the AOCO table +SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid = c.oid AND c.relname LIKE 'heap2co%'; + +-- AM and reloptions changed accordingly +SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'heap2co%'; + +-- pg_appendonly should reflect the changes in reloptions +SELECT c.relname,a.blocksize,a.compresslevel,a.checksum,a.compresstype,a.columnstore +FROM pg_appendonly a, pg_class c WHERE a.relid = c.oid AND relname like ('heap2co%'); + +DROP TABLE heap2co; +DROP TABLE heap2co2; +DROP TABLE heap2co3; +DROP TABLE heap2co4; + +-- Final scenario: the iterations of altering table from storage type "A" to "B" and back to "A". -- The following cases will cover all variations of such iterations: -- 1. Heap->AO->Heap->AO -- 2. AO->AOCO->AO->AOCO --- (TODO) 3. Heap->AOCO->Heap->AOCO +-- 3. Heap->AOCO->Heap->AOCO -- 1. Heap->AO->Heap->AO CREATE TABLE heapao(a int, b int); @@ -617,7 +690,19 @@ ALTER TABLE aoco SET ACCESS METHOD ao_column; ALTER TABLE aoco SET ACCESS METHOD ao_row; ALTER TABLE aoco SET ACCESS METHOD ao_column; --- Just checking data is intact. +-- Just checking data is intact. SELECT count(*) FROM aoco; DROP TABLE aoco; +-- 3. Heap->AOCO->Heap->AOCO +CREATE TABLE heapco(a int, b int); +CREATE INDEX heapcoindex ON heapco(b); +INSERT INTO heapco SELECT i,i FROM generate_series(1,5) i; + +ALTER TABLE heapco SET ACCESS METHOD ao_column; +ALTER TABLE heapco SET ACCESS METHOD heap; +ALTER TABLE heapco SET ACCESS METHOD ao_column; + +-- Just checking data is intact. +SELECT count(*) FROM heapco; +DROP TABLE heapco; From c47acdaf51b3140448b1779dbe0db6268d1c2874 Mon Sep 17 00:00:00 2001 From: Huansong Fu Date: Thu, 11 Aug 2022 12:50:43 -0700 Subject: [PATCH 7/8] ALTER TABLE SET ACCESS METHOD: AOCO->Heap support As part of the ATSETAM support, this commit adds support for changing AM of a table from AOCO to heap. E.g.: ``` CREATE TABLE foo (appendonly=true, orientation=column); ALTER TABLE foo SET ACCESS METHOD heap; -- Or: ALTER TABLE foo SET WITH (appendonly=false); ``` Optionally, user can specify reloptions in a WITH clause too, e.g.: ``` ALTER TABLE foo SET ACCESS METHOD heap WITH (fillfactor=70); ``` --- src/backend/catalog/pg_appendonly.c | 8 +++++--- src/backend/commands/cluster.c | 13 +++++++++--- .../regress/expected/alter_table_set_am.out | 20 +++++++++---------- src/test/regress/sql/alter_table_set_am.sql | 4 ++-- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/backend/catalog/pg_appendonly.c b/src/backend/catalog/pg_appendonly.c index 5b17075eeff..48881330bd7 100644 --- a/src/backend/catalog/pg_appendonly.c +++ b/src/backend/catalog/pg_appendonly.c @@ -31,6 +31,7 @@ #include "access/table.h" #include "catalog/dependency.h" #include "catalog/indexing.h" +#include "catalog/pg_attribute_encoding.h" #include "utils/builtins.h" #include "utils/inval.h" #include "utils/lsyscache.h" @@ -589,9 +590,10 @@ ATAOEntries(Form_pg_class relform1, Form_pg_class relform2, switch(relform2->relam) { case HEAP_TABLE_AM_OID: - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("alter table does not support switch from AOCO to Heap"))); + /* For pg_appendonly entries, it's the same as AO->Heap. */ + TransferAppendonlyEntries(relform1->oid, relform2->oid); + /* Remove the pg_attribute_encoding entries, since heap tables shouldn't have these. */ + RemoveAttributeEncodingsByRelid(relform1->oid); break; case AO_ROW_TABLE_AM_OID: /* For pg_appendonly entries, it's same as AO->AO/CO. */ diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 80554d50289..d7ba0c34b89 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -913,9 +913,16 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, Oid NewAccessMethod, CacheInvalidateRelcacheByRelid(OIDNewHeap); - cloneAttributeEncoding(OIDOldHeap, - OIDNewHeap, - RelationGetNumberOfAttributes(OldHeap)); + /* + * Copy the pg_attribute_encoding entries over if new table needs them. + * Note that in the case of AM change from heap/ao to aoco, we still need + * to do this since we created those entries for the heap/ao table at the + * phase 2 of ATSETAM (see ATExecCmd). + */ + if (NewAccessMethod == AO_COLUMN_TABLE_AM_OID) + cloneAttributeEncoding(OIDOldHeap, + OIDNewHeap, + RelationGetNumberOfAttributes(OldHeap)); table_close(OldHeap, NoLock); diff --git a/src/test/regress/expected/alter_table_set_am.out b/src/test/regress/expected/alter_table_set_am.out index 15da82171cb..f809c9cb14c 100644 --- a/src/test/regress/expected/alter_table_set_am.out +++ b/src/test/regress/expected/alter_table_set_am.out @@ -741,10 +741,10 @@ SELECT * FROM gp_toolkit.__gp_aoblkdir('ao2co3'); SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid = c.oid AND c.relname LIKE 'ao2co%'; relname | attnum | attoptions ---------+--------+--------------------------------------------------------- - ao2co | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} - ao2co | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} - ao2co2 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} - ao2co2 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + ao2co | 1 | {compresstype=zlib,compresslevel=5,blocksize=65536} + ao2co | 2 | {compresstype=zlib,compresslevel=5,blocksize=65536} + ao2co2 | 1 | {compresstype=zlib,compresslevel=5,blocksize=65536} + ao2co2 | 2 | {compresstype=zlib,compresslevel=5,blocksize=65536} ao2co3 | 1 | {blocksize=32768,compresstype=rle_type,compresslevel=3} ao2co3 | 2 | {blocksize=32768,compresstype=rle_type,compresslevel=3} ao2co4 | 1 | {blocksize=32768,compresstype=rle_type,compresslevel=3} @@ -788,7 +788,7 @@ INSERT INTO co2heap2 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2heap3 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2heap4 SELECT i,i FROM generate_series(1,5) i; -- Prior-ATSETAM checks: --- Check once that the AO tables have the custom reloptions +-- Check once that the AO tables have the custom reloptions SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2heap%'; relname | reloptions ----------+----------------------------------- @@ -869,7 +869,7 @@ SELECT count(*) FROM co2heap4; (1 row) -- No AO aux tables should be left. --- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. -- No need to test the other ones created by the alternative syntax SET WITH(). SELECT * FROM gp_toolkit.__gp_aoseg('co2heap'); ERROR: 'co2heap' is not an append-only row relation @@ -1278,10 +1278,10 @@ SELECT count(*) FROM gp_toolkit.__gp_aocsseg('heap2co3'); SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid = c.oid AND c.relname LIKE 'heap2co%'; relname | attnum | attoptions ----------+--------+----------------------------------------------------- - heap2co | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} - heap2co | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} - heap2co2 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} - heap2co2 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} + heap2co | 2 | {compresstype=zlib,compresslevel=5,blocksize=65536} + heap2co | 1 | {compresstype=zlib,compresslevel=5,blocksize=65536} + heap2co2 | 2 | {compresstype=zlib,compresslevel=5,blocksize=65536} + heap2co2 | 1 | {compresstype=zlib,compresslevel=5,blocksize=65536} heap2co3 | 2 | {blocksize=32768,compresslevel=3,compresstype=zlib} heap2co3 | 1 | {blocksize=32768,compresslevel=3,compresstype=zlib} heap2co4 | 2 | {blocksize=32768,compresslevel=3,compresstype=zlib} diff --git a/src/test/regress/sql/alter_table_set_am.sql b/src/test/regress/sql/alter_table_set_am.sql index 9520360981d..cc572c2fc1f 100644 --- a/src/test/regress/sql/alter_table_set_am.sql +++ b/src/test/regress/sql/alter_table_set_am.sql @@ -440,7 +440,7 @@ INSERT INTO co2heap3 SELECT i,i FROM generate_series(1,5) i; INSERT INTO co2heap4 SELECT i,i FROM generate_series(1,5) i; -- Prior-ATSETAM checks: --- Check once that the AO tables have the custom reloptions +-- Check once that the AO tables have the custom reloptions SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2heap%'; -- Check once that the AO tables have relfrozenxid = 0 SELECT relname, relfrozenxid FROM pg_class WHERE relname LIKE 'co2heap%'; @@ -475,7 +475,7 @@ SELECT count(*) FROM co2heap3; SELECT count(*) FROM co2heap4; -- No AO aux tables should be left. --- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. +-- Only testing 2 out of the 4 tables being created, where the tables were altered w/wo reloptions. -- No need to test the other ones created by the alternative syntax SET WITH(). SELECT * FROM gp_toolkit.__gp_aoseg('co2heap'); SELECT * FROM gp_toolkit.__gp_aovisimap('co2heap'); From abddf092964045a5be74555daa45e08340e4719c Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 22 Jan 2025 06:49:21 +0000 Subject: [PATCH 8/8] Fix cherry-pick issues. --- src/backend/access/common/reloptions_gp.c | 18 ++++-- src/backend/commands/tablecmds.c | 22 ++++--- src/include/access/reloptions.h | 2 +- .../regress/expected/alter_table_set_am.out | 58 +++++++++---------- src/test/regress/sql/alter_table_set_am.sql | 4 +- 5 files changed, 54 insertions(+), 50 deletions(-) diff --git a/src/backend/access/common/reloptions_gp.c b/src/backend/access/common/reloptions_gp.c index b6022e1ccc4..77f2f86bb1d 100644 --- a/src/backend/access/common/reloptions_gp.c +++ b/src/backend/access/common/reloptions_gp.c @@ -1670,7 +1670,7 @@ find_crsd(const char *column, List *stenc) List * transformColumnEncoding(const TableAmRoutine *tam, Relation rel, List *colDefs, List *stenc, List *withOptions, List *parentenc, - bool explicitOnly, bool createDefaultOne) + bool explicitOnly, bool createDefaultOne, bool appendonly) { ColumnReferenceStorageDirective *deflt = NULL; ListCell *lc; @@ -1713,7 +1713,10 @@ transformColumnEncoding(const TableAmRoutine *tam, Relation rel, List *colDefs, deflt = copyObject(c); - deflt->encoding = tam->transform_column_encoding_clauses(rel, deflt->encoding, true, false); + if (appendonly) + deflt->encoding = tam->transform_column_encoding_clauses(rel, deflt->encoding, true, false); + else + deflt->encoding = transformStorageEncodingClause(deflt->encoding, true); /* * The default encoding and the with clause better not @@ -1742,8 +1745,10 @@ transformColumnEncoding(const TableAmRoutine *tam, Relation rel, List *colDefs, * if current am not inmplement transform_column_encoding_clauses * then tmpenc not null but no need fill with options. */ - if (tam->transform_column_encoding_clauses) + if (tam->transform_column_encoding_clauses && appendonly) deflt->encoding = tam->transform_column_encoding_clauses(rel, tmpenc, false, false); + else + deflt->encoding = transformStorageEncodingClause(tmpenc, false); } } @@ -1781,9 +1786,10 @@ transformColumnEncoding(const TableAmRoutine *tam, Relation rel, List *colDefs, ColumnReferenceStorageDirective *s = find_crsd(d->colname, stenc); if (s) { - encoding = tam->transform_column_encoding_clauses(rel, s->encoding, true, false); + if (tam->transform_column_encoding_clauses) + encoding = tam->transform_column_encoding_clauses(rel, s->encoding, true, false); } else { - if (deflt) + if (deflt && deflt->encoding != NULL) encoding = copyObject(deflt->encoding); else if (!explicitOnly) { @@ -1796,7 +1802,7 @@ transformColumnEncoding(const TableAmRoutine *tam, Relation rel, List *colDefs, else if (d->typeName) { /* get encoding by type, still need do transform and validate */ encoding = get_type_encoding(d->typeName); - if (tam->transform_column_encoding_clauses) + if (tam->transform_column_encoding_clauses && appendonly) encoding = tam->transform_column_encoding_clauses(rel, encoding, true, true); } if (!encoding && createDefaultOne) { diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index fdc30e59200..c60b4daff27 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -584,7 +584,7 @@ static bool prebuild_temp_table(Relation rel, RangeVar *tmpname, DistributedBy * static void checkATSetDistributedByStandalone(AlteredTableInfo *tab, Relation rel); -static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOptions); +static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOptions, Oid newAm); static void clear_rel_opts(Relation rel); @@ -1147,7 +1147,8 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, stmt->options, parentenc, relkind == RELKIND_PARTITIONED_TABLE, - AMHandlerIsAoCols(amHandlerOid) /* createDefaultOne*/); + AMHandlerIsAoCols(amHandlerOid) /* createDefaultOne*/, true); + if (!AMHandlerSupportEncodingClause(tam) && relkind != RELKIND_PARTITIONED_TABLE) stmt->attr_encodings = NIL; } @@ -1334,7 +1335,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, false, accessMethodId != AO_COLUMN_TABLE_AM_OID && !stmt->partbound && !stmt->partspec - /* errorOnEncodingClause */); + /* errorOnEncodingClause */, true); AddRelationAttributeEncodings(rel, part_attr_encodings); } @@ -4823,7 +4824,7 @@ AlterTable(AlterTableStmt *stmt, LOCKMODE lockmode, /* * Populate the column encoding option for each column in the relation. */ -static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOptions) +static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOptions, Oid newAccessMethod) { int attno; List *colDefs = NIL; @@ -4841,7 +4842,7 @@ static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOpti colDefs = lappend(colDefs, cd); } - tam = GetTableAmRoutineByAmId(rel->rd_rel->relam); + tam = GetTableAmRoutineByAmId(newAccessMethod); List *attr_encodings = transformColumnEncoding(tam /* TableAmRoutine */, rel, colDefs /*column clauses*/, @@ -4849,7 +4850,8 @@ static void populate_rel_col_encodings(Relation rel, List *stenc, List *withOpti withOptions /*withOptions*/, NULL, false, - RelationIsAoCols(rel)); + newAccessMethod == AO_COLUMN_TABLE_AM_OID, RelationIsAppendOptimized(rel)); + AddRelationAttributeEncodings(rel, attr_encodings); } @@ -6217,7 +6219,7 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, /* If we are changing AM to AOCO, add pg_attribute_encoding entries for each column. */ if (tab->newAccessMethod == AO_COLUMN_TABLE_AM_OID) - populate_rel_col_encodings(rel, NULL, (List*)cmd->def); + populate_rel_col_encodings(rel, NULL, (List*)cmd->def, tab->newAccessMethod); break; case AT_SetTableSpace: /* SET TABLESPACE */ @@ -8719,7 +8721,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, NULL /* withOptions */, NULL /* parentenc */, false /* explicitOnly */, - RelationIsAoCols(rel) /* createDefaultOne */); + RelationIsAoCols(rel) /* createDefaultOne */, true); /* * Store the encoding clause for AO/CO tables. */ @@ -16121,10 +16123,6 @@ ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, bool repl_repl[Natts_pg_class]; const TableAmRoutine * newAM; static char *validnsps[] = HEAP_RELOPT_NAMESPACES; - Oid tableam; - - /* Get the new table AM if applicable. Otherwise get the one from the reltion. */ - tableam = (newam != InvalidOid) ? newam : rel->rd_rel->relam; if (defList == NIL && operation != AT_ReplaceRelOptions) return; /* nothing to do */ diff --git a/src/include/access/reloptions.h b/src/include/access/reloptions.h index a52b526d6a9..719f67b0e2e 100644 --- a/src/include/access/reloptions.h +++ b/src/include/access/reloptions.h @@ -305,7 +305,7 @@ extern void validate_and_adjust_options(StdRdOptions *result, relopt_value *opti extern void validateAOCOColumnEncodingClauses(List *aocoColumnEncoding); extern List *transformColumnEncoding(const TableAmRoutine *tam, Relation rel, List *colDefs, List *stenc, List *withOptions, List *parentenc, - bool explicitOnly, bool createDefaultOne); + bool explicitOnly, bool createDefaultOne, bool appendonly); List* transfromColumnEncodingAocoRootPartition(List *colDefs, List *stenc, List *withOptions, bool errorOnEncodingClause); diff --git a/src/test/regress/expected/alter_table_set_am.out b/src/test/regress/expected/alter_table_set_am.out index f809c9cb14c..f1657176122 100644 --- a/src/test/regress/expected/alter_table_set_am.out +++ b/src/test/regress/expected/alter_table_set_am.out @@ -755,8 +755,8 @@ SELECT c.relname, a.attnum, a.attoptions FROM pg_attribute_encoding a, pg_class SELECT c.relname, a.amname, c.reloptions FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE c.relname LIKE 'ao2co%'; relname | amname | reloptions ---------+-----------+--------------------------------------------------------- - ao2co | ao_column | - ao2co2 | ao_column | + ao2co | ao_column | {blocksize=65536,compresslevel=5} + ao2co2 | ao_column | {blocksize=65536,compresslevel=5} ao2co3 | ao_column | {blocksize=32768,compresstype=rle_type,compresslevel=3} ao2co4 | ao_column | {blocksize=32768,compresstype=rle_type,compresslevel=3} (4 rows) @@ -766,8 +766,8 @@ SELECT c.relname,a.blocksize,a.compresslevel,a.checksum,a.compresstype,a.columns FROM pg_appendonly a, pg_class c WHERE a.relid = c.oid AND relname like ('ao2co%'); relname | blocksize | compresslevel | checksum | compresstype | columnstore ---------+-----------+---------------+----------+--------------+------------- - ao2co | 32768 | 0 | t | | t - ao2co2 | 32768 | 0 | t | | t + ao2co | 65536 | 5 | t | zlib | t + ao2co2 | 65536 | 5 | t | zlib | t ao2co3 | 32768 | 3 | t | rle_type | t ao2co4 | 32768 | 3 | t | rle_type | t (4 rows) @@ -812,14 +812,14 @@ SELECT relname, relfrozenxid FROM pg_class WHERE relname LIKE 'co2heap%'; SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2heap%'; relname | attnum | attoptions ----------+--------+----------------------------------------------------- - co2heap | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} - co2heap | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} - co2heap2 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} - co2heap2 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} - co2heap3 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} - co2heap3 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} - co2heap4 | 1 | {compresstype=zlib,blocksize=65536,compresslevel=5} - co2heap4 | 2 | {compresstype=zlib,blocksize=65536,compresslevel=5} + co2heap | 1 | {compresstype=zlib,compresslevel=5,blocksize=65536} + co2heap | 2 | {compresstype=zlib,compresslevel=5,blocksize=65536} + co2heap2 | 1 | {compresstype=zlib,compresslevel=5,blocksize=65536} + co2heap2 | 2 | {compresstype=zlib,compresslevel=5,blocksize=65536} + co2heap3 | 1 | {compresstype=zlib,compresslevel=5,blocksize=65536} + co2heap3 | 2 | {compresstype=zlib,compresslevel=5,blocksize=65536} + co2heap4 | 1 | {compresstype=zlib,compresslevel=5,blocksize=65536} + co2heap4 | 2 | {compresstype=zlib,compresslevel=5,blocksize=65536} (8 rows) CREATE TEMP TABLE relfilebeforeco2heap AS @@ -934,8 +934,8 @@ DROP TABLE co2heap2; DROP TABLE co2heap3; DROP TABLE co2heap4; -- Scenario 7: AOCO to AO -CREATE TABLE co2ao(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); -CREATE TABLE co2ao2(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE TABLE co2ao(a int, b int) WITH (appendonly=true, orientation=column, compresstype=zlib, compresslevel=3); +CREATE TABLE co2ao2(a int, b int) WITH (appendonly=true, orientation=column, compresstype=zlib, compresslevel=3); CREATE TABLE co2ao3(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); CREATE TABLE co2ao4(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); CREATE INDEX aoi ON co2ao(b); @@ -949,8 +949,8 @@ INSERT INTO co2ao4 SELECT i,i FROM generate_series(1,5) i; SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; relname | reloptions ---------+--------------------------------------------------------- - co2ao | {compresstype=rle_type,compresslevel=3,blocksize=65536} - co2ao2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao | {compresstype=zlib,compresslevel=3,blocksize=65536} + co2ao2 | {compresstype=zlib,compresslevel=3,blocksize=65536} co2ao3 | {compresstype=rle_type,compresslevel=3,blocksize=65536} co2ao4 | {compresstype=rle_type,compresslevel=3,blocksize=65536} (4 rows) @@ -959,8 +959,8 @@ SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; relname | compresstype | compresslevel | blocksize ---------+--------------+---------------+----------- - co2ao | rle_type | 3 | 65536 - co2ao2 | rle_type | 3 | 65536 + co2ao | zlib | 3 | 65536 + co2ao2 | zlib | 3 | 65536 co2ao3 | rle_type | 3 | 65536 co2ao4 | rle_type | 3 | 65536 (4 rows) @@ -969,10 +969,10 @@ SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, SELECT c.relname, a.attnum, attoptions FROM pg_attribute_encoding a, pg_class c WHERE a.attrelid=c.oid AND c.relname LIKE 'co2ao%'; relname | attnum | attoptions ---------+--------+--------------------------------------------------------- - co2ao | 1 | {compresstype=rle_type,compresslevel=3,blocksize=65536} - co2ao | 2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} - co2ao2 | 1 | {compresstype=rle_type,compresslevel=3,blocksize=65536} - co2ao2 | 2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} + co2ao | 1 | {compresstype=zlib,compresslevel=3,blocksize=65536} + co2ao | 2 | {compresstype=zlib,compresslevel=3,blocksize=65536} + co2ao2 | 1 | {compresstype=zlib,compresslevel=3,blocksize=65536} + co2ao2 | 2 | {compresstype=zlib,compresslevel=3,blocksize=65536} co2ao3 | 1 | {compresstype=rle_type,compresslevel=3,blocksize=65536} co2ao3 | 2 | {compresstype=rle_type,compresslevel=3,blocksize=65536} co2ao4 | 1 | {compresstype=rle_type,compresslevel=3,blocksize=65536} @@ -1056,7 +1056,7 @@ SELECT count(*) FROM co2ao4; SELECT * FROM gp_toolkit.__gp_aoseg('co2ao'); segment_id | segno | eof | tupcount | varblockcount | eof_uncompressed | modcount | formatversion | state ------------+-------+-----+----------+---------------+------------------+----------+---------------+------- - 0 | 0 | 88 | 3 | 1 | 88 | 1 | 3 | 1 + 0 | 0 | 72 | 3 | 1 | 88 | 1 | 3 | 1 1 | 0 | 40 | 1 | 1 | 40 | 1 | 3 | 1 2 | 0 | 40 | 1 | 1 | 40 | 1 | 3 | 1 (3 rows) @@ -1103,8 +1103,8 @@ SELECT gp_segment_id, (gp_toolkit.__gp_aoblkdir('co2ao3')).* FROM gp_dist_random SELECT c.relname, p.compresstype, p.compresslevel, p.blocksize FROM pg_class c, pg_appendonly p WHERE c.relname LIKE 'co2ao%' AND c.oid = p.relid; relname | compresstype | compresslevel | blocksize ---------+--------------+---------------+----------- - co2ao | | 0 | 32768 - co2ao2 | | 0 | 32768 + co2ao | zlib | 3 | 65536 + co2ao2 | zlib | 3 | 65536 co2ao3 | zlib | 7 | 32768 co2ao4 | zlib | 7 | 32768 (4 rows) @@ -1121,10 +1121,10 @@ SELECT c.relname, a.amname FROM pg_class c JOIN pg_am a ON c.relam = a.oid WHERE -- Only the new tables altered w/ reloptions supplies should have reloptions. SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'co2ao%'; - relname | reloptions ----------+------------------------------------- - co2ao | - co2ao2 | + relname | reloptions +---------+----------------------------------------------------- + co2ao | {compresstype=zlib,compresslevel=3,blocksize=65536} + co2ao2 | {compresstype=zlib,compresslevel=3,blocksize=65536} co2ao3 | {compresstype=zlib,compresslevel=7} co2ao4 | {compresstype=zlib,compresslevel=7} (4 rows) diff --git a/src/test/regress/sql/alter_table_set_am.sql b/src/test/regress/sql/alter_table_set_am.sql index cc572c2fc1f..f3c1a235c9d 100644 --- a/src/test/regress/sql/alter_table_set_am.sql +++ b/src/test/regress/sql/alter_table_set_am.sql @@ -507,8 +507,8 @@ DROP TABLE co2heap3; DROP TABLE co2heap4; -- Scenario 7: AOCO to AO -CREATE TABLE co2ao(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); -CREATE TABLE co2ao2(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); +CREATE TABLE co2ao(a int, b int) WITH (appendonly=true, orientation=column, compresstype=zlib, compresslevel=3); +CREATE TABLE co2ao2(a int, b int) WITH (appendonly=true, orientation=column, compresstype=zlib, compresslevel=3); CREATE TABLE co2ao3(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); CREATE TABLE co2ao4(a int, b int) WITH (appendonly=true, orientation=column, compresstype=rle_type, compresslevel=3); CREATE INDEX aoi ON co2ao(b);