From 9c3bb808b0c70d3c8a085edb4d190fcfe68fc3ff Mon Sep 17 00:00:00 2001 From: Ken Barber Date: Wed, 20 Mar 2024 16:55:34 +0000 Subject: [PATCH 1/2] Create extensions in the correct schemas and include copy schemas command --- docs/include/copy-schemas.rst | 13 +++++++ docs/include/copy.rst | 1 + docs/include/help.rst | 1 + docs/ref/pgcopydb_copy.rst | 9 +++++ src/bin/pgcopydb/cli_copy.c | 63 +++++++++++++++++++++++++++++++++ src/bin/pgcopydb/dump_restore.c | 27 ++++++++++++-- src/bin/pgcopydb/extensions.c | 7 ++-- tests/extensions/copydb.sh | 10 ++++++ 8 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 docs/include/copy-schemas.rst diff --git a/docs/include/copy-schemas.rst b/docs/include/copy-schemas.rst new file mode 100644 index 000000000..02cf87018 --- /dev/null +++ b/docs/include/copy-schemas.rst @@ -0,0 +1,13 @@ +:: + + pgcopydb copy schemas: Create all the schemas found in the source database in the target + usage: pgcopydb copy schemas --source ... --target ... [ --table-jobs ... --index-jobs ... ] + + --source Postgres URI to the source database + --target Postgres URI to the target database + --dir Work directory to use + --filters Use the filters defined in + --restart Allow restarting when temp files exist already + --resume Allow resuming operations after a failure + --not-consistent Allow taking a new snapshot on the source database + diff --git a/docs/include/copy.rst b/docs/include/copy.rst index 29a9c3d32..a8ad5e973 100644 --- a/docs/include/copy.rst +++ b/docs/include/copy.rst @@ -14,4 +14,5 @@ sequences Copy the current value from all sequences in database from source to target indexes Create all the indexes found in the source database in the target constraints Create all the constraints found in the source database in the target + schemas Create all the schemas found in the source database in the target diff --git a/docs/include/help.rst b/docs/include/help.rst index 3fa7c7384..e99177d57 100644 --- a/docs/include/help.rst +++ b/docs/include/help.rst @@ -30,6 +30,7 @@ sequences Copy the current value from all sequences in database from source to target indexes Create all the indexes found in the source database in the target constraints Create all the constraints found in the source database in the target + schemas Create all the schemas found in the source database in the target pgcopydb dump schema Dump source database schema as custom files in work directory diff --git a/docs/ref/pgcopydb_copy.rst b/docs/ref/pgcopydb_copy.rst index e23eb92ba..a5b8aadcc 100644 --- a/docs/ref/pgcopydb_copy.rst +++ b/docs/ref/pgcopydb_copy.rst @@ -198,6 +198,15 @@ is found existing already on the target database. .. include:: ../include/copy-constraints.rst +pgcopydb copy schemas +--------------------- + +pgcopydb copy schemas - Creates all the schemas found in the source database in the target + +The command ``pgcopydb copy schemas`` fetches all the CREATE SCHEMA commands from the pre-data section of pg_dump. Ownership and ACLs can also be restored if they pre-exist, or the ``pgcopydb copy roles`` command can be used beforehand. + +.. include:: ../include/copy-schemas.rst + Description ----------- diff --git a/src/bin/pgcopydb/cli_copy.c b/src/bin/pgcopydb/cli_copy.c index 977d6ee96..fab585fe2 100644 --- a/src/bin/pgcopydb/cli_copy.c +++ b/src/bin/pgcopydb/cli_copy.c @@ -28,6 +28,7 @@ static void cli_copy_sequences(int argc, char **argv); static void cli_copy_indexes(int argc, char **argv); static void cli_copy_constraints(int argc, char **argv); static void cli_copy_blobs(int argc, char **argv); +static void cli_copy_schemas(int argc, char **argv); static CommandLine copy_db_command = make_command( @@ -202,6 +203,21 @@ static CommandLine copy_constraints_command = cli_copy_db_getopts, cli_copy_constraints); +static CommandLine copy_schemas_command = + make_command( + "schemas", + "Create all the schemas found in the source database in the target", + " --source ... --target ... [ --table-jobs ... --index-jobs ... ] ", + " --source Postgres URI to the source database\n" + " --target Postgres URI to the target database\n" + " --dir Work directory to use\n" + " --filters Use the filters defined in \n" + " --restart Allow restarting when temp files exist already\n" + " --resume Allow resuming operations after a failure\n" + " --not-consistent Allow taking a new snapshot on the source database\n", + cli_copy_db_getopts, + cli_copy_schemas); + static CommandLine *copy_subcommands[] = { ©_db_command, ©_roles_command, @@ -213,6 +229,7 @@ static CommandLine *copy_subcommands[] = { ©_sequence_command, ©_indexes_command, ©_constraints_command, + ©_schemas_command, NULL }; @@ -657,3 +674,49 @@ cli_copy_extensions(int argc, char **argv) exit(EXIT_CODE_INTERNAL_ERROR); } } + + +/* + * cli_copy_schemas implements copying schemas + */ +static void +cli_copy_schemas(int argc, char **argv) +{ + CopyDataSpec copySpecs = { 0 }; + + (void) cli_copy_prepare_specs(©Specs, DATA_SECTION_SCHEMAS); + + /* + * First, we need to open a snapshot that we're going to re-use in all our + * connections to the source database. When the --snapshot option has been + * used, instead of exporting a new snapshot, we can just re-use it. + */ + if (!copydb_prepare_snapshot(©Specs)) + { + /* errors have already been logged */ + exit(EXIT_CODE_INTERNAL_ERROR); + } + + /* fetch schema information from source catalogs, including filtering */ + if (!copydb_fetch_schema_and_prepare_specs(©Specs)) + { + /* errors have already been logged */ + exit(EXIT_CODE_INTERNAL_ERROR); + } + + if (!copydb_dump_source_schema(©Specs, + copySpecs.sourceSnapshot.snapshot, + PG_DUMP_SECTION_PRE_DATA)) + { + /* errors have already been logged */ + exit(EXIT_CODE_INTERNAL_ERROR); + } + + /* Now restore the pre-data, but only the schemas */ + if (!copydb_target_prepare_schema(©Specs)) + { + log_error( + "Failed to prepare schema on the target database, see above for details"); + exit(EXIT_CODE_TARGET); + } +} diff --git a/src/bin/pgcopydb/dump_restore.c b/src/bin/pgcopydb/dump_restore.c index 3afaec019..531aa8e8e 100644 --- a/src/bin/pgcopydb/dump_restore.c +++ b/src/bin/pgcopydb/dump_restore.c @@ -646,12 +646,35 @@ copydb_write_restore_list(CopyDataSpec *specs, PostgresDumpSection section) bool skip = false; + log_trace("Processing dumpId %d: %s %u %u %s", + item->dumpId, + item->description, + item->catalogOid, + item->objectOid, + item->restoreListName); + + /* + * Skip everything except SCHEMAS when specs->section == DATA_SECTION_SCHEMAS + */ + if (specs->section == DATA_SECTION_SCHEMAS && + (item->isCompositeTag ? + item->tagType != ARCHIVE_TAG_TYPE_SCHEMA : + item->desc != ARCHIVE_TAG_SCHEMA)) + { + skip = true; + log_debug("Skipping non schema %d: %s %u %s", + item->dumpId, + item->description, + item->objectOid, + item->restoreListName); + } + /* * Skip COMMENT ON EXTENSION when either of the option * --skip-extensions or --skip-ext-comment has been used. */ - if ((specs->skipExtensions || - specs->skipCommentOnExtension) && + if (!skip && (specs->skipExtensions || + specs->skipCommentOnExtension) && item->isCompositeTag && item->tagKind == ARCHIVE_TAG_KIND_COMMENT && item->tagType == ARCHIVE_TAG_TYPE_EXTENSION) diff --git a/src/bin/pgcopydb/extensions.c b/src/bin/pgcopydb/extensions.c index ecb5d4bbe..b3cdd97c4 100644 --- a/src/bin/pgcopydb/extensions.c +++ b/src/bin/pgcopydb/extensions.c @@ -233,8 +233,8 @@ copydb_copy_extensions_hook(void *ctx, SourceExtension *ext) HASH_FIND(hh, context->reqs, extname, strlen(extname), req); appendPQExpBuffer(sql, - "create extension if not exists \"%s\" cascade", - ext->extname); + "create extension if not exists \"%s\" with schema \"%s\" cascade", + ext->extname, ext->extnamespace); if (req != NULL) { @@ -248,6 +248,7 @@ copydb_copy_extensions_hook(void *ctx, SourceExtension *ext) log_error("Failed to build CREATE EXTENSION sql buffer: " "Out of Memory"); (void) destroyPQExpBuffer(sql); + return false; } log_info("Creating extension \"%s\"", ext->extname); @@ -255,6 +256,8 @@ copydb_copy_extensions_hook(void *ctx, SourceExtension *ext) if (!pgsql_execute(dst, sql->data)) { log_error("Failed to create extension \"%s\"", ext->extname); + (void) destroyPQExpBuffer(sql); + return false; } (void) destroyPQExpBuffer(sql); diff --git a/tests/extensions/copydb.sh b/tests/extensions/copydb.sh index cf94a7af9..215ef7b45 100755 --- a/tests/extensions/copydb.sh +++ b/tests/extensions/copydb.sh @@ -31,6 +31,8 @@ EOF psql -a -1 ${PGCOPYDB_SOURCE_PGURI_SU} < Date: Thu, 18 Apr 2024 15:20:52 +0100 Subject: [PATCH 2/2] No extension changes --- src/bin/pgcopydb/extensions.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/bin/pgcopydb/extensions.c b/src/bin/pgcopydb/extensions.c index b3cdd97c4..ecb5d4bbe 100644 --- a/src/bin/pgcopydb/extensions.c +++ b/src/bin/pgcopydb/extensions.c @@ -233,8 +233,8 @@ copydb_copy_extensions_hook(void *ctx, SourceExtension *ext) HASH_FIND(hh, context->reqs, extname, strlen(extname), req); appendPQExpBuffer(sql, - "create extension if not exists \"%s\" with schema \"%s\" cascade", - ext->extname, ext->extnamespace); + "create extension if not exists \"%s\" cascade", + ext->extname); if (req != NULL) { @@ -248,7 +248,6 @@ copydb_copy_extensions_hook(void *ctx, SourceExtension *ext) log_error("Failed to build CREATE EXTENSION sql buffer: " "Out of Memory"); (void) destroyPQExpBuffer(sql); - return false; } log_info("Creating extension \"%s\"", ext->extname); @@ -256,8 +255,6 @@ copydb_copy_extensions_hook(void *ctx, SourceExtension *ext) if (!pgsql_execute(dst, sql->data)) { log_error("Failed to create extension \"%s\"", ext->extname); - (void) destroyPQExpBuffer(sql); - return false; } (void) destroyPQExpBuffer(sql);