diff --git a/docs/integrations/data-ingestion/clickpipes/mysql/source/aurora.md b/docs/integrations/data-ingestion/clickpipes/mysql/source/aurora.md index 052a08197dd..970b5056ace 100644 --- a/docs/integrations/data-ingestion/clickpipes/mysql/source/aurora.md +++ b/docs/integrations/data-ingestion/clickpipes/mysql/source/aurora.md @@ -9,6 +9,12 @@ import rds_backups from '@site/static/images/integrations/data-ingestion/clickpi import parameter_group_in_blade from '@site/static/images/integrations/data-ingestion/clickpipes/postgres/source/rds/parameter_group_in_blade.png'; import security_group_in_rds_mysql from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/source/rds/security-group-in-rds-mysql.png'; import edit_inbound_rules from '@site/static/images/integrations/data-ingestion/clickpipes/postgres/source/rds/edit_inbound_rules.png'; +import aurora_config from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/rds_config.png'; +import binlog_format from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_format.png'; +import binlog_row_image from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_image.png'; +import binlog_row_metadata from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_metadata.png'; +import edit_button from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/edit_button.png'; +import enable_gtid from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/enable_gtid.png'; import Image from '@theme/IdealImage'; # Aurora MySQL source setup guide @@ -33,15 +39,65 @@ mysql=> call mysql.rds_set_configuration('binlog retention hours', 24); ``` If this configuration isn't set, Amazon RDS purges the binary logs as soon as possible, leading to gaps in the binary logs. -If not already configured, make sure to set these in the parameter group: -1. `binlog_format` to `ROW` +## Configure binlog settings in the parameter group {#binlog-parameter-group-aurora} + +The parameter group can be found when you click on your MySQL instance in the RDS Console, and then heading over to the `Configurations` tab. + +Where to find parameter group in RDS + +Upon clicking on the parameter group link, you will be taken to the page for it. You will see an Edit button in the top-right. + +Edit parameter group + +The following settings need to be set as follows: + +1. `binlog_format` to `ROW`. + +Binlog format to ROW + 2. `binlog_row_metadata` to `FULL` + +Binlog row metadata + 3. `binlog_row_image` to `FULL` + +Binlog row image + +Then click on `Save Changes` in the top-right. You may need to reboot your instance for the changes to take effect - a way of knowing this is if you see `Pending reboot` next to the parameter group link in the Configurations tab of the RDS instance.
:::tip If you have a MySQL cluster, the above parameters would be found in a [DB Cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.CreatingCluster.html) parameter group and not the DB instance group. ::: +## Enabling GTID Mode {#gtid-mode-aurora} +Global Transaction Identifiers (GTIDs) are unique IDs assigned to each committed transaction in MySQL. They simplify binlog replication and make troubleshooting more straightforward. + +If your MySQL instance is MySQL 5.7, 8.0 or 8.4, we recommend enabling GTID mode so that the MySQL ClickPipe can use GTID replication. + +To enable GTID mode for your MySQL instance, follow the steps as follows: +1. In the RDS Console, click on your MySQL instance. +2. Click on the `Configurations` tab. +3. Click on the parameter group link. +4. Click on the `Edit` button in the top-right corner. +5. Set the following parameters: + - `gtid_mode` to `OFF_PERMISSIVE` + - `enforce-gtid-consistency` to `ON` +6. Click on `Save Changes` in the top-right corner. +7. Reboot your instance for the changes to take effect. +8. Edit the parameter group again and set `gtid_mode` to `ON_PERMISSIVE`. Ensure `enforce-gtid-consistency` is `ON`. +9. Click on `Save Changes` in the top-right corner. +10. Reboot your instance for the changes to take effect. +11. Edit the parameter group again and set `gtid_mode` to `ON`. Ensure `enforce-gtid-consistency` is `ON`. +12. Click on `Save Changes` in the top-right corner. +13. Reboot your instance for the changes to take effect. + +GTID enabled + +
+:::tip +The MySQL ClickPipe also supports replication without GTID mode. However, enabling GTID mode is recommended for better performance and easier troubleshooting. +::: + ## Configure a database user {#configure-database-user-aurora} Connect to your Aurora MySQL instance as an admin user and execute the following commands: diff --git a/docs/integrations/data-ingestion/clickpipes/mysql/source/rds.md b/docs/integrations/data-ingestion/clickpipes/mysql/source/rds.md index 94ad922051c..dd36776b73b 100644 --- a/docs/integrations/data-ingestion/clickpipes/mysql/source/rds.md +++ b/docs/integrations/data-ingestion/clickpipes/mysql/source/rds.md @@ -9,6 +9,12 @@ import rds_backups from '@site/static/images/integrations/data-ingestion/clickpi import parameter_group_in_blade from '@site/static/images/integrations/data-ingestion/clickpipes/postgres/source/rds/parameter_group_in_blade.png'; import security_group_in_rds_mysql from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/source/rds/security-group-in-rds-mysql.png'; import edit_inbound_rules from '@site/static/images/integrations/data-ingestion/clickpipes/postgres/source/rds/edit_inbound_rules.png'; +import rds_config from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/rds_config.png'; +import binlog_format from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_format.png'; +import binlog_row_image from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_image.png'; +import binlog_row_metadata from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_metadata.png'; +import edit_button from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/edit_button.png'; +import enable_gtid from '@site/static/images/integrations/data-ingestion/clickpipes/mysql/enable_gtid.png'; import Image from '@theme/IdealImage'; # RDS MySQL source setup guide @@ -33,16 +39,67 @@ To specify the number of hours to retain binary logs on a DB instance, use the m mysql=> call mysql.rds_set_configuration('binlog retention hours', 24); ``` -If not already configured, make sure to set these in the parameter group: +## Configure binlog settings in the parameter group {#binlog-parameter-group-rds} + +The parameter group can be found when you click on your MySQL instance in the RDS Console, and then heading over to the `Configurations` tab. + +Where to find parameter group in RDS + +Upon clicking on the parameter group link, you will be taken to the page for it. You will see an Edit button in the top-right. + +Edit parameter group + +The following settings need to be set as follows: + 1. `binlog_format` to `ROW`. -2. `binlog_expire_logs_seconds` to a value >= `86400` (1 day). -3. `binlog_row_metadata` to `FULL` -4. `binlog_row_image` to `FULL` + +Binlog format to ROW + +2. `binlog_row_metadata` to `FULL` + +Binlog row metadata + +3. `binlog_row_image` to `FULL` + +Binlog row image + +Then click on `Save Changes` in the top-right. You may need to reboot your instance for the changes to take effect - a way of knowing this is if you see `Pending reboot` next to the parameter group link in the Configurations tab of the RDS instance. +
:::tip If you have a MySQL cluster, the above parameters would be found in a [DB Cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.CreatingCluster.html) parameter group and not the DB instance group. ::: +## Enabling GTID Mode {#gtid-mode-rds} +Global Transaction Identifiers (GTIDs) are unique IDs assigned to each committed transaction in MySQL. They simplify binlog replication and make troubleshooting more straightforward. + +If your MySQL instance is MySQL 5.7, 8.0 or 8.4, we recommend enabling GTID mode so that the MySQL ClickPipe can use GTID replication. + +To enable GTID mode for your MySQL instance, follow the steps as follows: +1. In the RDS Console, click on your MySQL instance. +2. Click on the `Configurations` tab. +3. Click on the parameter group link. +4. Click on the `Edit` button in the top-right corner. +5. Set the following parameters: + - `gtid_mode` to `OFF_PERMISSIVE` + - `enforce-gtid-consistency` to `ON` +6. Click on `Save Changes` in the top-right corner. +7. Reboot your instance for the changes to take effect. +8. Edit the parameter group again and set `gtid_mode` to `ON_PERMISSIVE`. Ensure `enforce-gtid-consistency` is `ON`. +9. Click on `Save Changes` in the top-right corner. +10. Reboot your instance for the changes to take effect. +11. Edit the parameter group again and set `gtid_mode` to `ON`. Ensure `enforce-gtid-consistency` is `ON`. +12. Click on `Save Changes` in the top-right corner. +13. Reboot your instance for the changes to take effect. + +GTID enabled + +
+:::tip +The MySQL ClickPipe also supports replication without GTID mode. However, enabling GTID mode is recommended for better performance and easier troubleshooting. +::: + + ## Configure a database user {#configure-database-user-rds} Connect to your RDS MySQL instance as an admin user and execute the following commands: diff --git a/scripts/aspell-ignore/en/aspell-dict.txt b/scripts/aspell-ignore/en/aspell-dict.txt index eb8926c49b1..f39ca6aebe9 100644 --- a/scripts/aspell-ignore/en/aspell-dict.txt +++ b/scripts/aspell-ignore/en/aspell-dict.txt @@ -380,6 +380,7 @@ Fuzzers GDPR GHCN GTID +GTIDs GTest GUID Gb diff --git a/static/images/integrations/data-ingestion/clickpipes/mysql/enable_gtid.png b/static/images/integrations/data-ingestion/clickpipes/mysql/enable_gtid.png new file mode 100644 index 00000000000..7e79e38e240 Binary files /dev/null and b/static/images/integrations/data-ingestion/clickpipes/mysql/enable_gtid.png differ diff --git a/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/aurora_config.png b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/aurora_config.png new file mode 100644 index 00000000000..d1888d1b9a5 Binary files /dev/null and b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/aurora_config.png differ diff --git a/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_format.png b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_format.png new file mode 100644 index 00000000000..69ac2109189 Binary files /dev/null and b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_format.png differ diff --git a/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_image.png b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_image.png new file mode 100644 index 00000000000..4c6b978dfbd Binary files /dev/null and b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_image.png differ diff --git a/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_metadata.png b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_metadata.png new file mode 100644 index 00000000000..5aed85376e6 Binary files /dev/null and b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/binlog_row_metadata.png differ diff --git a/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/edit_button.png b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/edit_button.png new file mode 100644 index 00000000000..07811a85d31 Binary files /dev/null and b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/edit_button.png differ diff --git a/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/rds_config.png b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/rds_config.png new file mode 100644 index 00000000000..289fd204db7 Binary files /dev/null and b/static/images/integrations/data-ingestion/clickpipes/mysql/parameter_group/rds_config.png differ