PS-10245 feature: Implement receiving binlog events in GTID mode (part 5)

CMakeLists.txt

@@ -224,6 +224,10 @@ set(source_files
   src/binsrv/basic_storage_backend.hpp
   src/binsrv/basic_storage_backend.cpp
 
+  src/binsrv/binlog_file_metadata_fwd.hpp
+  src/binsrv/binlog_file_metadata.hpp
+  src/binsrv/binlog_file_metadata.cpp
+
   src/binsrv/cout_logger.hpp
   src/binsrv/cout_logger.cpp
 

mtr/binlog_streaming/include/set_up_binsrv_environment.inc

@@ -14,6 +14,7 @@
 #   --let $binsrv_tls_version = TLSv1.2 (optional)
 #   --let $binsrv_idle_time = 10
 #   --let $binsrv_verify_checksum = TRUE | FALSE
+#   --let $binsrv_replication_mode = position | gtid
 #   --let $binsrv_checkpoint_size = 2M (optional)
 #   --let $binsrv_checkpoint_interval = 30s (optional)
 #   --source set_up_binsrv_environment.inc
@@ -82,7 +83,7 @@ eval SET @binsrv_config_json = JSON_OBJECT(
      'server_id', @@server_id + 1,
      'idle_time', $binsrv_idle_time,
      'verify_checksum', $binsrv_verify_checksum,
-     'mode', 'position'
+     'mode', '$binsrv_replication_mode'
   ),
   'storage', JSON_OBJECT(
      'backend', '$storage_backend',

mtr/binlog_streaming/r/resume_streaming.result

@@ -0,0 +1,105 @@
+*** Resetting replication at the very beginning of the test.
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+
+*** 1. Executing the Binlog Server utility to for the very first
+***    time on an empty storage and receive no real events.
+
+*** 1a. Executing the Binlog Server utility on a storage that has
+***     only one binlog file that has only magic payload in it and
+***     receive no events.
+
+*** Creating a simple table
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+*** 2. Executing the Binlog Server utility on a storage that has
+***    only one binlog file that has only magic payload in it and
+***    receive events from the CREATE TABLE transaction.
+
+*** 2a. Executing the Binlog Server utility on a storage that has
+***     only one binlog file that has CREATE TABLE transaction and
+***     receive no events.
+
+*** Filling the table with some data (one transaction with one
+*** insert and another one with two inserts).
+INSERT INTO t1 VALUES(DEFAULT);
+START TRANSACTION;
+INSERT INTO t1 VALUES(DEFAULT);
+INSERT INTO t1 VALUES(DEFAULT);
+COMMIT;
+
+*** 3. Executing the Binlog Server utility on a storage that has
+***    only one binlog file with some data and receive events from
+***    the INSERT transactions.
+
+*** 3a. Executing the Binlog Server utility on a storage that has
+***     only one binlog file with some data and receive no events.
+
+*** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+*** 4. Executing the Binlog Server utility on a storage that has
+***    only one binlog file with some data and receive a single
+***    ROTATE event.
+
+*** 4a. Executing the Binlog Server utility on a storage that has
+***     one binlog file with some data and another one with just
+***     magic payload and receive no events.
+
+*** Filling the table with some more data (one transaction with one
+*** insert and another one with two inserts).
+INSERT INTO t1 VALUES(DEFAULT);
+START TRANSACTION;
+INSERT INTO t1 VALUES(DEFAULT);
+INSERT INTO t1 VALUES(DEFAULT);
+COMMIT;
+
+*** 5. Executing the Binlog Server utility on a storage that has
+***    one binlog file with some data and another one with just
+***    magic payload and receive events from the second group of
+***    INSERT transactions.
+
+*** 5a. Executing the Binlog Server utility on a storage that has
+***     two binlog files with some data and receive no events
+
+*** Flushing the second binary log and switching to the third one.
+*** Immediately after that updating data inserted previously.
+FLUSH BINARY LOGS;
+UPDATE t1 SET id = id + 100;
+
+*** 6. Executing the Binlog Server utility on a storage that has
+***    two binlog files with some data and receive a ROTATE event
+***    followed by events from the UPDATE transaction
+
+*** 6a. Executing the Binlog Server utility on a storage that has
+***     three binlog files with some data and receive no events.
+
+*** Flushing the third binary log and switching to the fourth one.
+*** Immediately after that deleting some data updated previously and
+*** flushing one more time switching to the fifth binary log file.
+FLUSH BINARY LOGS;
+DELETE FROM t1 WHERE id <= 103;
+FLUSH BINARY LOGS;
+
+*** 7. Executing the Binlog Server utility on a storage that has
+***    three binlog files with some data and receive a ROTATE
+***    event followed by a events from the DELETE transaction
+***    followed by another ROTATE event
+
+*** 7a. Executing the Binlog Server utility on a storage that has
+***     five binlog files with some data and receive no events.
+
+*** Dropping the table.
+DROP TABLE t1;
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.

mtr/binlog_streaming/t/binsrv.test

@@ -42,6 +42,7 @@ INSERT INTO t1 VALUES(DEFAULT);
 --let $binsrv_read_timeout = 60
 --let $binsrv_idle_time = 10
 --let $binsrv_verify_checksum = $extracted_init_connect_variable_value
+--let $binsrv_replication_mode = position
 --let $binsrv_checkpoint_size = 1
 --source ../include/set_up_binsrv_environment.inc
 

mtr/binlog_streaming/t/checkpointing.test

@@ -58,7 +58,8 @@ DROP TABLE t1;
 --let $binsrv_read_timeout = 60
 --let $binsrv_idle_time = 10
 --let $binsrv_verify_checksum = TRUE
-# Enabling checkointing at about 40% of expected single binlog file size, so that it
+--let $binsrv_replication_mode = position
+# Enabling checkpointing at about 40% of expected single binlog file size, so that it
 # will happen twice before rotation.
 # Like wise, from the time point of view, make interval pretty low so that
 # at lease a few interval checkpointing events will happen.

mtr/binlog_streaming/t/kill_and_restart.test

@@ -130,6 +130,7 @@ INSERT INTO t1 VALUES(DEFAULT);
 --let $binsrv_read_timeout = 60
 --let $binsrv_idle_time = 10
 --let $binsrv_verify_checksum = TRUE
+--let $binsrv_replication_mode = position
 --source ../include/set_up_binsrv_environment.inc
 
 --echo

mtr/binlog_streaming/t/pull_mode.test

@@ -19,6 +19,7 @@ eval $stmt_reset_binary_logs_and_gtids;
 --let $binsrv_read_timeout = 3
 --let $binsrv_idle_time = 1
 --let $binsrv_verify_checksum = TRUE
+--let $binsrv_replication_mode = position
 --source ../include/set_up_binsrv_environment.inc
 
 --echo

mtr/binlog_streaming/t/resume_streaming.combinations

@@ -0,0 +1,15 @@
+[position_buffered]
+init-connect = SET @binsrv_buffering_mode = 'buffered'
+
+[position_unbuffered]
+init-connect = SET @binsrv_buffering_mode = 'unbuffered'
+
+[gtid_buffered]
+gtid-mode=on
+enforce-gtid-consistency
+init-connect = SET @binsrv_buffering_mode = 'buffered'
+
+[gtid_unbuffered]
+gtid-mode=on
+enforce-gtid-consistency
+init-connect = SET @binsrv_buffering_mode = 'unbuffered'

mtr/binlog_streaming/t/resume_streaming.test

@@ -0,0 +1,161 @@
+--source ../include/have_binsrv.inc
+
+--source ../include/v80_v84_compatibility_defines.inc
+
+# in case of --repeat=N, we need to start from a fresh binary log to make
+# this test deterministic
+--echo *** Resetting replication at the very beginning of the test.
+--disable_query_log
+eval $stmt_reset_binary_logs_and_gtids;
+--enable_query_log
+
+# identifying backend storage type ('file' or 's3')
+--source ../include/identify_storage_backend.inc
+
+# identifying utility buffering mode from the conbination
+--let $extracted_init_connect_variable_name = binsrv_buffering_mode
+--source ../include/extract_init_connect_variable_value.inc
+
+# creating data directory, configuration file, etc.
+--let $binsrv_connect_timeout = 20
+--let $binsrv_read_timeout = 60
+--let $binsrv_idle_time = 10
+--let $binsrv_verify_checksum = TRUE
+--let $binsrv_replication_mode = `SELECT IF(@@global.gtid_mode, 'gtid', 'position')`
+if ($extracted_init_connect_variable_value == 'buffered')
+{
+  --let $binsrv_checkpoint_size = 1G
+}
+if ($extracted_init_connect_variable_value == 'unbuffered')
+{
+  --let $binsrv_checkpoint_size = 1
+}
+--source ../include/set_up_binsrv_environment.inc
+
+--echo
+--echo *** 1. Executing the Binlog Server utility to for the very first
+--echo ***    time on an empty storage and receive no real events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 1a. Executing the Binlog Server utility on a storage that has
+--echo ***     only one binlog file that has only magic payload in it and
+--echo ***     receive no events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Creating a simple table
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+--echo
+--echo *** 2. Executing the Binlog Server utility on a storage that has
+--echo ***    only one binlog file that has only magic payload in it and
+--echo ***    receive events from the CREATE TABLE transaction.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 2a. Executing the Binlog Server utility on a storage that has
+--echo ***     only one binlog file that has CREATE TABLE transaction and
+--echo ***     receive no events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Filling the table with some data (one transaction with one
+--echo *** insert and another one with two inserts).
+INSERT INTO t1 VALUES(DEFAULT);
+START TRANSACTION;
+INSERT INTO t1 VALUES(DEFAULT);
+INSERT INTO t1 VALUES(DEFAULT);
+COMMIT;
+
+--echo
+--echo *** 3. Executing the Binlog Server utility on a storage that has
+--echo ***    only one binlog file with some data and receive events from
+--echo ***    the INSERT transactions.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 3a. Executing the Binlog Server utility on a storage that has
+--echo ***     only one binlog file with some data and receive no events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+--echo
+--echo *** 4. Executing the Binlog Server utility on a storage that has
+--echo ***    only one binlog file with some data and receive a single
+--echo ***    ROTATE event.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 4a. Executing the Binlog Server utility on a storage that has
+--echo ***     one binlog file with some data and another one with just
+--echo ***     magic payload and receive no events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Filling the table with some more data (one transaction with one
+--echo *** insert and another one with two inserts).
+INSERT INTO t1 VALUES(DEFAULT);
+START TRANSACTION;
+INSERT INTO t1 VALUES(DEFAULT);
+INSERT INTO t1 VALUES(DEFAULT);
+COMMIT;
+
+--echo
+--echo *** 5. Executing the Binlog Server utility on a storage that has
+--echo ***    one binlog file with some data and another one with just
+--echo ***    magic payload and receive events from the second group of
+--echo ***    INSERT transactions.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 5a. Executing the Binlog Server utility on a storage that has
+--echo ***     two binlog files with some data and receive no events
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Flushing the second binary log and switching to the third one.
+--echo *** Immediately after that updating data inserted previously.
+FLUSH BINARY LOGS;
+UPDATE t1 SET id = id + 100;
+
+--echo
+--echo *** 6. Executing the Binlog Server utility on a storage that has
+--echo ***    two binlog files with some data and receive a ROTATE event
+--echo ***    followed by events from the UPDATE transaction
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 6a. Executing the Binlog Server utility on a storage that has
+--echo ***     three binlog files with some data and receive no events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Flushing the third binary log and switching to the fourth one.
+--echo *** Immediately after that deleting some data updated previously and
+--echo *** flushing one more time switching to the fifth binary log file.
+FLUSH BINARY LOGS;
+DELETE FROM t1 WHERE id <= 103;
+FLUSH BINARY LOGS;
+
+--echo
+--echo *** 7. Executing the Binlog Server utility on a storage that has
+--echo ***    three binlog files with some data and receive a ROTATE
+--echo ***    event followed by a events from the DELETE transaction
+--echo ***    followed by another ROTATE event
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 7a. Executing the Binlog Server utility on a storage that has
+--echo ***     five binlog files with some data and receive no events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Dropping the table.
+DROP TABLE t1;
+
+# cleaning up
+--source ../include/tear_down_binsrv_environment.inc

mtr/binlog_streaming/t/ssl_connection.test

@@ -36,6 +36,7 @@ eval GRANT REPLICATION SLAVE ON *.* TO '$x509_user_name'@'$localhost_ip';
 --let $binsrv_read_timeout = 60
 --let $binsrv_idle_time = 10
 --let $binsrv_verify_checksum = TRUE
+--let $binsrv_replication_mode = position
 
 # a query that checks SSL connection status
 --let $ssl_status_query = SELECT IF(VARIABLE_VALUE = '', '<unspecified>', VARIABLE_VALUE) AS SSL_status FROM performance_schema.session_status WHERE VARIABLE_NAME = 'Ssl_version'