-
Notifications
You must be signed in to change notification settings - Fork 2
[Percona Live 2021] Case study
DONG CHAN, SUNG edited this page Apr 22, 2021
·
16 revisions
First, create an account and schema to be used in this test as follows.
DROP USER IF EXISTS 'repl'@'%';
DROP USER IF EXISTS 'target'@'%';
DROP USER IF EXISTS 'target01'@'%';
DROP USER IF EXISTS 'target02'@'%';
## ID: repl, PW: repl
CREATE USER 'repl'@'%' IDENTIFIED WITH 'mysql_native_password' AS '*A424E797037BF97C19A2E88CF7891C5C2038C039';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO `repl`@`%`;
## ID: target, PW: target
CREATE USER 'target'@'%' IDENTIFIED WITH 'mysql_native_password' AS '*E4F5876ABD8277202DB74BE1D22FCBBAB218967C';
GRANT ALL PRIVILEGES ON `target`.* TO `target`@`%`;
CREATE USER 'target01'@'%' IDENTIFIED WITH 'mysql_native_password' AS '*4FE42250EFBD569111001D2A54D8DED5F2A4B683';
GRANT ALL PRIVILEGES ON `target01`.* TO `target01`@`%`;
CREATE USER 'target02'@'%' IDENTIFIED WITH 'mysql_native_password' AS '*DC8BFD6BE5F7DF92F69B75ED7D43EECA0F7A12EC';
GRANT ALL PRIVILEGES ON `target02`.* TO `target02`@`%`;git clone https://github.com/gywndi/uldra-replicator.git
cd uldra-replicator
mvn install
ls -al target/uldra-replicator-0.0.1.jar
-rw-r--r-- 1 chan staff 16795166 4 24 13:04 target/uldra-replicator-0.0.1.jar## origin schema
DROP DATABASE IF EXISTS origin;
CREATE DATABASE origin;
USE origin;
CREATE TABLE `s_user` (
`userid` varchar(10) NOT NULL,
`name` varchar(20) DEFAULT ' ',
`email` varchar(50) DEFAULT ' ',
`phone` varchar(20) DEFAULT ' ',
`addr` varchar(100) DEFAULT ' ',
`reg_dttm` datetime DEFAULT NULL,
`mod_tmsp` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`userid`)
) ENGINE=InnoDB;
## target schema
DROP DATABASE IF EXISTS target;
CREATE DATABASE target;
USE target;
CREATE TABLE `t_user_name` (
`userid` varchar(10) NOT NULL,
`name` varchar(20) DEFAULT ' ',
`reg_dttm` datetime DEFAULT NULL,
`mod_tmsp` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`userid`)
) ENGINE=InnoDB;
CREATE TABLE `t_user_contact` (
`userid` varchar(10) NOT NULL,
`email` varchar(50) DEFAULT ' ',
`phone` varchar(20) DEFAULT ' ',
`addr` varchar(100) DEFAULT ' ',
`reg_dttm` datetime DEFAULT NULL,
`mod_tmsp` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`userid`)
) ENGINE=InnoDB;
reset master;workerCount: 16
wokerQueueSize: 5000
binlogServer: 127.0.0.1:3306
binlogServerID: 5
binlogServerUsername: repl
binlogServerPassword: repl
binlogInfoFile: "binlog.info"
binlogPolicies:
- name: origin.s_user
groupKey: userid
replicatPolicies:
- name: "t_user_name"
colums: ["userid", "name", "reg_dttm", "mod_tmsp"]
- name: "t_user_contact"
colums: ["userid", "email", "phone", "addr", "reg_dttm", "mod_tmsp"]
dataSources:
- !!org.apache.commons.dbcp2.BasicDataSource
url: jdbc:mysql://127.0.0.1:3306/target?autoReconnect=true&useSSL=false&sessionVariables=SQL_MODE='NO_AUTO_VALUE_ON_ZERO'
username: target
password: target
maxTotal: 30
maxWaitMillis: 100
validationQuery: SELECT 1
testOnBorrow: false
testOnReturn: false
testWhileIdle: true
timeBetweenEvictionRunsMillis: 60000
minEvictableIdleTimeMillis : 1200000
numTestsPerEvictionRun : 10rm -rf binlog.info
java -jar target/uldra-replicator-0.0.1.jar --config-file="uldra-config.yml"##########################
## Insert source table
##########################
mysql> insert into origin.s_user values ('user01', 'user01-name', 'user01@email.com', '00000-01', 'street01', now(), now());
mysql> insert into origin.s_user values ('user02', 'user02-name', 'user02@email.com', '00000-02', 'street02', now(), now());
mysql> insert into origin.s_user values ('user03', 'user03-name', 'user03@email.com', '00000-03', 'street03', now(), now());
mysql> insert into origin.s_user values ('user04', 'user04-name', 'user04@email.com', '00000-04', 'street04', now(), now());
mysql> insert into origin.s_user values ('user05', 'user05-name', 'user05@email.com', '00000-05', 'street05', now(), now());
mysql> insert into origin.s_user values ('user06', 'user06-name', 'user06@email.com', '00000-06', 'street06', now(), now());
mysql> insert into origin.s_user values ('user07', 'user07-name', 'user07@email.com', '00000-07', 'street07', now(), now());
mysql> insert into origin.s_user values ('user08', 'user08-name', 'user08@email.com', '00000-08', 'street08', now(), now());
mysql> insert into origin.s_user values ('user09', 'user09-name', 'user09@email.com', '00000-09', 'street09', now(), now());
## Result
mysql> select * from target.t_user_name;
+--------+-------------+---------------------+---------------------+
| userid | name | reg_dttm | mod_tmsp |
+--------+-------------+---------------------+---------------------+
| user01 | user01-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user02 | user02-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user03 | user03-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user04 | user04-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user05 | user05-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user06 | user06-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user07 | user07-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user08 | user08-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user09 | user09-name | 2021-04-22 16:15:02 | 2021-04-22 16:15:02 |
+--------+-------------+---------------------+---------------------+
9 rows in set (0.01 sec)
mysql> select * from target.t_user_contact;
+--------+------------------+----------+----------+---------------------+---------------------+
| userid | email | phone | addr | reg_dttm | mod_tmsp |
+--------+------------------+----------+----------+---------------------+---------------------+
| user01 | user01@email.com | 00000-01 | street01 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user02 | user02@email.com | 00000-02 | street02 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user03 | user03@email.com | 00000-03 | street03 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user04 | user04@email.com | 00000-04 | street04 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user05 | user05@email.com | 00000-05 | street05 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user06 | user06@email.com | 00000-06 | street06 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user07 | user07@email.com | 00000-07 | street07 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user08 | user08@email.com | 00000-08 | street08 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user09 | user09@email.com | 00000-09 | street09 | 2021-04-22 16:15:02 | 2021-04-22 16:15:02 |
+--------+------------------+----------+----------+---------------------+---------------------+
9 rows in set (0.01 sec)
##########################
## Update source table
##########################
## Update single column
mysql> update origin.s_user set phone = '1111-01' limit 3;
Query OK, 3 rows affected (0.00 sec)
## Result
mysql> select * from target.t_user_contact;
+--------+------------------+----------+----------+---------------------+---------------------+
| userid | email | phone | addr | reg_dttm | mod_tmsp |
+--------+------------------+----------+----------+---------------------+---------------------+
| user01 | user01@email.com | 1111-01 | street01 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user02 | user02@email.com | 1111-01 | street02 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user03 | user03@email.com | 1111-01 | street03 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user04 | user04@email.com | 00000-04 | street04 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user05 | user05@email.com | 00000-05 | street05 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user06 | user06@email.com | 00000-06 | street06 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user07 | user07@email.com | 00000-07 | street07 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user08 | user08@email.com | 00000-08 | street08 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user09 | user09@email.com | 00000-09 | street09 | 2021-04-22 16:15:02 | 2021-04-22 16:15:02 |
+--------+------------------+----------+----------+---------------------+---------------------+
9 rows in set (0.01 sec)
## Update multi-columns
mysql> update origin.s_user set name=concat(userid, '-newname'), phone = '2222-01' limit 3;
Query OK, 3 rows affected (0.00 sec)
mysql> select * from target.t_user_name;
+--------+----------------+---------------------+---------------------+
| userid | name | reg_dttm | mod_tmsp |
+--------+----------------+---------------------+---------------------+
| user01 | user01-newname | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user02 | user02-newname | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user03 | user03-newname | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user04 | user04-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user05 | user05-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user06 | user06-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user07 | user07-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user08 | user08-name | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user09 | user09-name | 2021-04-22 16:15:02 | 2021-04-22 16:15:02 |
+--------+----------------+---------------------+---------------------+
9 rows in set (0.01 sec)
## Result
mysql> select * from target.t_user_contact;
+--------+------------------+----------+----------+---------------------+---------------------+
| userid | email | phone | addr | reg_dttm | mod_tmsp |
+--------+------------------+----------+----------+---------------------+---------------------+
| user01 | user01@email.com | 2222-01 | street01 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user02 | user02@email.com | 2222-01 | street02 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user03 | user03@email.com | 2222-01 | street03 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user04 | user04@email.com | 00000-04 | street04 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user05 | user05@email.com | 00000-05 | street05 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user06 | user06@email.com | 00000-06 | street06 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user07 | user07@email.com | 00000-07 | street07 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user08 | user08@email.com | 00000-08 | street08 | 2021-04-22 16:15:01 | 2021-04-22 16:15:01 |
| user09 | user09@email.com | 00000-09 | street09 | 2021-04-22 16:15:02 | 2021-04-22 16:15:02 |
+--------+------------------+----------+----------+---------------------+---------------------+
9 rows in set (0.01 sec)
##########################
## Delete source table
##########################
## Delete all
mysql> delete from origin.s_user;
Query OK, 9 rows affected (0.03 sec)
## Result
mysql> select * from target.t_user_name;
Empty set (0.01 sec)
mysql> select * from target.t_user_contact;
Empty set (0.00 sec)
## origin schema
DROP DATABASE IF EXISTS origin;
CREATE DATABASE origin;
USE origin;
CREATE TABLE `s_user` (
`userid` varchar(10) NOT NULL,
`name` varchar(20) DEFAULT ' ',
`email` varchar(50) DEFAULT ' ',
`phone` varchar(20) DEFAULT ' ',
`addr` varchar(100) DEFAULT ' ',
`reg_dttm` datetime DEFAULT NULL,
`mod_tmsp` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`userid`)
) ENGINE=InnoDB;
CREATE TABLE `s_user_status` (
`userid` varchar(10) NOT NULL,
`vaild_yn` varchar(1) DEFAULT 'Y',
`login_dttm` varchar(16) DEFAULT ' ',
`login_cnt` int DEFAULT '1',
`reg_dttm` datetime,
`mod_tmsp` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`userid`)
) ENGINE=InnoDB;
## target schema
DROP DATABASE IF EXISTS target;
CREATE DATABASE target;
USE target;
CREATE TABLE `t_user_all_info` (
`userid` varchar(10) NOT NULL,
`name` varchar(20) DEFAULT ' ',
`email` varchar(50) DEFAULT ' ',
`phone` varchar(20) DEFAULT ' ',
`addr` varchar(100) DEFAULT ' ',
`vaild_yn` varchar(1) DEFAULT 'Y',
`login_dttm` varchar(16) DEFAULT ' ',
`login_cnt` int DEFAULT '0',
`reg_dttm` datetime,
`mod_tmsp` timestamp NULL DEFAULT NULL,
PRIMARY KEY (`userid`)
) ENGINE=InnoDB;
reset master;workerCount: 16
wokerQueueSize: 5000
binlogServer: 127.0.0.1:3306
binlogServerID: 5
binlogServerUsername: repl
binlogServerPassword: repl
binlogInfoFile: "binlog.info"
binlogPolicies:
- name: origin.s_user
groupKey: userid
replicatPolicies:
- name: "t_user_all_info"
upsertMode: true
colums: ["userid", "name", "email", "phone", "addr", "reg_dttm", "mod_tmsp"]
- name: origin.s_user_status
groupKey: userid
replicatPolicies:
- name: "t_user_all_info"
upsertMode: true
colums: ["userid", "vaild_yn", "login_dttm", "login_cnt", "mod_tmsp"]
dataSources:
- !!org.apache.commons.dbcp2.BasicDataSource
url: jdbc:mysql://127.0.0.1:3306/target?autoReconnect=true&useSSL=false&sessionVariables=SQL_MODE='NO_AUTO_VALUE_ON_ZERO'
username: target
password: target
maxTotal: 30
maxWaitMillis: 100
validationQuery: SELECT 1
testOnBorrow: false
testOnReturn: false
testWhileIdle: true
timeBetweenEvictionRunsMillis: 60000
minEvictableIdleTimeMillis : 1200000
numTestsPerEvictionRun : 10rm -rf binlog.info
java -jar target/uldra-replicator-0.0.1.jar --config-file="uldra-config.yml"##############################
## Insert source table: s_user
##############################
mysql> insert into origin.s_user values ('user01', 'user01-name', 'user01@email.com', '00000-01', 'street01', now(), now());
mysql> insert into origin.s_user values ('user02', 'user02-name', 'user02@email.com', '00000-02', 'street02', now(), now());
mysql> insert into origin.s_user values ('user03', 'user03-name', 'user03@email.com', '00000-03', 'street03', now(), now());
mysql> insert into origin.s_user values ('user04', 'user04-name', 'user04@email.com', '00000-04', 'street04', now(), now());
mysql> insert into origin.s_user values ('user05', 'user05-name', 'user05@email.com', '00000-05', 'street05', now(), now());
mysql> insert into origin.s_user values ('user06', 'user06-name', 'user06@email.com', '00000-06', 'street06', now(), now());
mysql> insert into origin.s_user values ('user07', 'user07-name', 'user07@email.com', '00000-07', 'street07', now(), now());
mysql> insert into origin.s_user values ('user08', 'user08-name', 'user08@email.com', '00000-08', 'street08', now(), now());
mysql> insert into origin.s_user values ('user09', 'user09-name', 'user09@email.com', '00000-09', 'street09', now(), now());
## Result
mysql> select * from target.t_user_all_info;
+--------+-------------+------------------+----------+----------+----------+------------+-----------+---------------------+---------------------+
| userid | name | email | phone | addr | vaild_yn | login_dttm | login_cnt | reg_dttm | mod_tmsp |
+--------+-------------+------------------+----------+----------+----------+------------+-----------+---------------------+---------------------+
| user01 | user01-name | user01@email.com | 00000-01 | street01 | Y | | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:45:23 |
| user02 | user02-name | user02@email.com | 00000-02 | street02 | Y | | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:45:23 |
| user03 | user03-name | user03@email.com | 00000-03 | street03 | Y | | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:45:23 |
| user04 | user04-name | user04@email.com | 00000-04 | street04 | Y | | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:45:23 |
| user05 | user05-name | user05@email.com | 00000-05 | street05 | Y | | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:45:23 |
| user06 | user06-name | user06@email.com | 00000-06 | street06 | Y | | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:45:23 |
| user07 | user07-name | user07@email.com | 00000-07 | street07 | Y | | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:45:23 |
| user08 | user08-name | user08@email.com | 00000-08 | street08 | Y | | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:45:23 |
| user09 | user09-name | user09@email.com | 00000-09 | street09 | Y | | 0 | 2021-04-22 18:45:24 | 2021-04-22 18:45:24 |
+--------+-------------+------------------+----------+----------+----------+------------+-----------+---------------------+---------------------+
9 rows in set (0.01 sec)
#####################################
## Insert source table: s_user_status
#####################################
mysql> insert into origin.s_user_status values ('user01', 'Y', '201901010714', '0', now(), now());
mysql> insert into origin.s_user_status values ('user02', 'Y', '201902120924', '0', now(), now());
mysql> insert into origin.s_user_status values ('user03', 'Y', '202003230134', '0', now(), now());
mysql> insert into origin.s_user_status values ('user04', 'Y', '202004040744', '0', now(), now());
mysql> insert into origin.s_user_status values ('user05', 'Y', '202005150854', '0', now(), now());
mysql> insert into origin.s_user_status values ('user06', 'Y', '202006261904', '0', now(), now());
mysql> insert into origin.s_user_status values ('user07', 'Y', '202007041014', '0', now(), now());
mysql> insert into origin.s_user_status values ('user08', 'Y', '202101151124', '0', now(), now());
mysql> insert into origin.s_user_status values ('user09', 'Y', '202102271234', '0', now(), now());
## Result
mysql> select * from target.t_user_all_info;
+--------+-------------+------------------+----------+----------+----------+--------------+-----------+---------------------+---------------------+
| userid | name | email | phone | addr | vaild_yn | login_dttm | login_cnt | reg_dttm | mod_tmsp |
+--------+-------------+------------------+----------+----------+----------+--------------+-----------+---------------------+---------------------+
| user01 | user01-name | user01@email.com | 00000-01 | street01 | Y | 201901010714 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user02 | user02-name | user02@email.com | 00000-02 | street02 | Y | 201902120924 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user03 | user03-name | user03@email.com | 00000-03 | street03 | Y | 202003230134 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user04 | user04-name | user04@email.com | 00000-04 | street04 | Y | 202004040744 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user05 | user05-name | user05@email.com | 00000-05 | street05 | Y | 202005150854 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user06 | user06-name | user06@email.com | 00000-06 | street06 | Y | 202006261904 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user07 | user07-name | user07@email.com | 00000-07 | street07 | Y | 202007041014 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user08 | user08-name | user08@email.com | 00000-08 | street08 | Y | 202101151124 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user09 | user09-name | user09@email.com | 00000-09 | street09 | Y | 202102271234 | 0 | 2021-04-22 18:45:24 | 2021-04-22 18:46:12 |
+--------+-------------+------------------+----------+----------+----------+--------------+-----------+---------------------+---------------------+
9 rows in set (0.00 sec)
#####################################
## Update source table: s_user_status
#####################################
mysql> update origin.s_user_status set login_cnt = login_cnt + 1 order by rand() limit 3;
Query OK, 3 rows affected (0.02 sec)
Rows matched: 3 Changed: 3 Warnings: 0
## Result
mysql> select * from target.t_user_all_info;
+--------+-------------+------------------+----------+----------+----------+--------------+-----------+---------------------+---------------------+
| userid | name | email | phone | addr | vaild_yn | login_dttm | login_cnt | reg_dttm | mod_tmsp |
+--------+-------------+------------------+----------+----------+----------+--------------+-----------+---------------------+---------------------+
| user01 | user01-name | user01@email.com | 00000-01 | street01 | Y | 201901010714 | 1 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user02 | user02-name | user02@email.com | 00000-02 | street02 | Y | 201902120924 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user03 | user03-name | user03@email.com | 00000-03 | street03 | Y | 202003230134 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user04 | user04-name | user04@email.com | 00000-04 | street04 | Y | 202004040744 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user05 | user05-name | user05@email.com | 00000-05 | street05 | Y | 202005150854 | 1 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user06 | user06-name | user06@email.com | 00000-06 | street06 | Y | 202006261904 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user07 | user07-name | user07@email.com | 00000-07 | street07 | Y | 202007041014 | 1 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user08 | user08-name | user08@email.com | 00000-08 | street08 | Y | 202101151124 | 0 | 2021-04-22 18:45:23 | 2021-04-22 18:46:12 |
| user09 | user09-name | user09@email.com | 00000-09 | street09 | Y | 202102271234 | 0 | 2021-04-22 18:45:24 | 2021-04-22 18:46:12 |
+--------+-------------+------------------+----------+----------+----------+--------------+-----------+---------------------+---------------------+
9 rows in set (0.01 sec)