MySQL Admin Cookbook [1st New edition] 1847197965, 9781847197962

Citation preview

0\64/$GPLQ &RRNERRN JUHDWUHFLSHVIRUPDVWHULQJ0\64/FRQÀJXUDWLRQDQG DGPLQLVWUDWLRQ

ĂŶŝĞů^ĐŚŶĞůůĞƌ hĚŽ^ĐŚǁĞĚƚ

%,50,1*+$0080%$,

0\64/$GPLQ&RRNERRN &RS\ULJKW‹3DFNW3XEOLVKLQJ

$OOULJKWVUHVHUYHG1RSDUWRIWKLVERRNPD\EHUHSURGXFHGVWRUHGLQDUHWULHYDOV\VWHPRU WUDQVPLWWHGLQDQ\IRUPRUE\DQ\PHDQVZLWKRXWWKHSULRUZULWWHQSHUPLVVLRQRIWKHSXEOLVKHU H[FHSWLQWKHFDVHRIEULHITXRWDWLRQVHPEHGGHGLQFULWLFDODUWLFOHVRUUHYLHZV (YHU\HIIRUWKDVEHHQPDGHLQWKHSUHSDUDWLRQRIWKLVERRNWRHQVXUHWKHDFFXUDF\RIWKH LQIRUPDWLRQSUHVHQWHG+RZHYHUWKHLQIRUPDWLRQFRQWDLQHGLQWKLVERRNLVVROGZLWKRXW ZDUUDQW\HLWKHUH[SUHVVRULPSOLHG1HLWKHUWKHDXWKRUVQRU3DFNW3XEOLVKLQJDQGLWVGHDOHUV DQGGLVWULEXWRUVZLOOEHKHOGOLDEOHIRUDQ\GDPDJHVFDXVHGRUDOOHJHGWREHFDXVHGGLUHFWO\RU LQGLUHFWO\E\WKLVERRN 3DFNW3XEOLVKLQJKDVHQGHDYRUHGWRSURYLGHWUDGHPDUNLQIRUPDWLRQDERXWDOORIWKHFRPSDQLHV DQGSURGXFWVPHQWLRQHGLQWKLVERRNE\WKHDSSURSULDWHXVHRIFDSLWDOV+RZHYHU3DFNW 3XEOLVKLQJFDQQRWJXDUDQWHHWKHDFFXUDF\RIWKLVLQIRUPDWLRQ

)LUVWSXEOLVKHG0DUFK

3URGXFWLRQ5HIHUHQFH

3XEOLVKHGE\3DFNW3XEOLVKLQJ/WG /LQFROQ5RDG 2OWRQ %LUPLQJKDP%3$8. ,6%1 ZZZSDFNWSXEFRP &RYHU,PDJHE\9LQD\DN&KLWWDU[email protected] 

&UHGLWV $XWKRUV 'DQLHO6FKQHOOHU

(GLWRULDO7HDP/HDGHU $NVKDUD$ZDUH

8GR6FKZHGW ,QGH[HU 5HYLHZHUV

5HNKD1DLU

.DL6HLGOHU 0DUF'HOLVOH $FTXLVLWLRQ(GLWRU 6DUDK&XOOLQJWRQ 'HYHORSPHQW(GLWRU 5HVKPD6XQGDUHVDQ 7HFKQLFDO(GLWRUV 3DOODYL.DFKDUH %KXSDOL.KXOH $DURQ5RVDULR &RS\(GLWRU /DNVKPL0HQRQ

3URMHFW7HDP/HDGHU /DWD%DVDQWDQL 3URMHFW&RRUGLQDWRU 6KXEKDQMDQ&KDWWHUMHH 3URRIUHDGHU &KULV6PLWK *UDSKLFV *HHWDQMDOL6DZDQW 3URGXFWLRQ&RRUGLQDWRU 0HOZ\Q$UXQ' VD &RYHU:RUN 0HOZ\Q$UXQ' VD

$ERXWWKH$XWKRUV 'DQLHO6FKQHOOHUZRUNVDVDVRIWZDUHGHYHORSHUGDWDEDVHDGPLQLVWUDWRUDQGJHQHUDO

,7SURIHVVLRQDOIRUDQLQGHSHQGHQWVRIWZDUHYHQGRULQWKHUHWDLOVHFWRU$IWHUVXFFHVVIXOO\ JUDGXDWLQJIURPWKH8QLYHUVLW\RI&RRSHUDWLYH(GXFDWLRQLQ+HLGHQKHLP*HUPDQ\ZLWKD GHJUHHLQ%XVLQHVV&RPSXWHU6FLHQFHKHVWDUWHGKLVFDUHHUDVDSURIHVVLRQDOVRIWZDUH GHYHORSHUIRFXVHGRQWKH0LFURVRIWWHFKQRORJ\VWDFN,QKHVWDUWHGIRFXVLQJRQ HQWHUSULVHOHYHO-DYDGHYHORSPHQWDQGKDVVLQFHJDLQHGH[WHQVLYHNQRZOHGJHDQGH[SHULHQFH LPSOHPHQWLQJODUJHVFDOHV\VWHPVEDVHGRQ-DYD((DQGUHODWLRQDOGDWDEDVHVHVSHFLDOO\ 0\64/VLQFHYHUVLRQ

&XUUHQWO\KHLVPRVWO\LQYROYHGZLWKWKHRQJRLQJGHYHORSPHQWRIIUDPHZRUNOHYHO IXQFWLRQDOLW\LQFOXGLQJFXVWRPL]DWLRQDQGH[WHQVLRQRIDQ250EDVHGSHUVLVWHQFHOD\HU +HLVLQYROYHGLQGLIIHUHQWRSHQVRXUFHSURMHFWVVXFKDV)LQG%XJV(FOLSVHDQG&KHFNVW\OH DQGLQIUHTXHQWO\EORJVDERXW-DYD0\64/:LQGRZV/LQX[DQGRWKHULQVDQLWLHVDW http://www.danielschneller.com tŚĞŶ/ĮƌƐƚǁĂƐĂƐŬĞĚďLJWĂĐŬƚWƵďůŝƐŚŝŶŐǁŚĞƚŚĞƌ/ǁŽƵůĚďĞŝŶƚĞƌĞƐƚĞĚŝŶ ǁƌŝƟŶŐĂŬĂďŽƵƚDLJ^Y>ŽŶŚƌŝƐƚŵĂƐǀĞϮϬϬϴůŝƩůĞĚŝĚ/ŬŶŽǁŚŽǁŵƵĐŚ ǁŽƌŬ͕ƐƚƌĞƐƐ͕ďƵƚĂůƐŽǁŚĂƚĂůŽƚŽĨĨƵŶ/ǁĂƐŚĞĂĚĞĚĨŽƌ͘ EŽǁ͕ƚŚĂƚƚŚĞŬŝƐĮŶĂůůLJĚŽŶĞ/ǁŽƵůĚůŝŬĞƚŽƚŚĂŶŬƚŚŽƐĞƉĞŽƉůĞǁŝƚŚŽƵƚ ǁŚŽŵŐĞƫŶŐŝƚĚŽŶĞǁŽƵůĚŚĂǀĞďĞĞŶŝŵƉŽƐƐŝďůĞ͘ &ŝƌƐƚŽĨĂůů͕/ΖĚůŝŬĞƚŽƚŚĂŶŬhĚŽĨŽƌĂŐƌĞĞŝŶŐƚŽďĞŵLJĐŽͲĂƵƚŚŽƌ͘tŝƚŚŽƵƚŚŝŵ͕ ƚŚŝƐǁŚŽůĞƚŚŝŶŐǁŽƵůĚŚĂǀĞƚĂŬĞŶĂůŽƚůŽŶŐĞƌĂŶĚǁŽƵůĚŚĂǀĞďĞĞŶŶŽƚŚĂůĨĂƐ ƵƐĞĨƵůĂƐ/ďĞůŝĞǀĞŝƚŚĂƐƚƵƌŶĞĚŽƵƚŶŽǁ͘ /ǁŽƵůĚĂůƐŽůŝŬĞƚŽƚŚĂŶŬƚŚĞƚĞĂŵĂƚWĂĐŬƚWƵďůŝƐŚŝŶŐͶŵŽƐƚŝŵƉŽƌƚĂŶƚůLJ ĨŽƌŶŽƟĐŝŶŐĂŶĚƌĞĂĚŝŶŐŵLJďůŽŐ͕ĐŽŶƐĞƋƵĞŶƚůLJĐŽŶƚĂĐƟŶŐŵĞƚŽŐĞƚƚŚĞ ǁŚŽůĞƚŚŝŶŐƐƚĂƌƚĞĚͶďƵƚĂůƐŽĨŽƌƚĂŬŝŶŐĐĂƌĞŽĨƐĐŚĞĚƵůĞƐ͕ƉƌŽǀŝĚŝŶŐƐƵƉƉŽƌƚ͕ ŐƵŝĚĂŶĐĞĂŶĚĨĞĞĚďĂĐŬ͕ĂŶĚŬĞĞƉŝŶŐƵƐŽŶƚƌĂĐŬƚŚĞǁŚŽůĞǁĂLJ͘ >ĂƐƚ͕ďƵƚďLJŶŽŵĞĂŶƐůĞĂƐƚ͕/ǁĂŶƚƚŽƚŚĂŶŬ:ĞŶŶLJͶĨŽƌĞŶĐŽƵƌĂŐŝŶŐŵĞƚŽ ǁƌŝƚĞĂŬŝŶƚŚĞĮƌƐƚƉůĂĐĞ͕ĂŶĚƚŚĞŶŵĂŬŝŶŐƐƵƌĞ/ŶĞǀĞƌƌĂŶŽƵƚŽĨƚĞĂ͕ ĐŽŽŬŝĞƐ͕ŽƌŵŽƟǀĂƟŽŶŽŶƚŚĞĐŽƵŶƚůĞƐƐĞǀĞŶŝŶŐƐ/ƐƉĞŶƚƐŝƫŶŐŝŶĨƌŽŶƚŽĨƚŚĞ ŬĞLJďŽĂƌĚŝŶƐƚĞĂĚŽĨǁŝƚŚŚĞƌ͘/ĚĞĚŝĐĂƚĞƚŚŝƐŬƚŽŚĞƌ͘

8GR6FKZHGWKDVRYHUWHQ\HDUVRIH[SHULHQFHLQWKH,7LQGXVWU\DVDSURIHVVLRQDO-DYDGH

YHORSHUDQGVRIWZDUHDUFKLWHFW+HLVKHDGRIWKH-DYDDUFKLWHFWXUHWHDPDQGGHSXW\KHDGRI WKH-DYDGHYHORSPHQWGHSDUWPHQWDWWKH,7VHUYLFHSURYLGHUIRU*HUPDQ\ VPDUNHWOHDGHULQ WKH'R,W START SLAVE;

:KHQZHZLVKWRGUDZ\RXUDWWHQWLRQWRDSDUWLFXODUSDUWRIDFRGHEORFNWKHUHOHYDQWOLQHVRU LWHPVDUHVHWLQEROG slave> SHOW SLAVE STATUS\G ************************** 1. row *************************** ... Slave_IO_Running: Yes Slave_SQL_Running: Yes ...

1HZWHUPVDQGLPSRUWDQWZRUGVDUHVKRZQLQEROG:RUGVWKDW\RXVHHRQWKHVFUHHQLQ PHQXVRUGLDORJER[HVIRUH[DPSOHDSSHDULQWKHWH[WOLNHWKLV -> -> 

CREATE DATABASE repdb; USE repdb; CREATE TABLE tblA ( id INT(10) PRIMARY KEY NOT NULL, label VARCHAR(30) ) ENGINE=InnoDB; CREATE TABLE tblB ( name VARCHAR(20) PRIMARY KEY NOT NULL, age INT(3) ) ENGINE=InnoDB;

&KDSWHU master> INSERT INTO tblA VALUES -> (1, 'label 1'), -> (2, 'label 2'), -> (3, 'label 3'); master> INSERT INTO tblB VALUES -> ('Peter', 55), -> ('Paul', 43), -> ('Mary', 25);

,QVHUWLQJWKLVGDWDDOUHDG\FUHDWHVELQORJLQIRUPDWLRQ GRANT REPLICATION SLAVE -> ON *.* -> TO 'repblack'@'localhost' -> IDENTIFIED BY 'blackpass';

&RQÀJXUHWKHÀOWHUEODFNKROH LQVWDQFHZLWKDFRQÀJXUDWLRQÀOHQDPHGmy.black WKDW FRQWDLQVDWOHDVWWKHIROORZLQJ [client] port

= 3308

socket

= /home/ds/blacktest/black.sock

[mysqld_safe] socket

= /home/ds/blacktest/black.sock

[mysqld] log-slave-updates skip-innodb default-storage-engine=blackhole user pid-file socket port basedir

= = = = =

mysql /home/ds/blacktest/black.pid /home/ds/blacktest/black.sock 3308 /home/ds/blacktest 

5HSOLFDWLRQ datadir tmpdir language bind-address server-id log-bin relay-log

= /home/ds/blacktest/data.black = /tmp = /home/ds/blacktest/share/mysql/english = 127.0.0.1 = 2 = /home/ds/blacktest/black-bin.log = /home/ds/blacktest/black-relay.log

1RWLFHWKDWDOORFFXUUHQFHVRIPDVWHUKDYHEHHQUHSODFHGZLWKEODFN 0RUHRYHUWKHserver-idVHWWLQJKDVEHHQFKDQJHGDQGWKH log-slave-updatesskip-innodbDQGdefault-storage-engine RSWLRQVKDYHEHHQDGGHG7KHVHFRQGRQHSUHYHQWVWKLVLQVWDQFHIURP FUHDWLQJibdataWDEOHVSDFHÀOHVZKLFKZRXOGQRWEHXVHGODWHUDQ\ZD\ 7KHODVWRQHVSHFLÀHVZKLFKVWRUDJHHQJLQHWRXVHZKHQDCREATETABLE VWDWHPHQWGRHVQRWH[SOLFLWO\VSHFLI\RQHRULIWKHVSHFLÀHGHQJLQHLVQRW DYDLODEOH:HZLOOFRPHEDFNWRWKLVVRRQ



0DNHVXUHWKLVLQVWDQFHEDVLFDOO\ZRUNVE\VWDUWLQJLWWKHVDPHZD\DVWKHPDVWHU EHIRUH\RXZLOOQRWVHHWKH,QQR'%PHVVDJHVKHUHRIFRXUVH  ~/blacktest$ xterm -T BLACK -e bin/mysqld \ > --defaults-file=my.black \ > --console &



&UHDWHDVHWRIGXPSÀOHVIURPWKHPDVWHU WRVHWXSERWKWKHEODFNKROHÀOWHUDQGDQH[DPSOHVODYH7KHGHWDLOVRQZK\ZHQHHG WZRDQGLQZKLFKZD\VWKH\DUHGLIIHUHQWZLOOEHH[SODLQHGODWHU8VHWKHVHFRPPDQGV WRFUHDWHWKHÀOHVQHHGHG ~/blacktest$ bin/mysqldump -S master.sock -uroot \ > --master-data \ > --single-transaction \ > --no-create-info \ > --ignore-table=repdb.tblA \ > repdb > master_data.sql ~/blacktest$ bin/mysqldump -S master.sock -uroot \ > --no-data \ > repdb > master_struct.sql



&KDSWHU



&RQQHFWWRWKHÀOWHUVHUYHUFUHDWHWKHGDWDEDVHPDNHLWWKHGHIDXOWGDWDEDVHDQG ÀQDOO\LPSRUWWKHVWUXFWXUHLQIRUPDWLRQFUHDWHGEHIRUH ~/blacktest$ bin/mysql -uroot -S black.sock --prompt='black> ' black> CREATE DATABASE repdb; black> USE repdb; black> source master_black.sql;

$WWKLVSRLQWZHQRZKDYHWKHVWUXFWXUHRIWKHPDVWHUWUDQVIHUUHGWRWKHÀOWHUHQJLQH DGDSWHGWRXVHWKHEODFNKROHHQJLQHIRUDOOWKHWDEOHV 

6HWXSWKHUHSOLFDWLRQEHWZHHQPDVWHUDQGÀOWHUHQJLQH7RGRVRZH QHHGWRNQRZWKHH[DFWSRVLWLRQIURPZKHUHWKHÀOWHUZLOOVWDUWUHSOLFDWLQJ([WUDFWWKLV LQIRUPDWLRQIURPWKHSUHYLRXVO\WDNHQGDWDGXPSOLNHWKLV

~/blacktest$ head -n 30 master_data.sql | grep 'CHANGE MASTER TO'

:ULWHGRZQWKDWLQIRUPDWLRQZHZLOOQHHGLWLQDPRPHQW 

0RGLI\WKHmy.blackFRQÀJXUDWLRQÀOHWRFRQWDLQWKHIROORZLQJLQWKH[mysqld] VHFWLRQ replicate-ignore-table=repdb.tblA replicate-do-table=repdb.tblB

7KLVLVDYHU\VLPSOHÀOWHUVHWXSLQDUHDODSSOLFDWLRQVFHQDULRWKHVHUXOHVZLOO SUREDEO\EHPRUHFRPSOH[ 

5HVWDUWWKHÀOWHUHQJLQHWRDFWLYDWHWKHQHZFRQÀJXUDWLRQ ~/blacktest$ bin/mysqladmin -uroot -S black.sock shutdown ~/blacktest$ xterm -T BLACK -e bin/mysqld \ > --defaults-file=my.black \ > --console &



5HFRQQHFWWKHFOLHQWFRQQHFWHGWRWKHEODFNKROHHQJLQH7RGRWKLVMXVWLVVXHD SELECT1;FRPPDQG

([HFXWHWKHIROORZLQJFRPPDQGWRKRRNXSWKHÀOWHUWRWKHPDVWHU%HVXUHWRÀOOLQWKH YDOXHV\RXZURWHGRZQDPRPHQWDJRLQWKHVWDWHPHQW black> -> -> -> -> -> ->

CHANGE MASTER TO master_host='localhost', master_port=3307, master_user='repblack', master_password='blackpass', master_log_file='master-bin.000001', master_log_pos=1074; 

5HSOLFDWLRQ  5HWULHYHLQIRUPDWLRQUHTXLUHGWRVHWXSWKHÀOWHUVODYHSRUWLRQ:ULWHGRZQWKHUHVXOWV RIWKHSHOWMASTERSTATUSFRPPDQGWKH\ZLOOEHQHHGHGODWHU black> FLUSH LOGS; black> SHOW MASTER STATUS; +------------------+----------+---+---+ | File | Position | … | … | +------------------+----------+---+---+ | black-bin.000003 | 98 | | | +------------------+----------+---+---+



6WDUWWKHVODYHWKUHDGRQWKHÀOWHUHQJLQHDQGYHULI\WKDWHYHU\WKLQJLVJRLQJZHOO black> START SLAVE; black> SHOW SLAVE STATUS \G ************************** 1. row *************************** Slave_IO_State: Waiting for master to send event Master_Host: localhost Master_User: repblack Master_Port: 3307 Connect_Retry: 60 Master_Log_File: master-bin.000001 Read_Master_Log_Pos: 1074 Relay_Log_File: black-relay.000003 Relay_Log_Pos: 236 Relay_Master_Log_File: master-bin.000001 Slave_IO_Running: Yes Slave_SQL_Running: Yes ... Replicate_Do_Table: repdb.tblB Replicate_Ignore_Table: repdb.tblA ... Last_Errno: 0 Last_Error: Skip_Counter: 0 Exec_Master_Log_Pos: 1074 Relay_Log_Space: 236 ... Seconds_Behind_Master: 0

$WWKLVSRLQWZHKDYHVXFFHVVIXOO\HVWDEOLVKHGDUHSOLFDWLRQFRQQHFWLRQEHWZHHQWKH PDVWHUGDWDEDVHDQGWKHEODFNKROHEDVHGÀOWHULQVWDQFH



&KDSWHU



&KHFNWKDWQRWKLQJKDV\HWEHHQZULWWHQWR WKHÀOWHU VELQORJV%HFDXVHZHLVVXHGDFLUSHLOGSFRPPDQGRQWKHÀOWHULQVWDQFH WKHUHVKRXOGEHQRWKLQJLQWKHPRVWUHFHQWELQORJÀOH9HULI\WKLVDVIROORZV ~/blacktest$ bin/mysqlbinlog black-bin.000003



7HVWWKHÀOWHUVHWXSZLWKVRPHVWDWHPHQWVLVVXHGRQWKHPDVWHU master> UPDATE repdb.tblA -> SET label='modified label 3' -> WHERE id=3; master> INSERT INTO repdb.tblB -> VALUES ('John', 39);

:HZRXOGH[SHFWWRVHHWKHINSERTLQWKHELQORJÀOHRIWKHÀOWHULQVWDQFHEXWQRWWKH UPDATEVWDWHPHQWEHFDXVHLWPRGLÀHVtblAZKLFKLVWREHLJQRUHG 

9HULI\WKDWWKHUXOHVZRUNDVH[SHFWHGE\KDYLQJDQRWKHUORRNDWWKHÀOWHU VELQORJV ~/blacktest$ bin/mysqlbinlog black-bin.000003



5HSOLFDWLRQ 7KLVORRNVSUHFLVHO\DVH[SHFWHG³WKHINSERTLVSUHVHQWWKHUPDATELVQRZKHUHWR EHVHHQ 

6HWXSWKHFRQÀJXUDWLRQRIDVODYHXVLQJWKHVHVHWWLQJV [client] port socket

= 3309 = /home/ds/blacktest/slave.sock

[mysqld_safe] socket

= /home/ds/blacktest/slave.sock

[mysqld] user pid-file socket port basedir datadir tmpdir language

= = = = = = = =

bind-address

= 127.0.0.1

server-id relay-log

= 3 = /home/ds/blacktest/slave-relay.log

mysql /home/ds/blacktest/slave.pid /home/ds/blacktest/slave.sock 3309 /home/ds/blacktest /home/ds/blacktest/data.slave /tmp /home/ds/blacktest/share/mysql/english

1RWLFHWKDWDOORFFXUUHQFHVRIPDVWHUKDYHEHHQUHSODFHG ZLWKVODYH

$JDLQWKHserver-idVHWWLQJKDVEHHQFKDQJHGDQGWKHlog-slave-updates skip-innodbDQGdefault-storage-engineRSWLRQVWKDWZHUHSDUWRIWKHÀOWHU LQVWDQFH VFRQÀJXUDWLRQDUHQRWLQFOXGHG$OVRWKHlog-binSDUDPHWHUKDVEHHQ UHPRYHGEHFDXVHFKDQJHVRQWKHVODYHQHHGQRWEHUHFRUGHGVHSDUDWHO\  6WDUWXSWKHVODYHHQJLQH ' slave> CREATE DATABASE repdb; slave> USE repdb;

$WWKLVSRLQWWKHVODYHLVVHWXSDQGKDVDQHPSW\repdbGDWDEDVH  )LOOXSWKHVODYHGDWDEDVHZLWKWKHLQLWLDOVQDSVKRWRIWKHPDVWHU:HQHHGWRORDGWZR ÀOHVKHUH7KHGHWDLOVRIZK\DUHH[SODLQHGIXUWKHUGRZQLQWKH+RZLWZRUNVVHFWLRQ slave> source master_struct.sql; ... slave> source master_data.sql; ...

 9HULI\WKDW\RXFDQÀQGWKHGDWDIURPWKHPDVWHURQWKHVODYHQRZE\GRLQJDSELECT * FROMÀUVWWDEOHrepdb.tblADQGWKHQrepdb.tblB

7KHÀUVWSELECTVKRZVQRUHFRUGVEHFDXVHtblAZDVH[FOXGHGIURPWKHGXPS 7DEOHtblBFRQWDLQVWKHWKUHHUHFRUGVZHLQVHUWHGRQWKHPDVWHU 

&UHDWHDUHSOLFDWLRQXVHUDFFRXQWRQWKHÀOWHULQVWDQFHIRUWKHVODYHVWRXVH black> -> -> ->



GRANT REPLICATION SLAVE ON *.* TO 'repslave'@'localhost' IDENTIFIED BY 'slavepass';

&RQQHFWWKHVODYHWRWKHÀOWHUHQJLQH%HVXUHWRLQVHUWWKHFRUUHFW YDOXHVIRUMASTER_LOG_FILEDQGMASTER_LOG_POSLQWKHVWDWHPHQW7KRVHDUH WKHYDOXHV\RXZURWHGRZQZKHQ\RXLVVXHGWKHSHOWMASTERSTATUSFRPPDQGRQ WKHÀOWHUVHUYHUEHIRUHVWDUWLQJWKHUHSOLFDWLRQWKHUH 

5HSOLFDWLRQ slave> CHANGE MASTER TO -> master_host='localhost', -> master_port=3308, -> master_user='repslave', -> master_password='slavepass', -> master_log_file='black-bin.000003', -> master_log_pos=98; Query OK, 0 rows affected (0.01 sec)



6WDUWWKHVODYHDQGYHULI\WKDWLWVWDUWVXSFRUUHFWO\ slave> START SLAVE slave> SHOW SLAVE STATUS \G ************************** 1. row *************************** Slave_IO_State: Waiting for master to send event ... Relay_Master_Log_File: black-bin.000003 Slave_IO_Running: Yes Slave_SQL_Running: Yes ... Seconds_Behind_Master: 0

 $VVRRQDVWKHSUHYLRXVVWHSLVFRPSOHWHWKHUHSOLFDWLRQVKRXOGDOUHDG\KDYHXSGDWHG tblBRQWKHVODYHDQGLQVHUWHGWKHQHZ-RKQ UHFRUG9HULI\LWOLNHWKLV slave> SELECT * FROM repdb.tblB;

$SSDUHQWO\WKHUHSOLFDWLRQZRUNV "D:\MySQLBackup\MySQLDumpAllDatabases.sql.gz"

+RZLWZRUNV 7KHRQO\GLIIHUHQFHLQWKHDERYHFRPPDQGOLQHLQFRPSDULVRQWRWKHSUHYLRXVUHFLSHVEHVLGH WKHVOLJKWO\FKDQJHGWDUJHWÀOHQDPHWRUHÁHFWWKHFRPSUHVVHGFRQWHQW OLHVZLWKLQ WKH| gzip --fastSRUWLRQ7KLVUHGLUHFWVWKHRXWSXWRIWKHmysqldumpFRPPDQGWRWKH J]LSSURJUDPZKLFKFRPSUHVVHVGDWDRQWKHÁ\7KHFRPSUHVVHGGDWDVWUHDPLVWKHQZULWWHQ WRWKHJLYHQÀOHUHVXOWLQJLQDVLJQLÀFDQWO\UHGXFHGVL]H

7KHUH VPRUH« :KLOHWKHUHFLSHLWVHOILVQRWWRRFRPSOLFDWHGWKHUHDUHVRPHDVSHFWVWRFRQVLGHUZKLFKDUH GLVFXVVHGLQWKHIROORZLQJVHFWLRQV 

%DFNLQJ8SDQG5HVWRULQJ0\64/'DWD

$FKLHYLQJEHWWHUFRPSUHVVLRQUDWLR $VDEDOOSDUNÀJXUHIRUWKHFRPSUHVVLRQUDWLRIRUW\SLFDOGDWDEDVHVWKHFRPSUHVVHGGXPS ZLOOSUREDEO\EHUHGXFHGWRRQHWKLUGRILWVXQFRPSUHVVHGVL]H7RDFKLHYHEHWWHUFRPSUHVVLRQ DWWKHH[SHQVHRIUHGXFHGSHUIRUPDQFHDQGKLJKHU&38ORDG\RXFRXOGDOVROHDYHRXWWKH --fastRSWLRQRIJ]LSUHVXOWLQJLQDFRPSUHVVHGÀOHWKDWZLOOWDNHOHVVWKDQRIWKH RULJLQDOVL]H,IWKHVL]HRIWKHEDFNXSLVDQH[WUHPHO\FULWLFDOLVVXHIRU\RX\RXPLJKWHYHQ WU\WKH--bestRSWLRQDVDUHSODFHPHQWIRU--fast͕EXWEHZDUQHGWKDWWKLVPLJKWOHDGWRD GUDPDWLFLQFUHDVHLQH[HFXWLRQWLPHVZLWKPRVWO\PLQLPDOLPSURYHPHQWVLQVL]HWKLVLVZK\ZH UHFRPPHQGQRWXVLQJWKLVRSWLRQ

&RQVLGHULQJSHUIRUPDQFHIDFWRUV :LWKWKHDGGLWLRQDOFRPSUHVVLRQSDUWDGGHGWRWKHDERYHFRPPDQG\RXZRXOGLQWXLWLYHO\ H[SHFWWKDWWKLVQHFHVVDULO\KDVDQHJDWLYHLPSDFWRQWKHGXPSSHUIRUPDQFH6XUSULVLQJO\ WKLVLVQRWDOZD\VWKHFDVHDVWKHUHGXFHGVL]HRIWKHUHVXOWLQJÀOHDOVRGHFUHDVHVWKHDPRXQW RIGDWDZULWWHQWR\RXUGLVN7KLVUHGXFWLRQLQGLVN,2PLJKWPDNHXSIRUWKHDGGLWLRQDO FRPSUHVVLRQZRUNRUHYHQFDXVHDSHUIRUPDQFHLPSURYHPHQWmysql -u backup_usr -p"B4ckM3Up!" -e"START SLAVE SQL_THREAD"

+RZLWZRUNV 7KHUHFLSHEDVLFDOO\FRQVLVWVRIWKUHHSDUWVE\SHUIRUPLQJWKHRSHUDWLRQVGHVFULEHGLQVWHS UHSOLFDWLRQLVGLVDEOHGLQVWHSVDQGWKHEDFNXSLWVHOILVSHUIRUPHGLQVWHSUHSOLFDWLRQ LVHQDEOHGDJDLQ /HWXVKDYHDORRNDWWKHEDFNXSSDUWÀUVW,QVWHSWKHEDFNXSRIWKHGDWDVWRUHGLQWKH VODYHGDWDEDVHLVGRQHDVVKRZQLQWKHSUHYLRXVUHFLSHV7KHDGGLWLRQRIVWHSLVQHFHVVDU\ WRDOVREDFNXSWKHUHSOLFDWLRQVWDWHRIWKHVODYH,QGHWDLOWKLVFRSLHVWKHUHOD\ORJÀOHV [host]-relay-bin.indexDQG[hostname]-relay-bin.00x WKHrelay-log.info DQGWKHmaster.infoÀOHV7KHVHÀOHVDUHQHFHVVDU\WRUHFRYHUWKHVODYHIURPDFUDVK EHFDXVHZLWKRXWWKHVHÀOHVLWLVYHU\KDUGWRHVWDEOLVKDZRUNLQJUHSOLFDWLRQPHFKDQLVP ZLWKWKHPDVWHU,I\RXULQWHQWLRQLVWRSHUIRUPDEDFNXSRQO\IRUUHVWRULQJWKHPDVWHUWKHVH ÀOHVDUHQRWDEVROXWHO\QHFHVVDU\EXWZHVWURQJO\VXJJHVWLQFOXGLQJWKHVHÀOHVLQWKHEDFNXS DVZHOO 

%DFNLQJ8SDQG5HVWRULQJ0\64/'DWD 7KDW VDERXWLWFRQFHUQLQJWKHEDFNXSLWVHOI1RZZKDWLVWKHPRWLYDWLRQIRUVWHSVDQG" 7KHGHDFWLYDWLRQRIWKHVODYHUHSOLFDWLRQLQVWHSLVQHFHVVDU\WRSUHYHQWLQFRQVLVWHQFLHV GXULQJWKHEDFNXS$VGLVFXVVHGEHIRUHERWKWKHFRS\DSSURDFKDQGWKHmysqldumpEDFNXS KDYHUHVWULFWLRQVFRQFHUQLQJGDWDFRQVLVWHQFLHVIRUQRQWUDQVDFWLRQDOWDEOHVOLNH0\,6$0  ZKHQFRQFXUUHQWXSGDWHVRFFXUZKLOHWKHGDWDLVUHDGsource

D:\MySQLBackup\MySQLDumpAllDatabases.sql

+RZLWZRUNV $JDLQWKLVLVDSUHWW\VWUDLJKWIRUZDUGUHFLSHFRQQHFWWRWKHGDWDEDVHVWHS DQGUHVWRUHWKH GDWDIURPWKHGXPSVWHS  7KHXVHRIWKHmysqlFRPPDQGOLQHFOLHQWLVQRWDEVROXWHO\QHFHVVDU\EXWSOHDVHQRWHWKDW \RXZLOOQRWEHDEOHWRLVVXHWKHsourceFRPPDQGIURPDQ\RWKHU64/FOLHQW7KLVLVEHFDXVH LWLVQRWDUHJXODU64/VWDWHPHQWEXWDNH\ZRUGUHFRJQL]HGE\WKHmysqlFRPPDQGOLQH FOLHQWLWVHOIgrep --invert-match "^INSERT INTO .op_detail. VALUES .*" D:\MySQLBackup\MySQLDump.sql | mysql -u admin4mysql -p"As,ysp4M"

7KLVFRPPDQGXVHVgrepWRÀOWHUDOOOLQHVLQFOXGLQJDQINSERT INTO 'op_detail'DW WKHVWDUWRIWKHOLQHOHDYLQJDOORWKHUOLQHVXQWRXFKHG$OORWKHUOLQHVDUHSLSHGWRWKHmysql FRPPDQGOLQHFOLHQWWKDWLPSRUWVWKHUHPDLQLQJGXPS͘ 3OHDVHQRWHWKDWLIGDWDKDVUHIHUHQFHVWRHQWULHVLQWKHH[FOXGHGWDEOHWKHLPSRUWRIWKH GXPSZLOOQRWIDLO,QWKLVFDVH\RXZRXOGHQGXSZLWKGDWDLQFRQVLVWHQFLHVWKDWPLJKWFDXVH HUURUVPXFKODWHUVRPDNHVXUHWRH[FOXGHRQO\WDEOHVWKDWDUHQRWUHIHUHQFHGE\IRUHLJQNH\V 2WKHUZLVH\RXKDYHWRVHDUFKIRUSRVVLEOHIRUHLJQNH\FRQVWUDLQWYLRODWLRQVPDQXDOO\6RPH LGHDVRQKRZWRDFKLHYHWKLVDUHSUHVHQWHGIRUH[DPSOHDWhttp://dev.mysql.com/doc/ refman/5.1/en/innodb-foreign-key-constraints.html 

6HHDOVR f

'HÀQLQJDVSHFLÀFXVHUIRUDGPLQLVWUDWLYHWDVNV &UHDWLQJD64/GXPSRIDOOGDWDEDVHV

f

&UHDWLQJD64/GXPSRIVSHFLÀFGDWDEDVHV

f



&KDSWHU

3HUIRUPLQJDSRLQWLQWLPHUHFRYHU\XVLQJ WKHELQDU\ORJV 7KHSUHYLRXVUHFLSHVGHDOWZLWKKRZWRUHFRYHUGDWDIURPDEDFNXS$IWHUUHFRYHU\WKHGDWD VKRXOGEHLQWKHVDPHFRQGLWLRQDVLWZDVZKHQWKHEDFNXSZDVFUHDWHG8QIRUWXQDWHO\WKH GDWDWKDWZDVFKDQJHGDIWHUEDFNXSFUHDWLRQLVQRWUHVWRUHG 7RUHVWRUHWKHGDWDWRWKHSRLQWLQWLPHEHIRUHWKHUHFRYHU\\RXFDQXVHWKHbinlogÀOHVWKDW ZHUHFUHDWHGEHWZHHQEDFNXSDQGUHFRYHU\$VPHQWLRQHGLQWKH5RWDWLQJDQGSXUJHELQDU\ ORJVUHFLSHZHVXJJHVWHQDEOLQJELQDU\ORJJLQJHYHQLI\RXGRQRWKDYHDUHSOLFDWLRQVFHQDULR DVWKLVJLYHV\RXH[WHQGHGRSWLRQVLQEDFNXS ,QWKLVUHFLSHZHZLOOGLVFXVVKRZWRXVHWKHELQDU\ORJVWRUHVWRUHGDWDXSWRWKHODWHVW SRVVLEOHSRLQWLQWLPHEHIRUHWKHGDWDZDVORVW

*HWWLQJUHDG\ 7REHDEOHWRUHVWRUHWKHUHPDLQLQJGDWDXVLQJWKHELQDU\ORJJLQJLQIRUPDWLRQZHGHÀQLWHO\ QHHGWKHELQDU\ORJÀOHV,IDOOGDWDIURP\RXUVHUYHULVORVWIRUH[DPSOHLQFDVHRIDÀUH WKH binlogsDUHKRSHIXOO\DYDLODEOHIURPDWDSHRUDQ\RWKHUPHGLDWKDWLVQRWDIIHFWHGE\WKH GDWDORVV,IRQO\\RXUGDWDEDVHGDWDLVFRUUXSW\RXRIWHQKDYHWKHELQDU\ORJVVWLOOSUHVHQWRQ \RXUVHUYHU VGLVN ,IWKHELQDU\ORJVDUHVWLOOSUHVHQW\RXVKRXOGFRS\WKHELQDU\ORJÀOHVWRDGLIIHUHQWSRVLWLRQ EHIRUH\RXVWDUWUHFRYHULQJIURPDGXPS7KLVZD\\RXKDYHDOOELQDU\ORJÀOHVDYDLODEOHLQ H[DFWO\WKHVWDWHIURPWKHSRLQWLQWLPHEHIRUHWKHUHFRYHU\ :HDVVXPHWKHELQDU\ORJVHLWKHUDVDFRS\RIWKHbinlogÀOHVIURP\RXUVHUYHUGLVNRUDV UHFRYHUHGIURPEDFNXSPHGLD WREHVWRUHGXQGHUC:\tmp\binlogs\ 7RUHDGWKHFKDQJHVVWRUHGLQWKHELQDU\ORJVLQWRWKHGDWDEDVH\RXQHHGDQDSSURSULDWH 0\64/XVHUDJDLQKHUHadmin4mysqlZLWKSDVVZRUGAs,ysp4M  $QGPRVWLPSRUWDQWO\\RXQHHGWKHSRVLWLRQDWZKLFKUHFRYHU\IURPWKHELQDU\ORJÀOHV VKRXOGVWDUW7KHGXPS\RXUGDWDEDVHZDVUHVWRUHGIURPVKRXOGKDYHEHHQFUHDWHGXVLQJ WKH--master-dataRSWLRQVHHWKH7KHUH VPRUH«VHFWLRQRIWKH&UHDWLQJDQ64/GXPSRI DOOGDWDEDVHVUHFLSH RWKHUZLVHWKLVLQIRUPDWLRQLVQRWHDVLO\DYDLODEOH EXECUTE statement;

7KHÀUVWSETVWDWHPHQWFRQVWUXFWVDVWULQJZKLFKFRQWDLQVDSELECTVWDWHPHQW:KLOHLWLV QRWDOORZHGWRXVHYDULDEOHVIRUVWDWHPHQWVGLUHFWO\\RXFDQFRQVWUXFWDVWULQJWKDWFRQWDLQV DVWDWHPHQWDQGXVHYDULDEOHVIRUWKLV:LWKWKHQH[WWZROLQHV\RXSUHSDUHDVWDWHPHQWIURP WKHVWULQJDQGH[HFXWHLW 

&KDSWHU

+DQGOLQJ18//YDOXHV :LWKRXWIXUWKHUKDQGOLQJNULLYDOXHVLQWKHGDWD\RXH[SRUWXVLQJWKHSUHYLRXVVWDWHPHQW ZRXOGVKRZXSDV"NLQWKHUHVXOWLQJÀOH7KLVFRPELQDWLRQLVQRWUHFRJQL]HGIRUH[DPSOHE\ 0LFURVRIW([FHOZKLFKEUHDNVWKHÀOHIRUW\SLFDOXVDJH 7RSUHYHQWWKLV\RXQHHGWRUHSODFH NULLHQWULHVE\DSSURSULDWHYDOXHV$VVXPLQJWKDWWKHWDEOHsample.table1FRQVLVWVRID QXPHULFFROXPQaDQGDFKDUDFWHUFROXPQb\RXVKRXOGXVHWKHIROORZLQJVWDWHPHQW mysql> SELECT IFNULL(a, 0), IFNULL(b, "NULL") FROM sample.table1 INTO OUTFILE 'C:/target.csv' FIELDS ENCLOSED BY '"' TERMINATED BY ';' ESCAPED BY '"' LINES TERMINATED BY '\r\n';

7KHGRZQVLGHWRWKLVDSSURDFKLVWKDW\RXKDYHWROLVWDOOÀHOGVLQZKLFKDNULLYDOXH PLJKWRFFXU

+DQGOLQJOLQHEUHDNV ,I\RXWU\WRH[SRUWYDOXHVWKDWFRQWDLQWKHVDPHFKDUDFWHUFRPELQDWLRQXVHGIRUOLQH WHUPLQDWLRQLQWKHSELECT … INTO OUTFILEVWDWHPHQW0\64/ZLOOWU\WRHVFDSHWKH FKDUDFWHUFRPELQDWLRQZLWKWKHFKDUDFWHUVGHÀQHGE\WKHESCAPED BYFODXVH+RZHYHU WKLVZLOOQRWDOZD\VZRUNWKHZD\LWLVLQWHQGHG

UNION SELECT @timestamp

">

UNION SELECT @rows

">

UNION SELECT @header

"> ">

UNION SELECT CONCAT('#', @counter := @counter + 1, 

0DQDJLQJ'DWD ">

': ',

">

CONCAT_WS(' | ', c1, c2, c3),

">

' $')

">

FROM sample.table2

">

INTO OUTFILE '", file, "';");

->

SET @counter := 0;

->

PREPARE statement FROM @command;

->

EXECUTE statement;

->

END //

Query OK, 0 rows affected (0.00 sec) mysql> delimiter ;

7KLVURXWLQHDOORZVSDVVLQJWKHÀOHQDPHZKHQFDOOLQJLW mysql> call sample.export_table2_FileAsParam("C:/data.out"); Query OK, 0 rows affected (0.00 sec)

,PSRUWLQJGDWDIURPDVLPSOH&69ÀOH $FRPPRQWDVNZKHQZRUNLQJZLWKGDWDEDVHVLVWRLPSRUWGDWDIURPGLIIHUHQWVRXUFHV 8QIRUWXQDWHO\WKLVGDWDZLOOW\SLFDOO\QRWEHSURYLGHGDVDFRQYHQLHQWVHWRIZHOOIRUPHG64/ VWDWHPHQWVWKDW\RXFDQVLPSO\UXQDJDLQVW\RXUGDWDEDVH7KHUHIRUHKHUH\RXZLOOKDYHWR GHDOZLWKGDWDLQDGLIIHUHQWIRUPDW $VDFRPPRQGHQRPLQDWRUFKDUDFWHUVHSDUDWHGYDOXHV&69 DUHVWLOODSUHYDOHQWZD\RI H[FKDQJLQJGDWD,QWKLVFKDSWHUZHZLOOVKRZ\RXKRZWRLPSRUWGDWDVWRUHGLQ&69ÀOHV$VD W\SLFDOH[DPSOHZHZLOOXVHWKHÀOHIRUPDW0LFURVRIW([FHOSURGXFHVZKHQVWRULQJÀOHVXVLQJ WKH*.CSVÀOHW\SH 7KLVUHFLSHLVWKHFRXQWHUSDUWRIWKH([SRUWLQJGDWDWRDVLPSOH&69ÀOHUHFLSHLQWKLVFKDSWHU

*HWWLQJUHDG\ 7RVWHSWKURXJKWKLVUHFLSHZHZLOOGHÀQLWHO\QHHGDÀOHWRLPSRUWKHUHC:/source.csv  DQGDWDEOHWRLPSRUWWKHGDWDLQWRKHUHsample.table1 7KHVRXUFHÀOHDQGWDUJHWWDEOH KDYHWRKDYHDPDWFKLQJIRUPDWFRQFHUQLQJWKHQXPEHURIFROXPQVDQGWKHW\SHRIGDWD VWRUHGLQWKHP)XUWKHUPRUHDQDFFRXQWZLWKINSERTDQGFILESULYLOHJHVLVUHTXLUHGZHZLOO DVVXPHDQDFFRXQWsample_installLQWKLVUHFLSH



&KDSWHU

7KHVRXUFHÀOHKDVWREHORFDWHGRQWKHPDFKLQHWKDWUXQV\RXU0\64/ VHUYHUQRWRQWKHFOLHQWVLGH

+RZWRGRLW  &RQQHFWWRWKHGDWDEDVHXVLQJWKHsample_installDFFRXQW  ,VVXHWKHIROORZLQJ64/FRPPDQG mysql> LOAD DATA INFILE 'C:/source.csv' INTO TABLE sample.table1 FIELDS ENCLOSED BY '"' TERMINATED BY ';' ESCAPED BY '"' LINES TERMINATED BY '\r\n'; Query OK, 20 rows affected (0.06 sec) Records: 20

Deleted: 0

Skipped: 0

Warnings: 0

+RZLWZRUNV 7KHLOAD DATA INFILEFRPPDQGZRUNVDQDORJRXVWRWKHSELECT … INTO OUTFILE FRPPDQGGLVFXVVHGLQWKHSUHYLRXVUHFLSHVEXWDVDPHDQVIRULPSRUWLQJGDWDUDWKHUWKDQ H[SRUWLQJ7KHIRUPDWRSWLRQVDYDLODEOHIRUERWKFRPPDQGVDUHLGHQWLFDOVR\RXFDQW\SLFDOO\ LPSRUWGDWDH[SRUWHGE\DSELECT … INTO OUTFILEVWDWHPHQWXVLQJDLOAD DATA INFILEFRPPDQGZLWKWKHVDPHIRUPDWRSWLRQV $VPRVWÀOHVFRQVLVWRIOLQHVWHUPLQDWHGE\DVHTXHQFHRIDFDUULDJHUHWXUQDQGDOLQHIHHG FKDUDFWHUZHXVHWKHLINES TERMINATED BY '\r\n'RSWLRQ7KHFKRLFHRIWKHVHPLFRORQ FKDUDFWHU³DVDVHSDUDWRUIRUGLIIHUHQWÀHOGVRIHYHU\OLQHTERMINATED BY ';' ³LVPDLQO\ GXHWRWKHIDFWWKDW([FHOXVHVWKLVIRUPDW,I\RXKDSSHQWRUHFHLYH&69ÀOHVWKDWIRUH[DPSOH XVHDFRPPDLQVWHDG\RXKDYHWRDGMXVWWKLVDFFRUGLQJO\ 7KHWHUPFIELDS ENCLOSED BY '"'WHOOVWKHLPSRUWWRORRNIRUGRXEOHTXRWHVDWWKH VWDUWRIHYHU\ÀHOGLPSRUWHG,IWKHUHLVRQHWKHÀHOGLVFRQVLGHUHGWRHQGDWWKHQH[WGRXEOH TXRWH7REHDEOHWRKDYHGRXEOHTXRWHVLQVLGHDÀHOGYDOXHZHGHÀQHDQHVFDSHFKDUDFWHU ESCAPED BY '"' :LWKWKLVFRQVWHOODWLRQDVHTXHQFHRIWZRGRXEOHTXRWHVLVQRWWUHDWHG DVWKHHQGRIWKHÀHOGEXWDVDGRXEOHTXRWHFKDUDFWHUDVSDUWRIWKHYDOXH

7KHUH VPRUH 7KHGDWDLVUHDGIURPWKHÀOHXVLQJWKHGHIDXOWFKDUDFWHUVHWRIWKHGDWDEDVH,IWKHÀOHXVHVD GLIIHUHQWFKDUDFWHUHQFRGLQJ\RXFDQVSHFLI\WKLVE\DGGLQJDCHARACTER SETFODXVHDIWHU WKHWDEOHGHÀQLWLRQLOAD DATA INFILE … INTO TABLE sample.table1 CHARACTER SET utf8; 3OHDVHQRWHWKDWWKHFKDUDFWHUVHWVucs2utf16DQGutf32DUHQRWVXSSRUWHG DVRI0\64/YHUVLRQ 



0DQDJLQJ'DWD

6HHDOVR f

([SRUWLQJGDWDWRDVLPSOH&69ÀOH

,PSRUWLQJGDWDIURPFXVWRPÀOHIRUPDWV ,QWKHSUHYLRXVUHFLSH,PSRUWLQJGDWDIURPDVLPSOH&69ÀOHZHGLVFXVVHGDZD\RILPSRUWLQJ GDWDIURPDQLFHO\IRUPDWWHGÀOH8QIRUWXQDWHO\\RXVRPHWLPHVKDYHWRGHDOZLWKIDUOHVV FRQYHQLHQWGDWDVRXUFHV,QWKLVUHFLSHZHZLOOSUHVHQWVRPHPRUHDGYDQFHGWRSLFVRI LPSRUWLQJGDWDIURPÀOHVZLWKDOHVVVWULFWVWUXFWXUH 2EYLRXVO\LWLVQRWSRVVLEOHWRSUHVHQWDXQLYHUVDOUHFLSHIRUHYHU\ÀOHIRUPDWLPDJLQDEOHVR ZHZLOOXVHDQH[DPSOHWKDWFRYHUVVRPHRIWKHFRPPRQSUREOHPVRQHKDVWRWDFNOHZKHQ LPSRUWLQJGDWDIURPFXVWRPÀOHV)RUWKLVZHZLOOUHIHUWRWKHVDPHK\SRWKHWLFDOIRUPDWDVLQ ([SRUWGDWDWRDFXVWRPÀOHIRUPDWZKLFKGHÀQHVIRXULQLWLDOOLQHVFRQWDLQLQJQDPHRIWKH ÀOHDWLPHVWDPSDGHVFULSWLRQDQGWKHQXPEHURIURZV DKHDGHUOLQHZLWKWKHQDPHRIWKH FROXPQVDQGVXEVHTXHQWO\WKHURZVZLWKWKHDFWXDOGDWDWRLPSRUW(DFKGDWDURZVWDUWVZLWK DKDVKFKDUDFWHU# WKHOLQHQXPEHUDFRORQDQGDVSDFH7KHGDWDYDOXHVWKDWIROORZWKH URZQXPEHUDUHVHSDUDWHGE\DSLSH| FKDUDFWHUDQGWKHURZFORVHVZLWKDGROODUVLJQ$ 

*HWWLQJUHDG\ $JDLQWKHDFFRXQWXVHGLQWKHUHFLSHQHHGVWKHFILESULYLOHJHEHVLGHVWKHINSERT SHUPLVVLRQIRUWKHWDEOHWKHGDWDVKRXOGEHLPSRUWHGLQWR :LWKD64/FOLHQWDÀOHZLWK WKHDSSURSULDWHIRUPDWDQGDWDEOHDVWKHLPSRUWWDUJHWZHDUHUHDG\WRJR$VLQSUHYLRXV UHFLSHVZHXVHsample_installDVWKHDFFRXQWQDPHC:/source.txtDVWKHVRXUFHÀOH DQGsample.table2FRQVLVWLQJRIWKUHHFROXPQVc1c2DQGc3 DVWKHWDUJHWWDEOH:H DVVXPHWKHVRXUFHÀOHWRKDYHWKHIROORZLQJFRQWHQW Filename: C:/source.txt Description: This is a file for test import to sample.table2, columns c1, c2, and c3 2009-06-14 13:25:05 Row count: 3 #Row Nr: Column c1 | Column c2 | Column c3 $ #1: 209 | Some text in my test data | Some more text $ #2: 308 | Next test text for testing | Text to test $ #3: 406 | "A water | pipe" | Really? $

+RZWRGRLW  &RQQHFW\RXUIDYRULWHFOLHQWIRUH[DPSOHWKHmysqlFRPPDQGOLQHFOLHQW WR\RXU 0\64/VHUYHUXVLQJWKHsample_installDFFRXQW 

&KDSWHU

 ([HFXWHWKHIROORZLQJ64/FRPPDQG mysql> LOAD DATA INFILE "C:/source.txt" ->

INTO TABLE sample.table2

->

FIELDS TERMINATED BY ' | '

->

OPTIONALLY ENCLOSED BY '"'

->

LINES STARTING BY ':'

->

TERMINATED BY '\r\n'

->

IGNORE 5 LINES

->

SET c3=TRIM(TRAILING ' $' FROM c3);

Query OK, 3 rows affected (0.05 sec) Records: 3

Deleted: 0

Skipped: 0

Warnings: 0

+RZLWZRUNV /HWXVGLVVHFWWKHDERYHVWDWHPHQWE\KDYLQJDORRNDWWKHVRXUFHÀOHÀUVWRIDOOZHZDQW WRLPSRUWGDWDIURPWKHÀOHC:\source.txtLQWRWKHWDEOHsample.table2ZKLFKLV UHSUHVHQWHGE\WKHÀUVWWZROLQHVLOAD DATA INFILE … INTO TABLE …  $WWKHWRSRIWKHÀOHZHKDYHÀYHOLQHVWKHLQLWLDOIRXUOLQHVZLWKLQIRUPDWLRQDERXWWKHÀOHSOXV WKHKHDGHU WKDWVKRXOGQRWEHLPSRUWHGLQWRWKHWDUJHWWDEOH7RDFKLHYHWKLVWKHIGNORE 5 linesRSWLRQLVDGGHG 7KHUHPDLQLQJOLQHVDUHSUHÀ[HGZLWKDKDVKFKDUDFWHUWKHURZQXPEHUDQGDFRORQ7KLVSDUW RIHYHU\OLQHKDVWREHLJQRUHGZKLFKLVZKDWWKHLINES STARTING BY ':'RSWLRQGRHVLW WHOOV0\64/WRLJQRUHWKHÀUVWFRORQRIWKHOLQHDQGDQ\FKDUDFWHUEHIRUHLW%\GRLQJWKLVWKH URZQXPEHUSUHÀ[LVVNLSSHG $IWHUWKHSUHÀ[WKHOLQHVFRQWDLQWKHDFWXDOYDOXHVVHSDUDWHGE\SLSHFKDUDFWHUV7KHFIELDS TERMINATED BY ' | 'RSWLRQWHOOV0\64/KRZWRLGHQWLI\DÀHOGVHSDUDWRU:LWKWKH DGGLWLRQDOVHWWLQJOPTIONALLY ENCLOSED BY '"'WKHYDOXHLWVHOIPLJKWFRQWDLQWKLVÀHOG VHSDUDWRUVHTXHQFH³LIWKHZKROHYDOXHLVHQFORVHGE\GRXEOHTXRWHVWKLVLVWKHFDVHLQWKH ODVWURZRIWKHVDPSOHÀOH  $WWKLVSRLQWWKHUHLVRQO\RQHSUREOHPOHIWWKHOLQHVHQGZLWKDGROODUVLJQZKLFKLVQRW SDUWRIWKHODVWYDOXH$QLQWXLWLYHDSSURDFKZRXOGEHWRLQFOXGHWKLVFKDUDFWHULQWKHOLQH WHUPLQDWLRQVHTXHQFHZKLFKPHDQVWRXVH$\r\nDVDOLQHHQGLQJLQVWHDGRI\r\n  8QIRUWXQDWHO\WKLVGHÀQLWLRQRIDOLQHHQGGRHVQRWZRUNDVH[SHFWHGIRURXUH[DPSOHDVLW ZRXOGEUHDNWKHLQWHUSUHWDWLRQRIWKHÀUVWÀYHOLQHVZKLFKDUHQRWWHUPLQDWHGWKHVDPHZD\ $VDUHVXOWWKHÀUVWVL[OLQHVZRXOGEHFRQVLGHUHGDVRQHVLQJOHOLQHE\WKHLPSRUWEHFDXVH RQO\WKHVL[WKOLQHDFWXDOO\HQGVZLWKDFKDUDFWHUVHTXHQFHRI$\r\n7REHDEOHWRH[SOLFLWO\ H[FOXGHWKHKHDGHUOLQHVIURPWKHLPSRUWZHKDYHWRUHO\RQWKHWUDGLWLRQDOOLQHHQGLQJ GHÀQHGE\WKH[LINES] TERMINATED BY '\r\n'RSWLRQ 

0DQDJLQJ'DWD +HQFHWKHRSWLRQVIRUGHÀQLQJWKHÀHOGVHSDUDWRUVDQGWKHEHJLQQLQJDQGWHUPLQDWLRQRI DOLQHGRQRWDOORZXVWRJHWULGRIWKHFORVLQJGROODUVLJQ7KXVLWLVFRQVLGHUHGSDUWRIWKH ODVWYDOXHZKLFKLVDVVLJQHGWRFROXPQc37RÀQDOO\JHWULGRIWKLVSRVWÀ[WKHSETFODXVH RIWKHLOAD DATA INFILE FRPPDQGFRPHVLQKDQG\ZKLFKDOORZVWRFOHDUO\GHÀQH WKHYDOXHVWKDWDUHDVVLJQHGWRWKHFROXPQVLQWKHWDUJHWWDEOH7KHFORVLQJRSWLRQSET c3=TRIM(TRAILING ' $' FROM c3);GHÀQHVDZD\WRVWULSWKHXQZDQWHGSRVWÀ[IURP WKHODVWÀHOG ,IZHSXWLWDOOWRJHWKHUWKHLPSRUWZRUNVDVLQWHQGHG

7KHUH VPRUH $VZLWKH[SRUWLQJGDWDLWLVUHFRPPHQGHGWRFRQVLGHUXVLQJDQH[WHUQDOSURJUDPPLQJ ODQJXDJHWRLPSRUWPRUHFRPSOH[GDWDVWUXFWXUHVLQWR0\64/:KLOHLWLVSRVVLEOHWRLPSRUW UDWKHUVRSKLVWLFDWHGÀOHIRUPDWVXVLQJ0\64/FRPPDQGVDVZHOOLWLVRIWHQIDUPRUHHIÀFLHQW WRKDYHDIXOOEORZQSURJUDPPLQJODQJXDJHDWKDQGWRVROYHWKHWDVNRISDUVLQJLQSXWÀOHV 7KLVLVPRVWQRWDEO\WKHFDVHZKHQLWFRPHVWR;0/ÀOHV )RULPSRUWLQJGDWDIURP;0/ÀOHVFRQVLGHUXVLQJDQ;6/7SURFHVVRU WRSURGXFHFRUUHVSRQGLQJ64/FRPPDQGV

6HHDOVR f

,PSRUWLQJGDWDIURPDVLPSOH&69ÀOH

f

([SRUWLQJGDWDWRDFXVWRPÀOHIRUPDW



&KDSWHU

,QVHUWLQJQHZGDWDDQGXSGDWLQJGDWDLILW DOUHDG\H[LVWV 0DQLSXODWLQJGDWDLQDGDWDEDVHLVSDUWRIHYHU\GD\ZRUNDQGWKHEDVLF64/PHDQVRI INSERTUPDATEDQGDELETEPDNHWKLVDSUHWW\VWUDLJKWIRUZDUGDOPRVWWULYLDOWDVN³EXW LVWKLVDOZD\VWUXH" :KHQFRQVLGHULQJGDWDPDQLSXODWLRQPRVWRIWKHWLPHZHWKLQNRIDVLWXDWLRQZKHUHZH NQRZWKHFRQWHQWRIWKHGDWDEDVH:LWKWKLVLQIRUPDWLRQLWLVXVXDOO\SUHWW\HDV\WRÀQG DZD\RIFKDQJLQJWKHGDWDWKHZD\\RXLQWHQGWR%XWZKDWLI\RXKDYHWRFKDQJHGDWDLQ FLUFXPVWDQFHVZKHUH\RXGRQRWNQRZWKHDFWXDOGDWDEDVHFRQWHQWEHIRUHKDQG" INSERT INTO sample.config VALUES ("maxPriceDiscount", "25%") ON DUPLICATE KEY UPDATE value='25%'; Query OK, 1 row affected (0.05 sec)

+RZLWZRUNV 7KLVFRPPDQGLVHDVLO\H[SODLQHGEHFDXVHLWVLPSO\GRHVZKDWLWVD\VLWLQVHUWVDQHZURZLQ WKHWDEOHXVLQJWKHJLYHQYDOXHVDVORQJDVWKLVGRHVQRWFDXVHDGXSOLFDWHHQWU\LQHLWKHUWKH SULPDU\NH\RUDQRWKHUXQLTXHLQGH[,IDGXSOLFDWHUHFRUGH[LVWVWKHH[LVWLQJURZLVXSGDWHG DFFRUGLQJWRWKHFODXVHVGHÀQHGDIWHUON DUPLICATE KEY UPDATE :KLOHLWLVVRPHWLPHVWHGLRXVWRHQWHUVRPHRIWKHGDWDDQGFROXPQVWZRWLPHVRQFHIRUWKH INSERTDQGDVHFRQGWLPHIRUWKHUPDATE WKLVVWDWHPHQWDOORZVIRUDORWRIÁH[LELOLW\ZKHQLW FRPHVWRWKHPDQLSXODWLRQRISRWHQWLDOO\H[LVWLQJGDWD 3OHDVHQRWHWKDWZKHQH[HFXWLQJWKHDERYHVWDWHPHQWWKHUHVXOWGLIIHUVVOLJKWO\ZLWKUHVSHFW WRWKHQXPEHURIDIIHFWHGURZVGHSHQGLQJRQWKHDFWXDOGDWDSUHVHQWLQWKHGDWDEDVH:KHQ WKHUHFRUGGRHVQRWH[LVW\HWLWLVLQVHUWHGZKLFKUHVXOWVLQRQHDIIHFWHGURZ%XWLIWKHUHFRUG LVXSGDWHGUDWKHUWKDQLQVHUWHGLWUHSRUWVWZRDIIHFWHGURZVLQVWHDGHYHQLIRQO\RQHURZ JHWVXSGDWHG

7KHUH VPRUH 7KHINSERT INTO … ON DUPLICATE UPDATEFRQVWUXFWGRHVQRWZRUNZKHQWKHUHLV QRUNIQUERUPRIMARY KEYGHÀQHGRQWKHWDUJHWWDEOH,I\RXKDYHWRSURYLGHWKHVDPH VHPDQWLFVZLWKRXWKDYLQJDSSURSULDWHNH\GHÀQLWLRQVLQSODFHLWLVUHFRPPHQGHGWRXVHWKH WHFKQLTXHVGLVFXVVHGLQWKHQH[WUHFLSH

6HHDOVR f



,QVHUWLQJGDWDEDVHGRQH[LVWLQJGDWDEDVHFRQWHQW

&KDSWHU

,QVHUWLQJGDWDEDVHGRQH[LVWLQJGDWDEDVH FRQWHQW ,QWKHSUHYLRXVUHFLSH,QVHUWLQJQHZGDWDDQGXSGDWLQJGDWDLILWDOUHDG\H[LVWVZHGLVFXVVHG DPHWKRGWRHLWKHULQVHUWRUXSGDWHUHFRUGVGHSHQGLQJRQZKHWKHUWKHUHFRUGVDOUHDG\H[LVWLQ WKHGDWDEDVH$VLPLODUSUREOHPDULVHVZKHQ\RXQHHGWRLQVHUWGDWDWR\RXUGDWDEDVHEXWWKH GDWDWRLQVHUWGHSHQGVRQWKHGDWDLQ\RXUGDWDEDVH $VDQH[DPSOHFRQVLGHUDVLWXDWLRQLQZKLFK\RXQHHGWRLQVHUWDUHFRUGZLWKDFHUWDLQ PHVVDJHLQWRDWDEOHlogMsgsEXWWKHPHVVDJHLWVHOIVKRXOGEHGLIIHUHQWGHSHQGLQJRQWKH FXUUHQWV\VWHPODQJXDJHWKDWLVVWRUHGLQDFRQÀJXUDWLRQWDEOHconfig  ,WLVIDLUO\HDV\WRDFKLHYHDVLPLODUEHKDYLRUIRUDQUPDATEVWDWHPHQWEHFDXVHWKLVVXSSRUWVD WHEREFODXVHWKDWFDQEHXVHGWRRQO\SHUIRUPDQXSGDWHLIDFHUWDLQSUHFRQGLWLRQLVPHW UPDATE logMsgs SET message= CONCAT('Last update: ', NOW()) WHERE EXISTS (SELECT value FROM config WHERE name='lang' AND value = 'en'); UPDATE logMsgs SET message= CONCAT('Letztes Update: ', NOW()) WHERE EXISTS (SELECT value FROM config WHERE name='lang' AND value = 'de'); UPDATE logMsgs SET message= CONCAT('Actualisation derniere: ', NOW()) WHERE EXISTS (SELECT value FROM config WHERE name='lang' AND value = 'fr');

8QIRUWXQDWHO\WKLVDSSURDFKLVQRWDSSOLFDEOHWRINSERTFRPPDQGVDVWKHVHGRQRWVXSSRUW DWHEREFODXVH'HVSLWHWKLVPLVVLQJRSWLRQWKHIROORZLQJUHFLSHGHVFULEHVDPHWKRGWRPDNH INSERTVWDWHPHQWVH[HFXWHRQO\LIDQDSSURSULDWHSUHFRQGLWLRQLQWKHGDWDEDVHLVPHW

*HWWLQJUHDG\ $VEHIRUHZHDVVXPHDGDWDEDVHD64/FOLHQWmysql DQGD0\64/XVHUZLWKVXIÀFLHQW SULYLOHJHVINSERTDQGSELECTLQWKLVFDVH $GGLWLRQDOO\ZHQHHGDWDEOHWRLQVHUWGDWD LQWRKHUHlogMsgs DQGDFRQÀJXUDWLRQWDEOHconfigSOHDVHUHIHUWRWKHSUHYLRXVUHFLSH IRUGHWDLOV 



0DQDJLQJ'DWD

+RZWRGRLW  &RQQHFWWR\RXUGDWDEDVHXVLQJ\RXU64/FOLHQW  ([HFXWHWKHIROORZLQJ64/FRPPDQGV mysql> INSERT INTO sample.logMsgs(message) ->

SELECT CONCAT('Last update: ', NOW())

->

FROM sample.config WHERE name='lang' AND value='en';

Query OK, 0 rows affected (0.00 sec) Records: 0

Duplicates: 0

Warnings: 0

mysql> INSERT INTO sample.logMsgs(message) ->

SELECT CONCAT('Letztes Update: ', NOW())

->

FROM sample.config WHERE name='lang' AND value='de';

Query OK, 1 row affected (0.05 sec) Records: 1

Duplicates: 0

Warnings: 0

mysql> INSERT INTO sample.logMsgs(message) ->

SELECT CONCAT('Dernière actualisation: ', NOW())

->

FROM sample.config WHERE name='lang' AND value='fr';

Query OK, 0 rows affected (0.00 sec) Records: 0

Duplicates: 0

Warnings: 0

+RZLWZRUNV 2XUJRDOLVWRKDYHDQINSERTVWDWHPHQWWDNHLQWRDFFRXQWWKHSUHVHQWODQJXDJHVWRUHGLQ WKHGDWDEDVH7KHWULFNWRGRVRLVWRXVHDSELECTVWDWHPHQWDVLQSXWIRUWKHINSERT7KH SELECTFRPPDQGSURYLGHVDWHEREFODXVHVR\RXFDQXVHDFRQGLWLRQWKDWRQO\PDWFKHV IRUWKHUHVSHFWLYHODQJXDJH2QHUHVWULFWLRQRIWKLVVROXWLRQLVWKDW\RXFDQRQO\LQVHUWRQH UHFRUGDWDWLPHVRWKHVL]HRIVFULSWVPLJKWJURZFRQVLGHUDEO\LI\RXKDYHWRLQVHUWORWVRI GDWDDQGRUKDYHWRFRYHUPDQ\DOWHUQDWLYHV

7KHUH VPRUH ,I\RXKDYHPRUHWKDQMXVWDIHZYDOXHVWRLQVHUWLWLVPRUHFRQYHQLHQWWRKDYHWKHGDWDLQRQH SODFHUDWKHUWKDQGLVWULEXWHGRYHUVHYHUDOLQGLYLGXDOINSERTVWDWHPHQWV,QWKLVFDVHLWPLJKW PDNHVHQVHWRFRQVROLGDWHWKHGDWDE\SXWWLQJLWLQVLGHDWHPSRUDU\WDEOHWKHÀQDOINSERT VWDWHPHQWXVHVWKLVWHPSRUDU\WDEOHWRVHOHFWWKHDSSURSULDWHGDWDURZVIRULQVHUWLRQLQWRWKH WDUJHWWDEOH7KHGRZQVLGHRIWKLVDSSURDFKLVWKDWWKHXVHUQHHGVWKHCREATE TEMPORARY TABLESSULYLOHJHEXWLWW\SLFDOO\FRPSHQVDWHVZLWKPXFKFOHDQHUVFULSWV 

&KDSWHU

$IWHUFUHDWLQJWKHWHPSRUDU\WDEOHZLWKWKHÀUVWVWDWHPHQWZHLQVHUWGDWDLQWRWKHWDEOHZLWK WKHIROORZLQJINSERTVWDWHPHQW7KHQH[WVWDWHPHQWLQVHUWVWKHDSSURSULDWHGDWDLQWRWKH WDUJHWWDEOHsample.logMsgsE\VHOHFWLQJWKHDSSURSULDWHGDWDIURPWKHWHPSRUDU\GDWD WKDWPDWFKHVWKHODQJXDJHHQWU\IURPWKHconfigWDEOH7KHWHPSRUDU\WDEOHLVWKHQUHPRYHG DJDLQ7KHÀQDOSELECTVWDWHPHQWLVVROHO\IRUFKHFNLQJWKHUHVXOWVRIWKHRSHUDWLRQ

6HHDOVR f

,QVHUWLQJQHZGDWDDQGXSGDWLQJGDWDLILWDOUHDG\H[LVWV



0DQDJLQJ'DWD

'HOHWLQJDOOGDWDIURPODUJHWDEOHV $OPRVWHYHU\RQHZKRZRUNVZLWKGDWDEDVHVH[SHULHQFHVWKHFRQVWDQWJURZWKRIWKHGDWD VWRUHGLQWKHLUGDWDEDVHDQGLWLVW\SLFDOO\ZHOOEH\RQGWKHLQLWLDOHVWLPDWHV%HFDXVHRIWKDW \RXRIWHQHQGXSZLWKUDWKHUODUJHGDWDVHWV$QRWKHUFRPPRQREVHUYDWLRQLVWKDWLQPRVW GDWDEDVHVWKHUHDUHVRPHWDEOHVWKDWKDYHDVSHFLDOWHQGHQF\WRJURZHVSHFLDOO\ELJ ,IDWDEOH VVL]HUHDFKHVDYLUWXDOWKUHVKROGZKLFKLVKDUGWRGHÀQHDVLWGHSHQGVKHDYLO\ RQWKHDFFHVVSDWWHUQVDQGWKHGDWDVWUXFWXUHV LWJHWVKDUGHUDQGKDUGHUWRPDLQWDLQDQG SHUIRUPDQFHGHJUDGDWLRQPLJKWRFFXU)URPDFHUWDLQSRLQWRQLWLVHYHQGLIÀFXOWWRJHWULG RIGDWDLQWKHWDEOHDJDLQDVWKHVKHHUQXPEHURIUHFRUGVPDNHVGHOHWLRQDSUHWW\H[SHQVLYH WDVN7KLVSDUWLFXODUO\KROGVWUXHIRUVWRUDJHHQJLQHVZLWK0XOWL9HUVLRQ&RQFXUUHQF\&RQWURO 09&& LI\RXRUGHUWKHGDWDEDVHWRGHOHWHGDWDIURPWKHWDEOHLWPXVWQRWEHGHOHWHGULJKW DZD\EHFDXVH\RXPLJKWVWLOOUROOEDFNWKHGHOHWLRQ6RHYHQZKLOHWKHGHOHWLRQZDVLQLWLDWHG DFRQFXUUHQWTXHU\RQWKHWDEOHVWLOOKDVWREHDEOHWRVHHDOOWKHUHFRUGVGHSHQGLQJRQWKH WUDQVDFWLRQLVRODWLRQOHYHO 7RDFKLHYHWKLVWKHVWRUDJHHQJLQHZLOORQO\PDUNWKHUHFRUGVDV GHOHWHGEXWWKHDFWXDOGHOHWLRQWDNHVSODFHDIWHUWKHRSHUDWLRQLVFRPPLWWHGDQGDOORWKHU WUDQVDFWLRQVWKDWDFFHVVWKLVWDEOHDUHFORVHGDVZHOO ,I\RXKDYHWRGHDOZLWKODUJHGDWDVHWVWKHPRVWGLIÀFXOWWDVNLVWRRSHUDWHRQWKHSURGXFWLRQ V\VWHPZKLOHRWKHUSURFHVVHVFRQFXUUHQWO\ZRUNRQWKHGDWD,QWKHVHFLUFXPVWDQFHV\RXKDYH WRNHHSWKHGXUDWLRQRI\RXUPDLQWHQDQFHRSHUDWLRQVDVORZDVSRVVLEOHLQRUGHUWRPLQLPL]H WKHLPSDFWRQWKHUXQQLQJV\VWHP$VWKHGHOHWLRQRIGDWDIURPDODUJHWDEOHW\SLFDOO\VWDUWLQJ DWVHYHUDOPLOOLRQVRIURZV PLJKWWDNHTXLWHVRPHWLPHWKHIROORZLQJUHFLSHVKRZVDZD\RI PLQLPL]LQJWKHGXUDWLRQRIWKLVRSHUDWLRQLQRUGHUWRUHGXFHVLGHHIIHFWVOLNHORFNLQJHIIHFWV RUSHUIRUPDQFHGHJUDGDWLRQ 

*HWWLQJUHDG\ %HVLGHVDXVHUDFFRXQWZLWKDSSURSULDWHSULYLOHJHVDELETE \RXQHHGDVXIÀFLHQWO\ODUJH WDEOHWRGHOHWHGDWDIURP

)RUWKLVUHFLSHZHZLOOXVHWKHemployeesGDWDEDVHZKLFKLVDQH[DPSOH GDWDEDVHDYDLODEOHIURP0\64/http://dev.mysql.com/doc/ employee/en/employee.html. 7KLVGDWDEDVHSURYLGHVVRPHWDEOHVZLWKVHQVLEOHGDWDDQGVRPHSUHWW\ ODUJHWDEOHVWKHODUJHVWKDYLQJPRUHWKDQPLOOLRQUHFRUGV

:HDVVXPHWKDWWKHEmployeesGDWDEDVHZDVLQVWDOOHGZLWKDQ,QQR'%VWRUDJHHQJLQH HQDEOHG7RGHOHWHDOOURZVRIWKHODUJHVWWDEOHemployees.salariesLQDTXLFNZD\SOHDVH UHDGRQ 

&KDSWHU

+RZWRGRLW  &RQQHFWWR\RXUGDWDEDVH  (QWHUWKHIROORZLQJ64/FRPPDQG mysql> TRUNCATE TABLE employees.salaries; Query OK, 0 rows affected (0.16 sec)

+RZLWZRUNV 7KHTRUNCATE TABLEFRPPDQGLVDUDWKHUIDVWZD\RIGHOHWLQJDOOGDWDIURPDWDEOH)RU WDEOHVWKDWDUHQRWUHIHUHQFHGE\)RUHLJQNH\FRQVWUDLQWVPRUHRQWKDWODWHU WKHFRPPDQG EDVLFDOO\GURSVWKHWDEOHWHPSRUDULO\DQGUHFUHDWHVWKHWDEOHZLWKWKHVDPHVWUXFWXUHDV EHIRUH7KLVRSHUDWLRQKDVEDVLFDOO\DFRQVWDQWWLPHFKDUDFWHULVWLF³WKHDPRXQWRIGDWDVWRUHG LQVLGHWKHWDEOHGRHVQRWKDYHDQ\HIIHFWLQWKHWLPHQHHGHGIRUWKHTRUNCATEFRPPDQG %HIRUH0\64/WKHTRUNCATE TABLEVWDWHPHQWIRU,QQR'%WDEOHV ZDVDOZD\VHTXLYDOHQWWRDDELETEVWDWHPHQWUHJDUGOHVVRIZKHWKHU)RUHLJQ NH\FRQVWUDLQWVH[LVWRUQRW7RWDNHDGYDQWDJHRIWKHVSHHGLPSURYHPHQWV \RXKDYHWRXVH0\64/RUODWHU

,QFRPSDULVRQWRDFODVVLFDODELETE FROM employees.salaries;RSHUDWLRQWKH UHGXFWLRQLQWLPHQHHGHGLVVWULNLQJ 2SHUDWLRQ

7LPHQHHGHG

7581&$7(7$%/(

VHF

'(/(7(

PLQVHF

100

Time in seconds

80

60

40

20

0 TRUNCATETABLE

DELETE



0DQDJLQJ'DWD 7KHTRUNCATE TABLEFRPPDQGWDNHVRQO\DIUDFWLRQRIWKHWLPHQHHGHGIRUWKHDELETE +RZHYHUWKHUHDUHVRPHFDYHDWV )LUVWRIDOOWKHTRUNCATEFRPPDQGZLOORQO\KDYHWKHVSHHGDGYDQWDJHRQ,QQR'%WDEOHV LIWKHWDEOHLVQRWUHIHUHQFHGE\DQ\)RUHLJQNH\FRQVWUDLQWV%XWLIWKHWDEOHLVUHIHUHQFHG E\)RUHLJQNH\VWKHTRUNCATE TABLEFRPPDQGLVHTXLYDOHQWWRH[HFXWLQJDDELETE VWDWHPHQWZLWKQRWHEREFODXVHDOVRHOLPLQDWLQJDOOVSHHGGLIIHUHQFHV mysql> CREATE TABLE employees.salaries_referencer ( -> emp_no

INT,

-> from_date

DATE,

-> CONSTRAINT

salaries_fk

-> FOREIGN KEY (emp_no, from_date) -> REFERENCES

salaries (emp_no, from_date)

-> ON DELETE RESTRICT); Query OK, 0 rows affected (0.08 sec) mysql> TRUNCATE TABLE employees.salaries; Query OK, 0 rows affected (1 min 33.44 sec)

2SHUDWLRQ

7LPHQHHGHG

7581&$7(7$%/(ZLWKIRUHLJQNH\UHI

PLQVHF

'(/(7(

PLQVHF

100

Time in seconds

80 60 40 20 0 TRUNCATETABLE (w/FKreferences)

DELETE

)XUWKHUPRUHWKHTRUNCATEVWDWHPHQWUHTXLUHVWKHDROPSULYLOHJHEHIRUH0\64/ LWRQO\UHTXLUHVWKHDELETESHUPLVVLRQ ZKLFKIRUELGVXVHRIWKLVFRPPDQGIRUVRPHXVHUV



&KDSWHU

$QGÀQDOO\TRUNCATELVQRWDWUDQVDFWLRQVDIHFRPPDQG,I\RXH[HFXWHDTRUNCATE VWDWHPHQW\RXZLOOQRWEHDEOHWRSHUIRUPDUROOEDFNRQWKLVRSHUDWLRQDQ\PRUHDQGDQ\ RSHQRSHUDWLRQIURPWKHFXUUHQWWUDQVDFWLRQJHWVDXWRPDWLFDOO\FRPPLWWHGDVZHOO7KLV LVDFKDUDFWHULVWLFWKDWGLVTXDOLÀHVWKLVVWDWHPHQWIRUVLWXDWLRQVLQZKLFKWKHSRVVLELOLW\RI SHUIRUPLQJDUROOEDFNLVPDQGDWRU\\RXZLOOKDYHWRVWLFNZLWKWKHPXFKVORZHU DELETELQ WKHVHFDVHV

7KHUH VPRUH $VZHKDYHVHHQTRUNCATE TABLERQO\KDVSHUIRUPDQFHDGYDQWDJHVLIWKHUHLVQR)RUHLJQ NH\UHIHUHQFHWRWKHWDEOHWKDWLVWREHGHOHWHG+HUHZHZLOOGLVFXVVKRZWRXVHWKHVSHHG LPSURYHPHQWVHYHQLQFDVHRIH[LVWLQJUHIHUHQFHV

7HPSRUDULO\GLVDEOLQJ)RUHLJQNH\FRQVWUDLQWV 7RPDNHXVHRIWKHLQFUHDVHGVSHHGRITRUNCATE TABLEDOWKRXJKWKHWDUJHWWDEOHLVUHI HUHQFHGYLD)RUHLJQNH\V\RXFRXOGWHPSRUDULO\UHPRYHWKH)RUHLJQNH\FRQVWUDLQWVXVHWKH TRUNCATE TABLEFRPPDQGDQGUHHVWDEOLVKWKHUHIHUHQFHVDIWHUZDUGV8VLQJWKHDERYH H[DPSOHRIDWDEOHsalaries_referencerWKDWUHIHUHQFHVVDODULHV\RXFRXOGXVHWKH IROORZLQJVHTXHQFH mysql> ALTER TABLE employees.salaries_referencer -> DROP FOREIGN KEY salaries_fk; Query OK, 0 rows affected (0.19 sec) Records: 0

Duplicates: 0

Warnings: 0

mysql> TRUNCATE TABLE employees.salaries; Query OK, 0 rows affected (0.44 sec) mysql> ALTER TABLE employees.salaries_referencer -> ADD CONSTRAINT salaries_fk -> FOREIGN KEY (emp_no, from_date) -> REFERENCES salaries (emp_no, from_date) -> ON DELETE RESTRICT; Query OK, 0 rows affected (0.14 sec) Records: 0

Duplicates: 0

Warnings: 0



0DQDJLQJ'DWD :LWKWKLVVHTXHQFH\RXWHPSRUDULO\GLVDEOHWKH)RUHLJQNH\FRQVWUDLQWVWRKDYHTRUNCATE TABLEXVHWKHIDVWHUGHOHWLRQPHWKRG%HZDUHKRZHYHUWKDWWKLVPHWKRGPLJKWDOVR OHDGWRSUREOHPVZKHQWKHGHOHWLRQRIWKHWDEOHSURGXFHVORRVHHQGV,IWKHUHIHUHQFLQJ WDEOHsalaries.referencerKROGVUHFRUGVWKDWUHIHUHQFHGWKHQRZHPSW\WDUJHWWDEOH salariesWKHFUHDWLRQRIWKH)RUHLJQNH\FRQVWUDLQWVZLOOIDLO

$OVRNHHSLQPLQGWKDWWKLVVLWXDWLRQPLJKWDOVRRFFXUEHFDXVHRIFRQFXUUHQWSURFHVVHV ZKLFKDUHDEOHIRUWKHGXUDWLRQRIWKHGLVDEOHGFRQVWUDLQWV WRLQVHUWGDWDLQWRWKHWDEOHVWKDW YLRODWHWKHLQWHQGHGUHIHUHQWLDOLQWHJULW\ $VDQDOWHUQDWLYH\RXPLJKWEHWHPSWHGWRWHPSRUDULO\GLVDEOHWKH)RUHLJQNH\FKHFNVE\ VHWWLQJforeign_key_checksWR]HUR:KLOHWKLVZRUNVUHJDUGLQJWKHTRUNCATE TABLE SHUIRUPDQFHLWLVVWURQJO\GLVFRXUDJHGWRXVHWKLVRSWLRQEHFDXVHWKH)RUHLJQNH\LQWHJULW\LV QRWUHYDOLGDWHGZKHQWKH)RUHLJQNH\FKHFNVDUHHQDEOHGDJDLQ6R\RXULVNLQFRQVLVWHQWGDWD ZLWKUHVSHFWWRWKHUHIHUHQWLDOLQWHJULW\

'HOHWLQJDOOEXWDIUDJPHQWRIDODUJH WDEOH VGDWD ,QWKHSUHYLRXVUHFLSH'HOHWLQJDOOGDWDIURPODUJHWDEOHVZHGLVFXVVHGDPHWKRGRITXLFNO\ UHPRYLQJDOOGDWDIURPODUJHWDEOHVZKLOHDYRLGLQJSHUIRUPDQFHKLWV%XWH[SHULHQFHVKRZV WKDW\RXRIWHQPXVWQRWGHOHWHDOOGDWDEXWKDYHWRUHWDLQVRPHUHFRUGVDQGGHOHWHWKHUHVW 7KHTRUNCATE TABLEFRPPDQGGRHVQRWDOORZDQ\DGGLWLRQDOFODXVHVWRGHÀQHZKLFK UHFRUGVWRGHOHWHDQGZKLFKQRWLWDOZD\VGHOHWHVDOOHQWULHV 7KHLQWXLWLYHVROXWLRQWRWKLVZRXOGEHWRXVHDQRUPDODELETEFRPPDQGZLWKDWHEREFODXVH WKDWRQO\PDWFKHVWKHUHFRUGVWRGHOHWH)RUODUJHWDEOHVWKLVPLJKWSURYHTXLWHDQH[SHQVLYH RSHUDWLRQLQWHUPVRIGXUDWLRQ ,QWKLVUHFLSHZHZLOOVKRZ\RXKRZWRTXLFNO\UHPRYHPRVWRI WKHGDWDIURPODUJHWDEOHVZKLOHSUHVHUYLQJVRPHRIWKHUHFRUGV



&KDSWHU

*HWWLQJUHDG\ :HDJDLQQHHGD0\64/VHUYHUXSDQGUXQQLQJDQGD64/FOLHQWOLNHmysql )RUWKLVUHFLSH ZHDOVRQHHGDXVHUDFFRXQWZLWKSELECTINSERTDELETEDROPDQGCREATESULYLOHJHV IRUWKHWDUJHWGDWDEDVHZHZLOOXVHWKHsample_installXVHUWKURXJKRXWWKLVVHFWLRQ :H ZLOOIXUWKHUPRUHXVHWKH(PSOR\HHVVDPSOHGDWDEDVHLQDQ,QQR'%FRQWH[W7KLVGDWDEDVH ZDVLQWURGXFHGLQWKHSUHYLRXVUHFLSHDQGLVDYDLODEOHIRUIUHHRQWKH0\64/ZHEVLWH:HZLOO XVHWKHODUJHVWWDEOHsalariesZLWKPRUHWKDQPLOOLRQUHFRUGV DVWKHWDEOHWRGHOHWH IURP,QRXUH[DPSOHZHZLOOGHOHWHDOOUHFRUGVKDYLQJDfrom_dateEHIRUHWKHWKUHVKROGRI '2002-01-01 00:00:00.0'

+RZWRGRLW  &RQQHFWWRWKHGDWDEDVHXVLQJD64/FOLHQWDQGWKHsample_installDFFRXQW  ([HFXWHWKHIROORZLQJFRPPDQGV mysql> use employees; Database changed mysql> CREATE TABLE salaries_part -> SELECT * FROM salaries -> WHERE from_date >= "2002-01-01 00:00:00.0"; Query OK, 140930 rows affected (11.47 sec)Records: 140930

Duplicates: 0

Warnings: 0

mysql> TRUNCATE TABLE salaries; Query OK, 0 rows affected (0.05 sec) mysql> INSERT INTO salaries SELECT * from salaries_part; Query OK, 140930 rows affected (4.63 sec) Records: 140930

Duplicates: 0

Warnings: 0

mysql> DROP TABLE salaries_part; Query OK, 0 rows affected (0.06 sec)



0DQDJLQJ'DWD

+RZLWZRUNV )RUVSHHGLQJXSWKHGHOHWLRQRIPRVWRIWKHGDWDIURPDODUJHWDEOHZHXWLOL]HWKHVSHHG DGYDQWDJHRIWKHTRUNCATE TABLEVWDWHPHQWRYHUDDELETEFRPPDQG,QGHWDLOWKHVWHSV DUHDVIROORZV 7KHLQLWLDOUSEVWDWHPHQWLVIRUFRQYHQLHQFHRQO\VRZHGRQRWKDYHWRJLYHWKHemployees SUHÀ[IRUHYHU\WDEOH :LWKWKHQH[WVWDWHPHQWCREATE TABLE … SELECT * FROM … ZHVLPSO\FRS\WKHGDWD WKDWVKRXOGQRWEHUHPRYHGWRDQHZO\FUHDWHGWDEOHsalaries_part %HFDUHIXOWRDYRLGHUURUVZKHQLQYHUWLQJFRQGLWLRQVWRGHOHWHDOOHQWULHV EHIRUHWLPH;\RXKDYHWRFRS\DOOUHFRUGVYDOXHVODWHURUHTXDOWR;,I \RXFRS\RQO\UHFRUGVODWHUWKDQ;DOOUHFRUGVH[DFWO\DWWLPH;ZRXOGJHW GHOHWHGDVZHOO

7KLVWDEOHWHPSRUDULO\KROGVWKHGDWDZKLOHZHGHOHWHDOOGDWDIURPWKHODUJHsalaries WDEOHXVLQJTRUNCATELQWKHQH[WVWHS$IWHUZDUGVZHVLPSO\FRS\WKHSDUWLDOGDWDIURPWKH salaries_partWDEOHEDFNLQWRWKHRULJLQDOQRZHPSWLHG salariesWDEOH:LWKWKHÀQDO VWHSZHVFUDSWKHsalaries_partWDEOHDJDLQDVLWLVQRWQHHGHGDQ\PRUH SHOW VARIABLES LIKE 'innodb_fast_shutdown';

 /RJRIIIURP0\64/DQGVWRSWKH0\64/VHUYHU  /RFDWHWKH0\64/FRQÀJXUDWLRQÀOHXVXDOO\FDOOHGmy.cnfRUmy.iniRQ:LQGRZV  DQGRSHQLWLQDWH[WHGLWRU  /RFDWHWKHIROORZLQJSDUDPHWHUVLQWKH[mysqld]VHFWLRQ\RXYDOXHVZLOOYDU\ RIFRXUVH  [mysqld] ... innodb_log_group_home_dir=/var/lib/mysql/redolog innodb_log_file_size=32M innodb_log_buffer_size=64M innodb_log_files_in_group=2 ...



(GLWWKHDERYHFRQÀJXUDWLRQVHWWLQJVWRWKHLUQHZYDOXHV,I\RXUHTXLUHKHOSRQKRZWR ÀQGVXLWDEOHYDOXHVVHHWKH7KHUH VPRUHVHFWLRQRIWKLVUHFLSH

 6DYHWKHFRQÀJXUDWLRQÀOH  1DYLJDWHWRWKHGLUHFWRU\FRQÀJXUHGIRUinnodb_log_group_home_dir,IWKHUHLV QRVXFKVHWWLQJLQ\RXUFRQÀJXUDWLRQÀOHQDYLJDWHWR0\64/ VGDWDGLUHFWRU\WKDWLV WKHQWDNHQDVWKHGHIDXOW



&RQÀJXULQJ0\64/  0RYHWKHÀOHVZKRVHQDPHVVWDUWZLWKib_logfileWRDEDFNXSORFDWLRQ'RQRW FRS\WKHPWKH\PXVWEHUHPRYHGIURPWKHLURULJLQDOORFDWLRQ  5HVWDUWWKH0\64/VHUYHU  9HULI\WKDWQHZÀOHVDUHFUHDWHGDV\RXFRQÀJXUHGWKHP $ ls -l /var/lib/mysqld/redolog

,I\RXGRQRWVHHWKHQHZÀOHVDSSHDUDQGWKHVHUYHUGRHVQRWVWDUWXSFRUUHFWO\FKHFNWKH 0\64/HUURUORJIRUPHVVDJHV8VXDOO\WKHRQO\WKLQJWKDWFDQJRZURQJKHUHLVWKDW\RX HLWKHUPLVW\SHGWKHGLUHFWRU\QDPHRUGLGQRWDFWXDOO\UHPRYHWKHSUHYLRXVib_logfileÀOHV 7RUHVWRUHHYHU\WKLQJEDFNWRWKHRULJLQDOFRQÀJXUDWLRQUHVWRUH\RXUFRQÀJXUDWLRQÀOHIURPWKH EDFNXSDQGUHVWRUHWKHib_logfileÀOHV\RXPRYHGRXWWRWKHEDFNXSWRWKHLURULJLQDOORFDWLRQ

:KDWMXVWKDSSHQHG %\VHWWLQJinnodb_fast_shutdownWR0\RXWROGWKHVHUYHUWRÀQLVKZULWLQJDQ\SHQGLQJ FKDQJHVWRWKHGLVNEHIRUHDFWXDOO\H[LWLQJ7KLVPDNHVVXUHWKHUHDUHQRUHPDLQLQJ WUDQVDFWLRQVLQWKHFXUUHQWUHGRORJVWKDWFRXOGJHWORVWZKHQWKHVHÀOHVDUHUHSODFHG $IWHUWKDW\RXFRXOGFKDQJHWKHFRQÀJXUDWLRQWRQHZYDOXHVSRVVLEO\XVLQJDGLIIHUHQWQXPEHU RIÀOHVDQGGLIIHUHQWVL]HV 7KHQEHIRUHUHVWDUWLQJ\RXFRXOGPRYHWKHROGUHGRORJÀOHVRXWRIWKHZD\7KLVLVLPSRUWDQW EHFDXVHRWKHUZLVH0\64/ZRXOGFRPSODLQDERXWDPLVPDWFKEHWZHHQWKHVHWWLQJVÀOHDQG WKHDFWXDOVLWXDWLRQRQGLVN:KHQLWFRPHVXSÀQGLQJQRUHGRORJÀOHVLWZLOOFUHDWHQHZRQHV ZLWKWKHVHWWLQJVMXVWFRQÀJXUHG

7KHUH VPRUH 2IWHQZKHQWDONLQJDERXWWUDQVDFWLRQVWKHZRUGUROOEDFNFRPHVXS,WPHDQVWKDWLIVRPHWKLQJ JRHVZURQJLQWKHPLGGOHRIDSRVVLEO\FRPSOH[GDWDPDQLSXODWLRQRSHUDWLRQDQGLWKDVWREH DERUWHGWKHGDWDEDVHVHUYHUZLOOVDIHO\UHVWRUHHYHU\WKLQJEDFNWRWKHVWDWHLWZDVLQZKHQWKDW RSHUDWLRQEHJDQQRWOHDYLQJDQ\GDWDRQO\SDUWLDOO\GHOHWHGRUPRGLÀHG



&KDSWHU

7KHRSSRVLWHWHUP³UROOIRUZDUGRUUHGRLQJ³LVOHVVFRPPRQO\XVHG,WPHDQVWKDWZKHQHYHU WKHFRPSOH[RSHUDWLRQPHQWLRQHGHDUOLHUFRPSOHWHVVXFFHVVIXOO\\RXDUHJXDUDQWHHGWKDW QRWKLQJVKRUWRIDFWXDOKDUGZDUHIDLOXUHFRXOGOHDGWRWKHVHFKDQJHVEHLQJORVWDJDLQ 7KLVPLJKWDSSHDUREYLRXVEHFDXVHRQHZRXOGH[SHFWWKHGDWDEDVHVHUYHUQRWWRUHSRUW DQ\WKLQJDVVXFFHVVIXOXQOHVVLWZDVDFWXDOO\FRPSOHWHO\GRQH+RZHYHULIWKDWZHUHVRZULWH RSHUDWLRQVZRXOGEHFRPHSDLQIXOO\VORZDVWKHXQGHUO\LQJ,2VXEV\VWHPJHQHUDOO\PHDQLQJ KDUGGLVNV LVYHU\RIWHQWKHERWWOHQHFNFRPSRQHQW 7RHYDGHWKLVSRWHQWLDOSHUIRUPDQFHSUREOHPPRVWWUDQVDFWLRQDOGDWDEDVHV,QQR'%EHLQJ QRGLIIHUHQWXVHVWKHFRQFHSWRIDWUDQVDFWLRQ³RUUHGR³ORJWKDWDOORZVLWWRPRUHHIÀFLHQWO\ KDQGOHZULWHRSHUDWLRQVZLWKRXWULVNLQJGDWDLQWHJULW\7KHUHGRORJZRUNVDVDVRUWRIVFUDWFK SDGFRQWDLQLQJLQIRUPDWLRQRQZKDWUHPDLQVWREHGRQHWRWKHGDWDÀOHV:LWKLWVKHOSWKH VHUYHUFDQRSWLPL]HGLVNDFFHVVWRLPSURYHSHUIRUPDQFH 7KHLGHDOVL]HIRUWKHUHGRORJGHSHQGVRQWKHVL]HDQGQXPEHURIWUDQVDFWLRQVWKHVHUYHUKDV WRSURFHVV*HQHUDOO\VSHDNLQJWKHORJVKRXOGEHODUJHHQRXJKWRVWRUHDQ\VLQJOHWUDQVDFWLRQ SOXVDERXWSHUFHQW$VDUXOHRIWKXPEWKHWRWDOORJVL]HWKHQXPEHURIORJÀOHVWLPHVWKHLU LQGLYLGXDOVL]H QHHGQRWH[FHHGDERXWRIWKH,QQR'%EXIIHUSRROVL]H )RUPRUHLQIRUPDWLRQRQUHGRORJVDQGKRZWRGHWHUPLQHDVHQVLEOHVL]HVHWWLQJYLVLW http://mysqldump.azundris.com/archives/78-Configuring-InnoDB-AnInnoDB-tutorial.htmlIRUDGHWDLOHGGHVFULSWLRQRI,QQR'%ORJFRQÀJXUDWLRQ

8QGHUVWDQGLQJDQGFRQÀJXULQJLPSRUWDQW 0\64/DQG,QQR'%WLPHRXWRSWLRQV 0\64/ VFRQÀJXUDWLRQÀOHFDQFRQWDLQDYDULHW\RIGLIIHUHQWWLPHRXWVHWWLQJVHDFKUHVSRQVLEOH IRUDVSHFLÀFNLQGRIRSHUDWLRQRUFRQQHFWLRQ,QWKLVUHFLSHZHSUHVHQWDVHOHFWLRQRIWKHVH WLPHRXWVHWWLQJVDQGDVXJJHVWHGYDOXHWRJRDORQJZLWKHDFK7KH+RZLWZRUNVVHFWLRQKDV GHWDLOVRQHDFKYDOXHSUHVHQWHG ,QJHQHUDOWKHYDOXHVVXJJHVWHGKHUHVKRXOGEHDSSURSULDWHIRUERWK0\64/ YHUVLRQVDQG+RZHYHUSOHDVHQRWHWKDWDQ\RIWKHVHRSWLRQVPD\ ZHOOYDU\IRU\RXUHQYLURQPHQWGHSHQGLQJRQZKDWWKHUHTXLUHPHQWVDUHVR SOHDVHGRQRWVLPSO\XVHWKHVHYDOXHVYHUEDWLP

*HWWLQJUHDG\ 7RDSSO\WLPHRXWFRQÀJXUDWLRQVHWWLQJV\RXZLOOQHHGDFFHVVWRWKH0\64/FRQÀJXUDWLRQ ÀOH³W\SLFDOO\my.cnfRUmy.iniRQ:LQGRZV ³DQGWKHULJKWVWRUHVWDUWWKHVHUYHUWRKDYH DQ\FKDQJHVPDGHWRWKHFRQÀJXUDWLRQWDNHHIIHFW



&RQÀJXULQJ0\64/

+RZWRGRLW  /RFDWHWKH0\64/FRQÀJXUDWLRQÀOHDQGRSHQLWLQDWH[WHGLWRU  ,QWKH[mysqld]VHFWLRQVHWXSWKHIROORZLQJYDOXHV6RPHRIWKHRSWLRQVPD\ DOUHDG\EHSUHVHQWRWKHUVZLOOOLNHO\KDYHWREHDGGHG0DNHVXUHHDFKRSWLRQRQO\ DSSHDUVRQFH [mysqld] ...innodb_rollback_on_timeout innodb_lock_wait_timeout=50 interactive_timeout=1200 wait_timeout=28800 net_read_timeout=30 net_write_timeout=120 ...

 6DYHWKHÀOH  5HVWDUWWKH0\64/VHUYHU

+RZLWZRUNV %\VHWWLQJWKHYDOXHVDVGHVFULEHGLQWKHDERYHVHFWLRQ\RXWHOO0\64/WRXVHGLIIHUHQWYDOXHV WKDQWKHGHIDXOWVIRUWKHRSWLRQVPHQWLRQHG7KHQHZVHWWLQJVWDNHHIIHFWZLWKWKHVHUYHUUHVWDUW

6HWWLQJYDOXHVIRULQQRGEBUROOEDFNBRQBWLPHRXWLQQRGEBORFNBZDLWB WLPHRXW 7KHVHWZRVHWWLQJVDUHSUREDEO\WKHPRVWLPSRUWDQWDVUHJDUGVWKHORFNLQJEHKDYLRULQDQ\ 0\64/VHWXS6WDUWLQJZLWKYHUVLRQ0\64/FKDQJHGWKHUROOEDFNEHKDYLRULQFDVH DWLPHRXWRFFXUUHGEHFDXVHDWUDQVDFWLRQFRXOGQRWDFTXLUHDORFNIRUDURZ7KLVXVXDOO\ KDSSHQVZKHQDQRWKHUWUDQVDFWLRQLVVWLOOZRUNLQJRQWKDWURZDQGWKLVLVWREHH[SHFWHG LQQRUPDOGDWDEDVHRSHUDWLRQV7KHGDWDEDVHVHUYHUUROOVEDFNWKHHQWLUHWUDQVDFWLRQLQ WKLVFDVH$SSOLFDWLRQVVKRXOGEHGHVLJQHGWRUHVSRQGWRVXFKFRQGLWLRQVE\UHWU\LQJWKH HQWLUHWUDQVDFWLRQ 3ULRUWRWKLVZDVWKHGHIDXOWEHKDYLRUPDLQWDLQLQJWKHUXOHWKDWDWUDQVDFWLRQHLWKHU VXFFHHGVRUIDLOVDQGLVUROOHGEDFNFRPSOHWHO\DVDQDWRPLFHQWLW\,QDQGQHZHU YHUVLRQVWKHGHIDXOWZDVPRGLÀHGWRUROOEDFNRQO\WKHYHU\ODVWVWDWHPHQWRIWKHIDLOLQJ WUDQVDFWLRQLQVWHDGNHHSLQJWKHWUDQVDFWLRQRSHQ:KLOHWKHUHLVDUHDVRQEHKLQGWKDWFKDQJH IRUORQJWUDQVDFWLRQVLWFDQEHIDVWHUWRMXVWUHWU\WKHYHU\ODVWVWDWHPHQWWKDQZDLWIRUWKH UROOEDFNDQGWU\DJDLQIURPWKHVWDUW LWUHTXLUHVVSHFLDOSUHFDXWLRQVWDNHQRQWKHDSSOLFDWLRQ 

&KDSWHU

OHYHOWREHDEOHWRKDQGOHWKLVYHU\0\64/VSHFLÀFVFHQDULR

6HWWLQJWKHinnodb_rollback_on_timeoutRSWLRQLQWKHFRQÀJXUDWLRQ ÀOHUHVWRUHVWKHPRUHVWDQGDUGZD\RIUROOLQJEDFNWKHZKROHWUDQVDFWLRQLQ FDVHRIDORFNZDLWWLPHRXW:HVWURQJO\UHFRPPHQGHQDEOLQJWKLVRSWLRQ XQOHVV\RXDUHSHUIHFWO\VXUH\RXUDSSOLFDWLRQLVDZDUHRIWKH0\64/VSHFLÀF EHKDYLRUFRQÀJXUHGDVWKHGHIDXOW innodb_lock_wait_timeoutGHÀQHVWKHQXPEHURIVHFRQGVDWUDQVDFWLRQZLOOZDLWWR

DFTXLUHDQHFHVVDU\ORFNZKHQDVHFRQGWUDQVDFWLRQLVZRUNLQJFRQFXUUHQWO\RQWKHVDPH UHFRUGV7KHGHIDXOWYDOXHLVVHFRQGVDQGLIWKHORFNFRXOGQRWEHDFTXLUHGE\WKHQD WLPHRXWHUURUZLOORFFXUDQGWKHWUDQVDFWLRQZLOOEHUROOHGEDFN'HSHQGLQJRQKRZORQJWKH WUDQVDFWLRQVLQ\RXUVHWXSW\SLFDOO\WDNHWKLVYDOXHRIWHQQHHGVWREHDGMXVWHG,I\RXUHJXODUO\ KDYHEXONGDWDRSHUDWLRQVWKDWDIIHFWDORWRIURZV\RXZLOOZDQWWRLQFUHDVHWKLVYDOXH,IRQ WKHRWKHUKDQG\RXUV\VWHPQRUPDOO\XVHVYHU\VKRUWWUDQVDFWLRQVUHGXFLQJWKLVYDOXHPD\ KHOS\RXÀQGRXWDERXWSUREOHPVZLWKORFNFRQWHQWLRQHDUOLHU

6HWWLQJYDOXHVIRULQWHUDFWLYHBWLPHRXWZDLWBWLPHRXW interactive_timeoutGHÀQHVKRZORQJDQLQWHUDFWLYHFOLHQWFRQQHFWLRQFDQEHLGOH

EHIRUHWKHVHUYHUFORVHVLWDXWRPDWLFDOO\ ,GOH LQWKLVFRQWH[WUHIHUVWRWKHWLPHEHWZHHQWZR VWDWHPHQWVEHLQJH[HFXWHGZLWKQRDFWLYLW\LQEHWZHHQ :HUHFRPPHQGUHGXFLQJWKLVIURPWKHGHIDXOWYDOXHRIVHFRQGVKRXUV WRDPXFK ORZHUYDOXHOLNHVHFRQGVPLQXWHV 7KLVDOORZVWKHVHUYHUWRFORVHLGOHFRQQHFWLRQV DQGFRQVHUYHVRPHUHVRXUFHV 7KHFRXQWHUSDUWYDULDEOHIRUQRQLQWHUDFWLYHVHVVLRQVVXFKDVWKRVHIURPDQDSSOLFDWLRQ VHUYHU VFRQQHFWLRQSRROLVFDOOHGwait_timeoutDQGKDVWKHVDPHVHPDQWLFV'HSHQGLQJ RQ\RXUDSSOLFDWLRQ\RXPLJKWZDQWWROHDYHWKLVVHWWLQJRQDKLJKHUYDOXHDVPRVWFRQQHFWLRQ SRROVFDQEHFRQÀJXUHGWRUHOHDVHFRQQHFWLRQVDXWRPDWLFDOO\GHSHQGLQJRQFXUUHQW ORDGFRQGLWLRQV

6HWWLQJYDOXHVIRUQHWBUHDGBWLPHRXWQHWBZULWHBWLPHRXW 7KHSURWRFRO0\64/XVHVWRKDQGOHFRPPXQLFDWLRQEHWZHHQVHUYHUDQGFOLHQWVLVUDWKHU OLPLWHGLQGHVLJQDOORZLQJRQO\RQHRSHUDWLRQWREHFDUULHGRXWDWDWLPH$VLGHHIIHFWRIWKLVLV WKDWRQFHDGDWDWUDQVIHULQHLWKHUGLUHFWLRQKDVVWDUWHGWKHUHLVQRZD\IRULWWREHLQWHUUXSWHG LQDFRQWUROOHGPDQQHU 7KHnet_read_timeoutFRQWUROVKRZORQJDSLHFHRILQIRUPDWLRQFDQEHVHQWIURPWKHFOLHQW WRWKHVHUYHUEHIRUHWKHFRQQHFWLRQLVDERUWHG7KLVLVXVXDOO\QRWDSUREOHP³WKHGHIDXOW VHWWLQJLVVHFRQGV8QGHUQRUHJXODUFLUFXPVWDQFHVZLOODFRPPXQLFDWLRQLQWKDWGLUHFWLRQ WDNHVRORQJ 

&RQÀJXULQJ0\64/ 7KHnet_write_timeoutLVPRUHSUREOHPDWLFEHFDXVHIRUODUJHUHVXOWVHWVWKHGHIDXOW YDOXHRIVHFRQGVPLJKWEHWRRVKRUW7KLVLVHVSHFLDOO\WUXHIRUFOLHQWVWKDWIHWFKWKHUHVXOW LQVWUHDPLQJPRGHSRWHQWLDOO\SHUIRUPLQJWLPHFRQVXPLQJRSHUDWLRQVRQHDFKURZUHWULHYHG WKHUHE\PDNLQJWKHRYHUDOORSHUDWLRQWDNHORQJHUWKDQWKHWLPHRXW 7KHH[DFWYDOXHUHTXLUHGIRU\RXUVHWXSGHSHQGVRQKRZFOLHQWVIHWFKGDWDDQGZRUNZLWKLWVR \RXZLOOQHHGWRH[SHULPHQWDQGÀQGDVXLWDEOHYDOXH 2QHFDYHDWWRFRQVLGHUZLWKnet_write_timeoutLVWKDWLWPD\OHDGWRVHHPLQJO\UDQGRP IDLOXUHVRImysqldumpOLNHWKLV mysqldump: Error 2013: Lost connection to MySQL server during query when dumping table `tablename` at row: 935578

7KLVFDQKDSSHQLIWKHIROORZLQJFRQGLWLRQVDSSO\ f

net_write_timeoutLVVHWWRDORZYDOXH

f

max_allowed_packetLVVHWWRDODUJHYDOXH

'HSHQGLQJRQWKHVSHHGRIWKHQHWZRUNRYHUZKLFKmysqldumpKDVFRQQHFWHGDQGWKHVL]HRI WKHURZVEHLQJGXPSHGLWPD\EHQHFHVVDU\WRLQFUHDVHnet_write_timeoutPXFKKLJKHU DWOHDVWWRDVORQJDVLWWDNHVWRWUDQVPLWmax_allowed_packetE\WHVRYHUWKHQHWZRUNDQG ZULWHLWWRWKHRXWSXW mysqldumpLVDUHJXODUFOLHQWSURJUDPDQGVXEMHFWWRWKHnet_write_timeout VHWWLQJ:KHQWKHVHUYHUVHQGVURZVWREHGXPSHGWRmysqldumpLQFKXQNVRIXS WRmax_allowed_packetE\WHVGHSHQGLQJRQWKHQHWZRUNFRQQHFWLRQLQEHWZHHQ WKLVPLJKWWDNHORQJHUWKDQnet_write_timeoutDOORZVPDNLQJWKHVHUYHUFXWWKH FRQQHFWLRQHYHQWKRXJKQRWKLQJLVUHDOO\ZURQJ,QFUHDVLQJnet_write_timeout IRUWKHmysqldumpWRRO VVHVVLRQZRXOGUHPHG\WKLVEXWXQIRUWXQDWHO\DVRIWKH WLPHRIZULWLQJWKHUHLVQRVXFKVHWWLQJIRUmysqldump$ZRUNDURXQGLI\RXHQFRXQWHU WKLVSUREOHPLVWRWHPSRUDULO\LQFUHDVHWKHJOREDOVHUYHUnet_write_timeoutYDOXH $ mysql -uroot -e "SELECT @@GLOBAL.net_write_timeout AS oldvalue; SET GLOBAL net_write_timeout=600;" $P\VTOGXPS $ mysql -e "SET GLOBAL net_write_timeout=oldvalue;"

7KHÀUVWFRPPDQGZLOOGLVSOD\WKHFXUUHQWYDOXHIRUnet_write_timeoutDQGWKHQVHWLW WRPLQXWHV$IWHUWKDWWKHmysqldumpFDQWDNHSODFH)LQDOO\WKHROGYDOXHLVUHVHWMXVW PDNHVXUH\RXÀOOLQWKHFRUUHFWROGYDOXH  6HH0\64/%XJDWhttp://bugs.mysql.com/bug.php?id=46103IRU PRUHGHWDLOV



&KDSWHU

$GMXVWLQJWDEOHDQGGDWDEDVHQDPHOHWWHU FDVHKDQGOLQJIRUEHWWHUSODWIRUP LQGHSHQGHQFH 0\64/LVDYDLODEOHIRUDYDULHW\RISODWIRUPV³WKHPDMRURQHVEHLQJ:LQGRZVDQG/LQX[(YHQ WKRXJKGDWDÀOHVDUHFRPSDWLEOHDQGFDQEHWUDQVIHUUHGEHWZHHQSODWIRUPVDQGFRQÀJXUDWLRQ PRVWO\IROORZVWKHVDPHJHQHUDOSULQFLSOHVWKHUHLVDQLPSRUWDQWFDYHDWWRNQRZDERXWKRZ GLIIHUHQWRSHUDWLQJV\VWHPVKDQGOHÀOHQDPHV ,QWKLVUHFLSHZHZLOOVKRZ\RXKRZWRVHWXS0\64/LQDZD\VXFKWKDWLWLVPXFKOHVVOLNHO\WR UXQLQWRSUREOHPVZKHQPRYLQJGDWDÀOHVEHWZHHQSODWIRUPV%HFDXVH0\64/GDWDEDVHVDQG WDEOHVFRUUHODWHWRÀOHV\VWHPREMHFWVGLUHFWRULHVDQGÀOHV GLIIHUHQFHVLQKRZWKHRSHUDWLQJ V\VWHPRUUDWKHUWKHÀOHV\VWHP KDQGOHVÀOHDQGGLUHFWRU\QDPHVFDQOHDGWRXQGHVLUHG HIIHFWVHVSHFLDOO\ZKHQZRUNLQJLQKHWHURJHQRXVHQYLURQPHQWV :HJHQHUDOO\UHFRPPHQGVHWWLQJXSDOO\RXU0\64/VHUYHUVDVGHVFULEHGLQ WKLVUHFLSHWRSUHYHQWDQ\SUREOHPV

*HWWLQJUHDG\