python MySQLdb insert on update cannot handle more than 20 columns in update
biswadippaul opened this issue · comments
In python I am inserting into a table using "insert on update keyword". However when a certain number of columns are used in the update statement the query goes on forever. I have to kill it manually.In my case the number of columns are 21. So in the SQL5 variable if the number of updates are reduced to 10 this code starts working. I also checked with oursql module and it works with the oursql module. Can somebody tell me if this is a bug or I am doing something wrong. Just to add I have also changed the MYSQL variable max_allowed_packet to 1024M.
To replicate please find the code along with DDL below:
CREATE TABLE `bseb_scada_stg` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`date` date DEFAULT NULL,
`time` time DEFAULT NULL,
`schedule` double DEFAULT NULL,
`total_drawal` double DEFAULT NULL,
`ui_pow` double DEFAULT NULL,
`freq` double DEFAULT NULL,
`ui_rate` double DEFAULT NULL,
`purnea_pg_purnea_bseb` double DEFAULT NULL,
`purnea_pg_kishangunj` double DEFAULT NULL,
`muzaffarpur_pg_mtps_kanti` double DEFAULT NULL,
`kafen_pg_hajipur` double DEFAULT NULL,
`purnea_pg_madhepura_bseb` double DEFAULT NULL,
`kb_gen` double DEFAULT NULL,
`begusarai_b_sharif` double DEFAULT NULL,
`b_sharif_pg_b_sharif_bseb` double DEFAULT NULL,
`pusauli_pg_dehri_bseb` double DEFAULT NULL,
`pusauli_pg_arah_pg` int(11) DEFAULT NULL,
`patna_pg_khagaul_bseb` double DEFAULT NULL,
`patna_pg_fatwa_bseb` int(11) DEFAULT NULL,
`k_gaon_ntpc_k_gaon_bseb` double DEFAULT NULL,
`k_gaon_ntpc_sabour_bseb` double DEFAULT NULL,
`lalmatia_sabour_sult` int(11) DEFAULT NULL,
`b_sharif_barhi_barhi_end` double DEFAULT NULL,
`barhi_rajgir` double DEFAULT NULL,
`b_sharif_tenughat` double DEFAULT NULL,
`sonenar_garhwa` double DEFAULT NULL,
`sultangunj_deoghar` int(11) DEFAULT NULL,
`sonenagar_rihand` int(11) DEFAULT NULL,
`karmanasa_sahupuri` int(11) DEFAULT NULL,
`gaya_pg_bodhgaya_bseb` double DEFAULT NULL,
`gaya_pg_dehri_bseb` double DEFAULT NULL,
`patna_pg_sipara_bseb` double DEFAULT NULL,
`banka_pg_banka_bseb` double DEFAULT NULL,
`lakhisarai_lakhisarai_pg` double DEFAULT NULL,
`lakhisarai_jamui` double DEFAULT NULL,
`PK` int(11) DEFAULT NULL,
`U_I_Ind` int(11) DEFAULT NULL,
`Load_Ind` int(11) DEFAULT NULL,
`Load_Date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `date_time_UNIQUE` (`date`,`time`)) ENGINE=InnoDB DEFAULT CHARSET=utf8;
import MySQLdb
data = [('04/11/2014', '0:01:31', '2129.7', '2166.9', '-37.2', '50.03', '106.8', '123.2413445', '40.52749252', '339.7173157', '96.14920044', '106.1057739', '78.25858307', '120.7000008', '-272.524292', '69.84365845', '-9.233032227', '127.5198746', '0', '31.44219208', '49.53681183', '0', '0.04003052', '0.030030521', '-175.3537292', '34.77833557', '0', '0', '0', '-110.5733643', '168.9000015', '218.2078094', '24.44289398', '60.0909462', '66.49983215'), ('04/11/2014', '0:02:01', '2129.7', '2168', '-38.3', '50', '178', '123.1911049', '40.55540085', '340.3117371', '94.96035767', '106.1057739', '78.25858307', '120.7000008', '-270.7372437', '68.05661011', '-7.445983887', '127.5198746', '0', '31.44219208', '49.53681183', '0', '0.04003052', '0.030030521', '-177.5875549', '34.24224854', '0', '0', '0', '-110.5733643', '168.9000015', '218.2078094', '24.44289398', '62.95358276', '62.94747925'), ('04/11/2014', '0:02:30', '2129.7', '2176.3', '-46.6', '49.97', '240.5', '123.0962105', '40.45493317', '340.9061584', '87.23275757', '106.1057739', '78.25858307', '120.6000023', '-271.6307373', '70.14151001', '-8.637390137', '127.5198746', '0', '31.44219208', '49.53681183', '0', '0.030030521', '0.030030521', '-176.6940308', '34.77833557', '0', '0', '0', '-112.1370392', '167.0999985', '218.2078094', '24.44289398', '64.015625', '66.52424622')]
conn = MySQLdb.connect(passwd="****",db="***")
SQL5 = """insert into `bseb_scada_stg`
(`date`,
`time`,
`schedule`,
`total_drawal`,
`ui_pow`,
`freq`,
`ui_rate`,
`purnea_pg_purnea_bseb`,
`purnea_pg_kishangunj`,
`muzaffarpur_pg_mtps_kanti`,
`kafen_pg_hajipur`,
`purnea_pg_madhepura_bseb`,
`kb_gen`,
`begusarai_b_sharif`,
`b_sharif_pg_b_sharif_bseb`,
`pusauli_pg_dehri_bseb`,
`pusauli_pg_arah_pg`,
`patna_pg_khagaul_bseb`,
`patna_pg_fatwa_bseb`,
`k_gaon_ntpc_k_gaon_bseb`,
`k_gaon_ntpc_sabour_bseb`,
`lalmatia_sabour_sult`,
`b_sharif_barhi_barhi_end`,
`barhi_rajgir`,
`b_sharif_tenughat`,
`sonenar_garhwa`,
`sultangunj_deoghar`,
`sonenagar_rihand`,
`karmanasa_sahupuri`,
`gaya_pg_bodhgaya_bseb`,
`gaya_pg_dehri_bseb`,
`patna_pg_sipara_bseb`,
`banka_pg_banka_bseb`,
`lakhisarai_lakhisarai_pg`,
`lakhisarai_jamui`)
values(
STR_TO_DATE(%s,'%%d/%%m/%%Y'),
STR_TO_DATE(%s,'%%H:%%i:%%s'),
%s, %s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s, %s, %s, %s, %s,
%s, %s, %s)
on duplicate key
update `schedule` = values(schedule),
`total_drawal` = values(total_drawal),
`ui_pow` = values(ui_pow),
`freq` = values(freq),
`ui_rate` = values(ui_rate),
`purnea_pg_purnea_bseb` = values(purnea_pg_purnea_bseb),
`purnea_pg_kishangunj` = values(purnea_pg_kishangunj),
`muzaffarpur_pg_mtps_kanti` = values(muzaffarpur_pg_mtps_kanti),
`kafen_pg_hajipur` = values(kafen_pg_hajipur),
`purnea_pg_madhepura_bseb` = values(purnea_pg_madhepura_bseb),
`kb_gen` = values(kb_gen),
`begusarai_b_sharif` = values(begusarai_b_sharif),
`b_sharif_pg_b_sharif_bseb` = values(b_sharif_pg_b_sharif_bseb),
`pusauli_pg_dehri_bseb` = values(pusauli_pg_dehri_bseb),
`pusauli_pg_arah_pg` = values(pusauli_pg_arah_pg),
`patna_pg_khagaul_bseb` = values(patna_pg_khagaul_bseb),
`patna_pg_fatwa_bseb` = values(patna_pg_fatwa_bseb),
`k_gaon_ntpc_k_gaon_bseb` = values(k_gaon_ntpc_k_gaon_bseb),
`k_gaon_ntpc_sabour_bseb` = values(k_gaon_ntpc_sabour_bseb),
`lalmatia_sabour_sult` = values(lalmatia_sabour_sult),
`b_sharif_barhi_barhi_end` = values(b_sharif_barhi_barhi_end),
`barhi_rajgir` = values(barhi_rajgir),
`b_sharif_tenughat` = values(b_sharif_tenughat),
`sonenar_garhwa` = values(sonenar_garhwa),
`sultangunj_deoghar` = values(sultangunj_deoghar),
`sonenagar_rihand` = values(sonenagar_rihand),
`karmanasa_sahupuri` = values(karmanasa_sahupuri),
`gaya_pg_bodhgaya_bseb` = values(gaya_pg_bodhgaya_bseb),
`gaya_pg_dehri_bseb` = values(gaya_pg_dehri_bseb),
`patna_pg_sipara_bseb` = values(patna_pg_sipara_bseb),
`banka_pg_banka_bseb` = values(banka_pg_banka_bseb),
`lakhisarai_lakhisarai_pg` = values(lakhisarai_lakhisarai_pg),
`lakhisarai_jamui` = values(lakhisarai_jamui)"""
cursor = conn.cursor()
cursor.executemany(SQL5, data)
conn.commit()
cursor.close()