Improving batch inserts into MySQL

We had a custom class to copy data from Oracle to MySQL at AWS. It was sending each statement individually, even though we batched them.

import java.sql.*;
 
public class test {
 
  public static void main(String args[]) throws Exception {
    Class.forName("com.mysql.jdbc.Driver");
    Class.forName("oracle.jdbc.driver.OracleDriver");
    Connection connOra = DriverManager.getConnection("jdbc:oracle:thin:dbuser/******@host:1521/servicename");
    Connection conn = DriverManager.getConnection("jdbc:mysql://host.amazonaws.com/dbname?user=dbuser&password=*********");
    Statement stm = connOra.createStatement();
    stm.setFetchSize(1000);
    ResultSet rst = connOra.createStatement().executeQuery("select * from tab");
    PreparedStatement pst = conn.prepareStatement("insert into tab col1,col2,col3,col4) values (?,?,?,?)");
    ResultSetMetaData rsmd = rst.getMetaData();
    int j = 0;
    while (rst.next()) {
      for (int i = 1; i <= rsmd.getColumnCount(); i++) {
        if (rsmd.getColumnTypeName(i).equals("DATE"))
          pst.setDate(i,rst.getDate(i));
        if (rsmd.getColumnTypeName(i).equals("TIMESTAMP"))
          pst.setTimestamp(i,rst.getTimestamp(i));
        if (rsmd.getColumnTypeName(i).equals("NUMBER"))
          pst.setLong(i,rst.getLong(i));
        if (rsmd.getColumnTypeName(i).equals("VARCHAR2"))
          pst.setString(i,rst.getString(i));
      }
      pst.addBatch();
      if (++j % 1000 == 0) {
        System.out.println("Starting executeBatch at " + new java.util.Date());
        pst.executeBatch();
        System.out.println("Completed executeBatch at " + new java.util.Date());
      }
    }
    pst.executeBatch();
  }
}

As soon as we added rewriteBatchedStatements=true to the JDBC URL for MySQL, the result was a 100 fold reduction in runtime.

Leave a Reply

Your email address will not be published. Required fields are marked *

This site uses Akismet to reduce spam. Learn how your comment data is processed.