| 1 | |
|
| 2 | |
|
| 3 | |
|
| 4 | |
|
| 5 | |
|
| 6 | |
|
| 7 | |
|
| 8 | |
|
| 9 | |
|
| 10 | |
|
| 11 | |
|
| 12 | |
|
| 13 | |
|
| 14 | |
|
| 15 | |
|
| 16 | |
|
| 17 | |
|
| 18 | |
|
| 19 | |
package net.sourceforge.addam.impexp; |
| 20 | |
|
| 21 | |
import net.sourceforge.addam.impexp.csv.CSVParser; |
| 22 | |
import net.sourceforge.addam.util.TableUtil; |
| 23 | |
|
| 24 | |
import org.apache.tools.ant.BuildException; |
| 25 | |
import org.apache.tools.ant.taskdefs.JDBCTask; |
| 26 | |
|
| 27 | |
import java.io.*; |
| 28 | |
import java.sql.*; |
| 29 | |
import java.util.ArrayList; |
| 30 | |
import java.util.Collections; |
| 31 | |
import java.util.List; |
| 32 | |
|
| 33 | 0 | public class ImportDataTask extends JDBCTask { |
| 34 | 0 | private boolean deleteFirst = false; |
| 35 | 0 | private boolean truncateFirst = false; |
| 36 | |
private Script pre; |
| 37 | |
private Script post; |
| 38 | 0 | private int batchSize = 0; |
| 39 | 0 | private final List<Tables> includes = new ArrayList<Tables>(); |
| 40 | 0 | private final List<Tables> excludes = new ArrayList<Tables>(); |
| 41 | |
protected File dir; |
| 42 | |
|
| 43 | |
|
| 44 | |
|
| 45 | |
|
| 46 | |
public void setDir(File tableDir) { |
| 47 | 0 | dir = tableDir; |
| 48 | 0 | } |
| 49 | |
|
| 50 | |
|
| 51 | |
|
| 52 | |
|
| 53 | |
|
| 54 | |
public void setTruncateFirst(boolean truncateFirst) { |
| 55 | 0 | this.truncateFirst = truncateFirst; |
| 56 | 0 | } |
| 57 | |
|
| 58 | |
|
| 59 | |
|
| 60 | |
|
| 61 | |
|
| 62 | |
public void setDeleteFirst(boolean deleteFirst) { |
| 63 | 0 | this.deleteFirst = deleteFirst; |
| 64 | 0 | } |
| 65 | |
|
| 66 | |
|
| 67 | |
|
| 68 | |
|
| 69 | |
public void addPre(Script script) { |
| 70 | 0 | if (this.pre != null) throw new BuildException("cannot set multiple 'pre' scripts"); |
| 71 | 0 | this.pre = script; |
| 72 | 0 | } |
| 73 | |
|
| 74 | |
|
| 75 | |
|
| 76 | |
|
| 77 | |
public void addPost(Script script) { |
| 78 | 0 | if (this.post != null) throw new BuildException("cannot set multiple 'post' scripts"); |
| 79 | 0 | this.post = script; |
| 80 | 0 | } |
| 81 | |
|
| 82 | |
public void setBatchSize(int batchSize) { |
| 83 | 0 | if (batchSize < 0) throw new BuildException("cannot set negative batchSize"); |
| 84 | 0 | this.batchSize = batchSize; |
| 85 | 0 | } |
| 86 | |
|
| 87 | |
|
| 88 | |
|
| 89 | |
|
| 90 | |
|
| 91 | |
|
| 92 | |
|
| 93 | |
|
| 94 | |
public void execute() throws BuildException { |
| 95 | |
|
| 96 | 0 | if (pre != null) { |
| 97 | 0 | executeScript(pre); |
| 98 | |
} |
| 99 | |
|
| 100 | 0 | if (deleteFirst || truncateFirst) { |
| 101 | 0 | List<String> reverseList = getTableNames(); |
| 102 | 0 | Collections.reverse(reverseList); |
| 103 | 0 | for (String tableName : reverseList) { |
| 104 | 0 | if (deleteFirst || truncateFirst) this.emptyTable(tableName); |
| 105 | 0 | } |
| 106 | |
} |
| 107 | |
|
| 108 | 0 | Connection connection = this.getConnection(); |
| 109 | |
try { |
| 110 | 0 | for (String name : getTableNames()) { |
| 111 | 0 | this.importTable(name, connection); |
| 112 | 0 | } |
| 113 | 0 | } finally { |
| 114 | 0 | try { |
| 115 | 0 | connection.close(); |
| 116 | 0 | } catch (SQLException e) { |
| 117 | |
|
| 118 | 0 | } |
| 119 | 0 | } |
| 120 | |
|
| 121 | 0 | if (post != null) { |
| 122 | 0 | executeScript(post); |
| 123 | |
} |
| 124 | 0 | } |
| 125 | |
|
| 126 | |
private void executeScript(Script script) { |
| 127 | |
try { |
| 128 | 0 | Connection conn = this.getConnection(); |
| 129 | 0 | for (Object o : script.getStatements()) { |
| 130 | 0 | String statementText = (String) o; |
| 131 | 0 | PreparedStatement stmt = null; |
| 132 | |
try { |
| 133 | 0 | if (statementText.startsWith("call")) { |
| 134 | 0 | stmt = conn.prepareCall(statementText); |
| 135 | 0 | System.out.println("calling: " + statementText); |
| 136 | 0 | } else { |
| 137 | 0 | stmt = conn.prepareStatement(statementText); |
| 138 | 0 | System.out.println("executing: " + statementText); |
| 139 | |
} |
| 140 | 0 | int rows = stmt.executeUpdate(); |
| 141 | 0 | System.out.println("...updated " + rows + " rows"); |
| 142 | 0 | } finally { |
| 143 | 0 | if (stmt != null) stmt.close(); |
| 144 | 0 | } |
| 145 | 0 | } |
| 146 | 0 | conn.commit(); |
| 147 | 0 | } catch (SQLException e) { |
| 148 | 0 | e.printStackTrace(); |
| 149 | 0 | throw new BuildException(e); |
| 150 | 0 | } |
| 151 | 0 | } |
| 152 | |
|
| 153 | |
private void emptyTable(String tableName) { |
| 154 | 0 | if (deleteFirst || truncateFirst) { |
| 155 | 0 | log("deleting from table... " + tableName); |
| 156 | |
try { |
| 157 | 0 | Connection conn = this.getConnection(); |
| 158 | 0 | Statement stmt = null; |
| 159 | |
try { |
| 160 | 0 | stmt = conn.createStatement(); |
| 161 | 0 | if (deleteFirst) stmt.executeUpdate("DELETE FROM " + tableName); |
| 162 | 0 | if (truncateFirst) stmt.executeUpdate("TRUNCATE TABLE " + tableName); |
| 163 | 0 | conn.commit(); |
| 164 | 0 | } finally { |
| 165 | 0 | if (stmt != null) stmt.close(); |
| 166 | 0 | } |
| 167 | 0 | } catch (SQLException e) { |
| 168 | 0 | e.printStackTrace(); |
| 169 | 0 | throw new BuildException(e); |
| 170 | 0 | } |
| 171 | |
} |
| 172 | 0 | } |
| 173 | |
|
| 174 | |
|
| 175 | |
|
| 176 | |
|
| 177 | |
|
| 178 | |
|
| 179 | |
|
| 180 | |
public void importTable(String tableName, Connection connection) throws BuildException { |
| 181 | 0 | File fromFile = new File(dir, TableUtil.getFileName(tableName)); |
| 182 | 0 | log("importing " + fromFile + " to table " + tableName); |
| 183 | |
|
| 184 | 0 | int row = 0, col = 0; |
| 185 | |
|
| 186 | 0 | Reader reader = null; |
| 187 | 0 | String cmd = null; |
| 188 | 0 | List record = null; |
| 189 | |
PreparedStatement stmt; |
| 190 | |
try { |
| 191 | |
|
| 192 | 0 | stmt = connection.prepareStatement("SELECT * FROM " + tableName); |
| 193 | 0 | stmt.setMaxRows(1); |
| 194 | 0 | ResultSet rs = stmt.executeQuery(); |
| 195 | 0 | ResultSetMetaData rsmd = rs.getMetaData(); |
| 196 | 0 | int numCols = rsmd.getColumnCount(); |
| 197 | |
|
| 198 | 0 | String columnNames[] = new String[numCols]; |
| 199 | 0 | int columnTypes[] = new int[numCols]; |
| 200 | 0 | String classNames[] = new String[numCols]; |
| 201 | 0 | DataCodec valueGetters[] = new DataCodec[numCols]; |
| 202 | 0 | for (int i = 1; i <= numCols; i++) { |
| 203 | 0 | columnNames[i - 1] = rsmd.getColumnName(i); |
| 204 | 0 | columnTypes[i - 1] = rsmd.getColumnType(i); |
| 205 | 0 | classNames[i - 1] = rsmd.getColumnClassName(i); |
| 206 | 0 | valueGetters[i - 1] = DataCodec.getCodec(classNames[i - 1]); |
| 207 | |
} |
| 208 | |
|
| 209 | 0 | rs.close(); |
| 210 | 0 | stmt.close(); |
| 211 | |
|
| 212 | 0 | reader = new FileReader(fromFile); |
| 213 | 0 | CSVParser parser = new CSVParser(reader); |
| 214 | |
|
| 215 | |
|
| 216 | 0 | StringBuffer colBuf = new StringBuffer(); |
| 217 | 0 | StringBuffer valBuf = new StringBuffer(); |
| 218 | 0 | record = parser.readLine(); |
| 219 | 0 | for (int i = 0; i < record.size(); i++) { |
| 220 | 0 | String columnName = record.get(i).toString(); |
| 221 | 0 | if (!columnName.equals(columnNames[i])) { |
| 222 | 0 | throw new BuildException("error in file " + fromFile + |
| 223 | 0 | " expected " + columnNames[i] + |
| 224 | 0 | " got " + columnName); |
| 225 | |
} |
| 226 | 0 | if (i > 0) { |
| 227 | 0 | colBuf.append(","); |
| 228 | 0 | valBuf.append(","); |
| 229 | |
} |
| 230 | 0 | colBuf.append(columnName); |
| 231 | 0 | valBuf.append("?"); |
| 232 | |
} |
| 233 | |
|
| 234 | 0 | cmd = "INSERT INTO " + tableName + " (" + colBuf.toString() + |
| 235 | 0 | ") VALUES (" + valBuf.toString() + ")"; |
| 236 | |
|
| 237 | |
|
| 238 | 0 | stmt = connection.prepareStatement(cmd); |
| 239 | 0 | for (record = parser.readLine(); record != null; record = parser.readLine()) { |
| 240 | 0 | row++; |
| 241 | |
|
| 242 | 0 | if (record.size() > 0) { |
| 243 | |
|
| 244 | 0 | for (int i = 0; i < record.size(); i++) { |
| 245 | 0 | col = i; |
| 246 | |
|
| 247 | 0 | String value = record.get(i).toString(); |
| 248 | |
|
| 249 | |
|
| 250 | |
|
| 251 | 0 | if (value != null && value.length() > 0) { |
| 252 | 0 | Object o = valueGetters[i].decode(value); |
| 253 | 0 | if (o instanceof java.sql.Timestamp) { |
| 254 | 0 | stmt.setTimestamp(i + 1, (Timestamp) o); |
| 255 | 0 | } else if (o instanceof byte[]) { |
| 256 | 0 | byte buf[] = (byte[]) o; |
| 257 | 0 | InputStream stream = new ByteArrayInputStream(buf); |
| 258 | 0 | stmt.setBinaryStream(i + 1, stream, buf.length); |
| 259 | 0 | } else if (o instanceof java.lang.String) { |
| 260 | 0 | stmt.setString(i + 1, (String) o); |
| 261 | 0 | } else { |
| 262 | 0 | stmt.setObject(i + 1, o, columnTypes[i]); |
| 263 | |
} |
| 264 | 0 | } else { |
| 265 | 0 | stmt.setNull(i + 1, columnTypes[i]); |
| 266 | |
} |
| 267 | |
} |
| 268 | 0 | if (batchSize == 0) { |
| 269 | 0 | stmt.execute(); |
| 270 | 0 | } else { |
| 271 | 0 | stmt.addBatch(); |
| 272 | 0 | if (row % batchSize == 0) { |
| 273 | 0 | stmt.executeBatch(); |
| 274 | |
} |
| 275 | |
} |
| 276 | |
} |
| 277 | |
} |
| 278 | 0 | if (batchSize > 0) { |
| 279 | 0 | stmt.executeBatch(); |
| 280 | |
} |
| 281 | 0 | log(" ...inserted " + row + " rows"); |
| 282 | 0 | stmt.close(); |
| 283 | 0 | connection.commit(); |
| 284 | 0 | } catch (BuildException e) { |
| 285 | |
try { |
| 286 | 0 | for (Object value : record) { |
| 287 | 0 | cmd = cmd.replaceFirst("\\?", "\"" + value + "\""); |
| 288 | 0 | } |
| 289 | 0 | } finally { |
| 290 | 0 | log("error executing statement " + cmd); |
| 291 | 0 | log(" at row" + row + " col" + col); |
| 292 | 0 | } |
| 293 | 0 | throw e; |
| 294 | 0 | } catch (DataCodec.NoSuchCodec e) { |
| 295 | 0 | throw new BuildException(e); |
| 296 | 0 | } catch (Exception e) { |
| 297 | 0 | if (record != null) { |
| 298 | 0 | for (Object value : record) { |
| 299 | 0 | cmd = cmd.replaceFirst("\\?", "\"" + value + "\""); |
| 300 | 0 | } |
| 301 | |
} |
| 302 | 0 | log("error executing statement " + cmd); |
| 303 | 0 | log(" at row" + row + " col" + col); |
| 304 | 0 | e.printStackTrace(); |
| 305 | 0 | throw new BuildException(e); |
| 306 | 0 | } finally { |
| 307 | 0 | try { |
| 308 | 0 | if (reader != null) { |
| 309 | 0 | reader.close(); |
| 310 | |
} |
| 311 | 0 | } catch (IOException e) { |
| 312 | 0 | log(e.toString()); |
| 313 | 0 | } |
| 314 | 0 | } |
| 315 | 0 | } |
| 316 | |
|
| 317 | |
|
| 318 | |
|
| 319 | |
|
| 320 | |
|
| 321 | |
|
| 322 | |
public void setIncludeFile(File tableFile) { |
| 323 | 0 | Tables tables = new Tables(); |
| 324 | 0 | tables.setFile(tableFile); |
| 325 | 0 | includes.add(tables); |
| 326 | 0 | } |
| 327 | |
|
| 328 | |
|
| 329 | |
|
| 330 | |
|
| 331 | |
|
| 332 | |
|
| 333 | |
public void setExcludeFile(File tableFile) { |
| 334 | 0 | Tables tables = new Tables(); |
| 335 | 0 | tables.setFile(tableFile); |
| 336 | 0 | excludes.add(tables); |
| 337 | 0 | } |
| 338 | |
|
| 339 | |
|
| 340 | |
|
| 341 | |
|
| 342 | |
public void setIncludesPattern(String pattern) { |
| 343 | 0 | Tables tables = new Tables(); |
| 344 | 0 | tables.setPattern(pattern); |
| 345 | 0 | includes.add(tables); |
| 346 | 0 | } |
| 347 | |
|
| 348 | |
|
| 349 | |
|
| 350 | |
|
| 351 | |
public void setExcludesPattern(String pattern) { |
| 352 | 0 | Tables tables = new Tables(); |
| 353 | 0 | tables.setPattern(pattern); |
| 354 | 0 | excludes.add(tables); |
| 355 | 0 | } |
| 356 | |
|
| 357 | |
|
| 358 | |
|
| 359 | |
|
| 360 | |
public void setIncludes(String names) { |
| 361 | 0 | Tables tables = new Tables(); |
| 362 | 0 | tables.setNames(names); |
| 363 | 0 | includes.add(tables); |
| 364 | 0 | } |
| 365 | |
|
| 366 | |
|
| 367 | |
|
| 368 | |
|
| 369 | |
public void setExcludes(String names) { |
| 370 | 0 | Tables tables = new Tables(); |
| 371 | 0 | tables.setNames(names); |
| 372 | 0 | excludes.add(tables); |
| 373 | 0 | } |
| 374 | |
|
| 375 | |
|
| 376 | |
|
| 377 | |
|
| 378 | |
|
| 379 | |
|
| 380 | |
protected List<String> getTableNames() { |
| 381 | 0 | List<String> names = new ArrayList<String>(); |
| 382 | 0 | for (File file : dir.listFiles()) { |
| 383 | 0 | String tableName = TableUtil.getTableName(file); |
| 384 | 0 | if (tableName != null) { |
| 385 | 0 | names.add(tableName); |
| 386 | |
} |
| 387 | |
} |
| 388 | 0 | return Tables.narrow(names, includes, excludes); |
| 389 | |
} |
| 390 | |
|
| 391 | |
} |