1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
|
19 | |
package net.sourceforge.addam.impexp; |
20 | |
|
21 | |
import net.sourceforge.addam.impexp.csv.CSVParser; |
22 | |
import net.sourceforge.addam.util.TableUtil; |
23 | |
|
24 | |
import org.apache.tools.ant.BuildException; |
25 | |
import org.apache.tools.ant.taskdefs.JDBCTask; |
26 | |
|
27 | |
import java.io.*; |
28 | |
import java.sql.*; |
29 | |
import java.util.ArrayList; |
30 | |
import java.util.Collections; |
31 | |
import java.util.List; |
32 | |
|
33 | 0 | public class ImportDataTask extends JDBCTask { |
34 | 0 | private boolean deleteFirst = false; |
35 | 0 | private boolean truncateFirst = false; |
36 | |
private Script pre; |
37 | |
private Script post; |
38 | 0 | private int batchSize = 0; |
39 | 0 | private final List<Tables> includes = new ArrayList<Tables>(); |
40 | 0 | private final List<Tables> excludes = new ArrayList<Tables>(); |
41 | |
protected File dir; |
42 | |
|
43 | |
|
44 | |
|
45 | |
|
46 | |
public void setDir(File tableDir) { |
47 | 0 | dir = tableDir; |
48 | 0 | } |
49 | |
|
50 | |
|
51 | |
|
52 | |
|
53 | |
|
54 | |
public void setTruncateFirst(boolean truncateFirst) { |
55 | 0 | this.truncateFirst = truncateFirst; |
56 | 0 | } |
57 | |
|
58 | |
|
59 | |
|
60 | |
|
61 | |
|
62 | |
public void setDeleteFirst(boolean deleteFirst) { |
63 | 0 | this.deleteFirst = deleteFirst; |
64 | 0 | } |
65 | |
|
66 | |
|
67 | |
|
68 | |
|
69 | |
public void addPre(Script script) { |
70 | 0 | if (this.pre != null) throw new BuildException("cannot set multiple 'pre' scripts"); |
71 | 0 | this.pre = script; |
72 | 0 | } |
73 | |
|
74 | |
|
75 | |
|
76 | |
|
77 | |
public void addPost(Script script) { |
78 | 0 | if (this.post != null) throw new BuildException("cannot set multiple 'post' scripts"); |
79 | 0 | this.post = script; |
80 | 0 | } |
81 | |
|
82 | |
public void setBatchSize(int batchSize) { |
83 | 0 | if (batchSize < 0) throw new BuildException("cannot set negative batchSize"); |
84 | 0 | this.batchSize = batchSize; |
85 | 0 | } |
86 | |
|
87 | |
|
88 | |
|
89 | |
|
90 | |
|
91 | |
|
92 | |
|
93 | |
|
94 | |
public void execute() throws BuildException { |
95 | |
|
96 | 0 | if (pre != null) { |
97 | 0 | executeScript(pre); |
98 | |
} |
99 | |
|
100 | 0 | if (deleteFirst || truncateFirst) { |
101 | 0 | List<String> reverseList = getTableNames(); |
102 | 0 | Collections.reverse(reverseList); |
103 | 0 | for (String tableName : reverseList) { |
104 | 0 | if (deleteFirst || truncateFirst) this.emptyTable(tableName); |
105 | 0 | } |
106 | |
} |
107 | |
|
108 | 0 | Connection connection = this.getConnection(); |
109 | |
try { |
110 | 0 | for (String name : getTableNames()) { |
111 | 0 | this.importTable(name, connection); |
112 | 0 | } |
113 | 0 | } finally { |
114 | 0 | try { |
115 | 0 | connection.close(); |
116 | 0 | } catch (SQLException e) { |
117 | |
|
118 | 0 | } |
119 | 0 | } |
120 | |
|
121 | 0 | if (post != null) { |
122 | 0 | executeScript(post); |
123 | |
} |
124 | 0 | } |
125 | |
|
126 | |
private void executeScript(Script script) { |
127 | |
try { |
128 | 0 | Connection conn = this.getConnection(); |
129 | 0 | for (Object o : script.getStatements()) { |
130 | 0 | String statementText = (String) o; |
131 | 0 | PreparedStatement stmt = null; |
132 | |
try { |
133 | 0 | if (statementText.startsWith("call")) { |
134 | 0 | stmt = conn.prepareCall(statementText); |
135 | 0 | System.out.println("calling: " + statementText); |
136 | 0 | } else { |
137 | 0 | stmt = conn.prepareStatement(statementText); |
138 | 0 | System.out.println("executing: " + statementText); |
139 | |
} |
140 | 0 | int rows = stmt.executeUpdate(); |
141 | 0 | System.out.println("...updated " + rows + " rows"); |
142 | 0 | } finally { |
143 | 0 | if (stmt != null) stmt.close(); |
144 | 0 | } |
145 | 0 | } |
146 | 0 | conn.commit(); |
147 | 0 | } catch (SQLException e) { |
148 | 0 | e.printStackTrace(); |
149 | 0 | throw new BuildException(e); |
150 | 0 | } |
151 | 0 | } |
152 | |
|
153 | |
private void emptyTable(String tableName) { |
154 | 0 | if (deleteFirst || truncateFirst) { |
155 | 0 | log("deleting from table... " + tableName); |
156 | |
try { |
157 | 0 | Connection conn = this.getConnection(); |
158 | 0 | Statement stmt = null; |
159 | |
try { |
160 | 0 | stmt = conn.createStatement(); |
161 | 0 | if (deleteFirst) stmt.executeUpdate("DELETE FROM " + tableName); |
162 | 0 | if (truncateFirst) stmt.executeUpdate("TRUNCATE TABLE " + tableName); |
163 | 0 | conn.commit(); |
164 | 0 | } finally { |
165 | 0 | if (stmt != null) stmt.close(); |
166 | 0 | } |
167 | 0 | } catch (SQLException e) { |
168 | 0 | e.printStackTrace(); |
169 | 0 | throw new BuildException(e); |
170 | 0 | } |
171 | |
} |
172 | 0 | } |
173 | |
|
174 | |
|
175 | |
|
176 | |
|
177 | |
|
178 | |
|
179 | |
|
180 | |
public void importTable(String tableName, Connection connection) throws BuildException { |
181 | 0 | File fromFile = new File(dir, TableUtil.getFileName(tableName)); |
182 | 0 | log("importing " + fromFile + " to table " + tableName); |
183 | |
|
184 | 0 | int row = 0, col = 0; |
185 | |
|
186 | 0 | Reader reader = null; |
187 | 0 | String cmd = null; |
188 | 0 | List record = null; |
189 | |
PreparedStatement stmt; |
190 | |
try { |
191 | |
|
192 | 0 | stmt = connection.prepareStatement("SELECT * FROM " + tableName); |
193 | 0 | stmt.setMaxRows(1); |
194 | 0 | ResultSet rs = stmt.executeQuery(); |
195 | 0 | ResultSetMetaData rsmd = rs.getMetaData(); |
196 | 0 | int numCols = rsmd.getColumnCount(); |
197 | |
|
198 | 0 | String columnNames[] = new String[numCols]; |
199 | 0 | int columnTypes[] = new int[numCols]; |
200 | 0 | String classNames[] = new String[numCols]; |
201 | 0 | DataCodec valueGetters[] = new DataCodec[numCols]; |
202 | 0 | for (int i = 1; i <= numCols; i++) { |
203 | 0 | columnNames[i - 1] = rsmd.getColumnName(i); |
204 | 0 | columnTypes[i - 1] = rsmd.getColumnType(i); |
205 | 0 | classNames[i - 1] = rsmd.getColumnClassName(i); |
206 | 0 | valueGetters[i - 1] = DataCodec.getCodec(classNames[i - 1]); |
207 | |
} |
208 | |
|
209 | 0 | rs.close(); |
210 | 0 | stmt.close(); |
211 | |
|
212 | 0 | reader = new FileReader(fromFile); |
213 | 0 | CSVParser parser = new CSVParser(reader); |
214 | |
|
215 | |
|
216 | 0 | StringBuffer colBuf = new StringBuffer(); |
217 | 0 | StringBuffer valBuf = new StringBuffer(); |
218 | 0 | record = parser.readLine(); |
219 | 0 | for (int i = 0; i < record.size(); i++) { |
220 | 0 | String columnName = record.get(i).toString(); |
221 | 0 | if (!columnName.equals(columnNames[i])) { |
222 | 0 | throw new BuildException("error in file " + fromFile + |
223 | 0 | " expected " + columnNames[i] + |
224 | 0 | " got " + columnName); |
225 | |
} |
226 | 0 | if (i > 0) { |
227 | 0 | colBuf.append(","); |
228 | 0 | valBuf.append(","); |
229 | |
} |
230 | 0 | colBuf.append(columnName); |
231 | 0 | valBuf.append("?"); |
232 | |
} |
233 | |
|
234 | 0 | cmd = "INSERT INTO " + tableName + " (" + colBuf.toString() + |
235 | 0 | ") VALUES (" + valBuf.toString() + ")"; |
236 | |
|
237 | |
|
238 | 0 | stmt = connection.prepareStatement(cmd); |
239 | 0 | for (record = parser.readLine(); record != null; record = parser.readLine()) { |
240 | 0 | row++; |
241 | |
|
242 | 0 | if (record.size() > 0) { |
243 | |
|
244 | 0 | for (int i = 0; i < record.size(); i++) { |
245 | 0 | col = i; |
246 | |
|
247 | 0 | String value = record.get(i).toString(); |
248 | |
|
249 | |
|
250 | |
|
251 | 0 | if (value != null && value.length() > 0) { |
252 | 0 | Object o = valueGetters[i].decode(value); |
253 | 0 | if (o instanceof java.sql.Timestamp) { |
254 | 0 | stmt.setTimestamp(i + 1, (Timestamp) o); |
255 | 0 | } else if (o instanceof byte[]) { |
256 | 0 | byte buf[] = (byte[]) o; |
257 | 0 | InputStream stream = new ByteArrayInputStream(buf); |
258 | 0 | stmt.setBinaryStream(i + 1, stream, buf.length); |
259 | 0 | } else if (o instanceof java.lang.String) { |
260 | 0 | stmt.setString(i + 1, (String) o); |
261 | 0 | } else { |
262 | 0 | stmt.setObject(i + 1, o, columnTypes[i]); |
263 | |
} |
264 | 0 | } else { |
265 | 0 | stmt.setNull(i + 1, columnTypes[i]); |
266 | |
} |
267 | |
} |
268 | 0 | if (batchSize == 0) { |
269 | 0 | stmt.execute(); |
270 | 0 | } else { |
271 | 0 | stmt.addBatch(); |
272 | 0 | if (row % batchSize == 0) { |
273 | 0 | stmt.executeBatch(); |
274 | |
} |
275 | |
} |
276 | |
} |
277 | |
} |
278 | 0 | if (batchSize > 0) { |
279 | 0 | stmt.executeBatch(); |
280 | |
} |
281 | 0 | log(" ...inserted " + row + " rows"); |
282 | 0 | stmt.close(); |
283 | 0 | connection.commit(); |
284 | 0 | } catch (BuildException e) { |
285 | |
try { |
286 | 0 | for (Object value : record) { |
287 | 0 | cmd = cmd.replaceFirst("\\?", "\"" + value + "\""); |
288 | 0 | } |
289 | 0 | } finally { |
290 | 0 | log("error executing statement " + cmd); |
291 | 0 | log(" at row" + row + " col" + col); |
292 | 0 | } |
293 | 0 | throw e; |
294 | 0 | } catch (DataCodec.NoSuchCodec e) { |
295 | 0 | throw new BuildException(e); |
296 | 0 | } catch (Exception e) { |
297 | 0 | if (record != null) { |
298 | 0 | for (Object value : record) { |
299 | 0 | cmd = cmd.replaceFirst("\\?", "\"" + value + "\""); |
300 | 0 | } |
301 | |
} |
302 | 0 | log("error executing statement " + cmd); |
303 | 0 | log(" at row" + row + " col" + col); |
304 | 0 | e.printStackTrace(); |
305 | 0 | throw new BuildException(e); |
306 | 0 | } finally { |
307 | 0 | try { |
308 | 0 | if (reader != null) { |
309 | 0 | reader.close(); |
310 | |
} |
311 | 0 | } catch (IOException e) { |
312 | 0 | log(e.toString()); |
313 | 0 | } |
314 | 0 | } |
315 | 0 | } |
316 | |
|
317 | |
|
318 | |
|
319 | |
|
320 | |
|
321 | |
|
322 | |
public void setIncludeFile(File tableFile) { |
323 | 0 | Tables tables = new Tables(); |
324 | 0 | tables.setFile(tableFile); |
325 | 0 | includes.add(tables); |
326 | 0 | } |
327 | |
|
328 | |
|
329 | |
|
330 | |
|
331 | |
|
332 | |
|
333 | |
public void setExcludeFile(File tableFile) { |
334 | 0 | Tables tables = new Tables(); |
335 | 0 | tables.setFile(tableFile); |
336 | 0 | excludes.add(tables); |
337 | 0 | } |
338 | |
|
339 | |
|
340 | |
|
341 | |
|
342 | |
public void setIncludesPattern(String pattern) { |
343 | 0 | Tables tables = new Tables(); |
344 | 0 | tables.setPattern(pattern); |
345 | 0 | includes.add(tables); |
346 | 0 | } |
347 | |
|
348 | |
|
349 | |
|
350 | |
|
351 | |
public void setExcludesPattern(String pattern) { |
352 | 0 | Tables tables = new Tables(); |
353 | 0 | tables.setPattern(pattern); |
354 | 0 | excludes.add(tables); |
355 | 0 | } |
356 | |
|
357 | |
|
358 | |
|
359 | |
|
360 | |
public void setIncludes(String names) { |
361 | 0 | Tables tables = new Tables(); |
362 | 0 | tables.setNames(names); |
363 | 0 | includes.add(tables); |
364 | 0 | } |
365 | |
|
366 | |
|
367 | |
|
368 | |
|
369 | |
public void setExcludes(String names) { |
370 | 0 | Tables tables = new Tables(); |
371 | 0 | tables.setNames(names); |
372 | 0 | excludes.add(tables); |
373 | 0 | } |
374 | |
|
375 | |
|
376 | |
|
377 | |
|
378 | |
|
379 | |
|
380 | |
protected List<String> getTableNames() { |
381 | 0 | List<String> names = new ArrayList<String>(); |
382 | 0 | for (File file : dir.listFiles()) { |
383 | 0 | String tableName = TableUtil.getTableName(file); |
384 | 0 | if (tableName != null) { |
385 | 0 | names.add(tableName); |
386 | |
} |
387 | |
} |
388 | 0 | return Tables.narrow(names, includes, excludes); |
389 | |
} |
390 | |
|
391 | |
} |