View Javadoc

1   package net.sf.flatpack.examples.largedataset.largecsvperformancetest;
2   
3   import java.io.BufferedReader;
4   import java.io.File;
5   import java.io.FileReader;
6   import java.util.HashMap;
7   import java.util.Map;
8   
9   import net.sf.flatpack.DataError;
10  import net.sf.flatpack.DataSet;
11  import net.sf.flatpack.brparse.BuffReaderDelimParser;
12  import net.sf.flatpack.brparse.BuffReaderParseFactory;
13  
14  /*
15   * Created on Dec 1, 2005
16   *
17   * TODO To change the template for this generated file go to
18   * Window - Preferences - Java - Code Style - Code Templates
19   */
20  
21  /**
22   * @author zepernick
23   * 
24   * TODO To change the template for this generated type comment go to Window -
25   * Preferences - Java - Code Style - Code Templates
26   */
27  public class CSVLarge {
28  
29      public static void main(final String[] args) {
30          try {
31              final Map settings = readSettings();
32              final String data = (String) settings.get("csvFile");
33              call(data);
34          } catch (final Exception e) {
35              // TODO Auto-generated catch block
36              e.printStackTrace();
37          }
38      }
39  
40      public static String getDefaultDataFile() {
41          return "LargeSampleCSV.csv";
42      }
43  
44      public static void call(final String data) throws Exception {
45          BuffReaderDelimParser pzparse = null;
46          try {
47  
48              // delimited by a comma
49              // text qualified by double quotes
50              // ignore first record
51              pzparse = (BuffReaderDelimParser) BuffReaderParseFactory.getInstance().newDelimitedParser(new File(data), ',', '"');
52  
53              final DataSet ds = pzparse.parse();
54              final long timeStarted = System.currentTimeMillis();
55              int totalCount = 0;
56              int tmpCount = 0;
57              while (ds.next()) {
58                  totalCount++;
59                  tmpCount++;
60                  if (tmpCount >= 2500) {
61                      System.out.println("Read " + totalCount + " Records...");
62                      tmpCount = 0;
63                  }
64              }
65              final long timeFinished = System.currentTimeMillis();
66  
67              String timeMessage = "";
68  
69              if (timeFinished - timeStarted < 1000) {
70                  timeMessage = (timeFinished - timeStarted) + " Milleseconds...";
71              } else {
72                  timeMessage = ((timeFinished - timeStarted) / 1000) + " Seconds...";
73              }
74  
75              System.out.println("");
76              System.out.println("********FILE PARSED IN: " + timeMessage + " ******");
77  
78              if (ds.getErrors() != null && ds.getErrors().size() > 0) {
79                  System.out.println("FOUND ERRORS IN FILE....");
80                  for (int i = 0; i < ds.getErrors().size(); i++) {
81                      final DataError de = (DataError) ds.getErrors().get(i);
82                      System.out.println("Error: " + de.getErrorDesc() + " Line: " + de.getLineNo());
83                  }
84              }
85          } catch (final Exception ex) {
86              ex.printStackTrace();
87          } finally {
88              pzparse.close();
89          }
90  
91      }
92  
93      private static Map readSettings() throws Exception {
94          final Map result = new HashMap();
95          FileReader fr = null;
96          BufferedReader br = null;
97          String line = null;
98  
99          try {
100             fr = new FileReader("settings.properties");
101             br = new BufferedReader(fr);
102 
103             while ((line = br.readLine()) != null) {
104                 if (line.trim().length() == 0 || line.startsWith("#") || line.indexOf("=") == -1) {
105                     continue;
106                 }
107 
108                 result.put(line.substring(0, line.indexOf("=")), line.substring(line.indexOf("=") + 1));
109             }
110         } finally {
111             if (fr != null) {
112                 fr.close();
113             }
114             if (br != null) {
115                 br.close();
116             }
117         }
118 
119         return result;
120 
121     }
122 
123 }