View Javadoc

1   package net.sf.pzfilereader.examples.largedataset.largecsvperformancetest;
2   
3   import java.io.BufferedReader;
4   import java.io.File;
5   import java.io.FileReader;
6   import java.util.HashMap;
7   import java.util.Map;
8   
9   import net.sf.pzfilereader.DataError;
10  import net.sf.pzfilereader.DataSet;
11  import net.sf.pzfilereader.brparse.BuffReaderDelimPZParser;
12  import net.sf.pzfilereader.brparse.BuffReaderPZParseFactory;
13  
14  /*
15   * Created on Dec 1, 2005
16   *
17   * TODO To change the template for this generated file go to
18   * Window - Preferences - Java - Code Style - Code Templates
19   */
20  
21  /**
22   * @author zepernick
23   * 
24   * TODO To change the template for this generated type comment go to Window -
25   * Preferences - Java - Code Style - Code Templates
26   */
27  public class CSVLarge {
28  
29      public static void main(final String[] args) {
30          try {
31              Map settings = readSettings();
32              String data = (String) settings.get("csvFile");
33              call(data);
34          } catch (Exception e) {
35              // TODO Auto-generated catch block
36              e.printStackTrace();
37          }
38      }
39  
40      public static String getDefaultDataFile() {
41          return "LargeSampleCSV.csv";
42      }
43  
44      public static void call(String data) throws Exception {
45          BuffReaderDelimPZParser pzparse = null;
46          try {
47  
48  
49              // delimited by a comma
50              // text qualified by double quotes
51              // ignore first record
52              pzparse = (BuffReaderDelimPZParser)BuffReaderPZParseFactory.getInstance().newDelimitedParser(new File(data), 
53                       ',', '"');
54      
55              final DataSet ds = pzparse.parse();
56              final long timeStarted = System.currentTimeMillis();
57              int totalCount = 0;
58              int tmpCount = 0;
59              while (ds.next()) {
60                  totalCount++;
61                  tmpCount++;
62                  if (tmpCount >= 2500) {
63                      System.out.println("Read " + totalCount + " Records...");
64                      tmpCount = 0;
65                  }
66              }
67              final long timeFinished = System.currentTimeMillis();
68  
69              String timeMessage = "";
70  
71              if (timeFinished - timeStarted < 1000) {
72                  timeMessage = (timeFinished - timeStarted) + " Milleseconds...";
73              } else {
74                  timeMessage = ((timeFinished - timeStarted) / 1000) + " Seconds...";
75              }
76  
77              System.out.println("");
78              System.out.println("********FILE PARSED IN: " + timeMessage + " ******");
79  
80              if (ds.getErrors() != null && ds.getErrors().size() > 0) {
81                  System.out.println("FOUND ERRORS IN FILE....");
82                  for (int i = 0; i < ds.getErrors().size(); i++) {
83                      final DataError de = (DataError) ds.getErrors().get(i);
84                      System.out.println("Error: " + de.getErrorDesc() + " Line: " + de.getLineNo());
85                  }
86              }
87          } catch (final Exception ex) {
88              ex.printStackTrace();
89          } finally {
90              pzparse.close();
91          }
92  
93      }
94  
95      private static Map readSettings() throws Exception {
96          final Map result = new HashMap();
97          FileReader fr = null;
98          BufferedReader br = null;
99          String line = null;
100 
101         try {
102             fr = new FileReader("settings.properties");
103             br = new BufferedReader(fr);
104 
105             while ((line = br.readLine()) != null) {
106                 if (line.trim().length() == 0 || line.startsWith("#") || line.indexOf("=") == -1) {
107                     continue;
108                 }
109 
110                 result.put(line.substring(0, line.indexOf("=")), line.substring(line.indexOf("=") + 1));
111             }
112         } finally {
113             if (fr != null) {
114                 fr.close();
115             }
116             if (br != null) {
117                 br.close();
118             }
119         }
120 
121         return result;
122 
123     }
124 
125 }