1 /*
2 * ObjectLab, http://www.objectlab.co.uk/open is supporting FlatPack.
3 *
4 * Based in London, we are world leaders in the design and development
5 * of bespoke applications for the securities financing markets.
6 *
7 * <a href="http://www.objectlab.co.uk/open">Click here to learn more</a>
8 * ___ _ _ _ _ _
9 * / _ \| |__ (_) ___ ___| |_| | __ _| |__
10 * | | | | '_ \| |/ _ \/ __| __| | / _` | '_ \
11 * | |_| | |_) | | __/ (__| |_| |__| (_| | |_) |
12 * \___/|_.__// |\___|\___|\__|_____\__,_|_.__/
13 * |__/
14 *
15 * www.ObjectLab.co.uk
16 *
17 * $Id: ColorProvider.java 74 2006-10-24 22:19:05Z benoitx $
18 *
19 * Copyright 2006 the original author or authors.
20 *
21 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
22 * use this file except in compliance with the License. You may obtain a copy of
23 * the License at
24 *
25 * http://www.apache.org/licenses/LICENSE-2.0
26 *
27 * Unless required by applicable law or agreed to in writing, software
28 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
29 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
30 * License for the specific language governing permissions and limitations under
31 * the License.
32 */
33 package net.sf.flatpack;
34
35 import java.io.BufferedReader;
36 import java.io.IOException;
37 import java.io.Reader;
38 import java.util.List;
39 import java.util.Map;
40
41 import org.slf4j.Logger;
42 import org.slf4j.LoggerFactory;
43
44 import net.sf.flatpack.structure.ColumnMetaData;
45 import net.sf.flatpack.structure.Row;
46 import net.sf.flatpack.util.FPConstants;
47 import net.sf.flatpack.util.FixedWidthParserUtils;
48 import net.sf.flatpack.util.ParserUtils;
49
50 /**
51 * @author xhensevb
52 *
53 */
54 public abstract class AbstractFixedLengthParser extends AbstractParser {
55 private static final Logger LOGGER = LoggerFactory.getLogger(AbstractFixedLengthParser.class);
56
57 protected AbstractFixedLengthParser(final Reader dataSourceReader, final String dataDefinition) {
58 super(dataSourceReader, dataDefinition);
59 }
60
61 protected AbstractFixedLengthParser(final Reader dataSourceReader) {
62 super(dataSourceReader);
63 }
64
65 @Override
66 protected DataSet doParse() {
67 try {
68 return doFixedLengthFile(getDataSourceReader());
69 } catch (final IOException e) {
70 LOGGER.error("error accessing/reading data", e);
71 }
72 return null;
73 }
74
75 /*
76 * This is the new version of doDelimitedFile using InputStrem instead of
77 * File. This is more flexible especially it is working with WebStart.
78 *
79 * puts together the dataset for fixed length file. This is used for PZ XML
80 * mappings, and SQL table mappings
81 */
82 private DataSet doFixedLengthFile(final Reader dataSource) throws IOException {
83
84 final DefaultDataSet ds = new DefaultDataSet(getPzMetaData(), this);
85
86 try (BufferedReader br = new BufferedReader(dataSource)) {
87 // gather the conversion properties
88 ds.setPZConvertProps(ParserUtils.loadConvertProperties());
89
90 final Map<String, Integer> recordLengths = ParserUtils.calculateRecordLengths(getPzMetaData());
91
92 // Read in the flat file
93 String line = null;
94 int lineCount = 0;
95 // map of record lengths corresponding to the ID's in the columnMD
96 // array loop through each line in the file
97 while ((line = br.readLine()) != null) {
98 String originalLine = line;
99 lineCount++;
100 // empty line skip past it
101 if (line.trim().length() == 0) {
102 continue;
103 }
104
105 final String mdkey = FixedWidthParserUtils.getCMDKey(getPzMetaData(), line);
106 final int recordLength = recordLengths.get(mdkey);
107
108 if (line.length() > recordLength) {
109 // Incorrect record length on line log the error. Line will not
110 // be included in the
111 // dataset
112 if (isIgnoreExtraColumns()) {
113 // user has choosen to ignore the fact that we have too many bytes in the fixed
114 // width file. Truncate the line to the correct length
115 line = line.substring(0, recordLength);
116 addError(ds, "TRUNCATED LINE TO CORRECT LENGTH", lineCount, 1, isStoreRawDataToDataError() ? originalLine : null);
117 } else {
118 addError(ds, "LINE TOO LONG. LINE IS " + originalLine.length() + " LONG. SHOULD BE " + recordLength, lineCount, 2,
119 isStoreRawDataToDataError() ? originalLine : null);
120 continue;
121 }
122 } else if (line.length() < recordLength) {
123 if (isHandlingShortLines()) {
124 // We can pad this line out
125 line += ParserUtils.padding(recordLength - line.length(), ' ');
126
127 // log a warning
128 addError(ds, "PADDED LINE TO CORRECT RECORD LENGTH", lineCount, 1, isStoreRawDataToDataError() ? originalLine : null);
129
130 } else {
131 addError(ds, "LINE TOO SHORT. LINE IS " + line.length() + " LONG. SHOULD BE " + recordLength, lineCount, 2,
132 isStoreRawDataToDataError() ? originalLine : null);
133 continue;
134 }
135 }
136
137 final Row row = new Row();
138 row.setMdkey(mdkey.equals(FPConstants.DETAIL_ID) ? null : mdkey); // try
139
140 final List<ColumnMetaData> cmds = ParserUtils.getColumnMetaData(mdkey, getPzMetaData());
141 row.addColumn(FixedWidthParserUtils.splitFixedText(cmds, line, isPreserveLeadingWhitespace(), isPreserveTrailingWhitespace()));
142 row.setRowNumber(lineCount);
143
144 if (isFlagEmptyRows()) {
145 // user has elected to have the parser flag rows that are empty
146 row.setEmpty(ParserUtils.isListElementsEmpty(row.getCols()));
147 }
148 if (isStoreRawDataToDataSet()) {
149 // user told the parser to keep a copy of the raw data in the row
150 // WARNING potential for high memory usage here
151 row.setRawData(line);
152 }
153
154 // add the row to the array
155 ds.addRow(row);
156 }
157 } finally {
158 closeReaders();
159 }
160 return ds;
161 }
162 }