CsvUtil.qm.dox.h
// -*- mode: c++; indent-tabs-mode: nil -*-
// @file CsvUtil.qm Qore user module for working with CSV files
/* CsvUtil.qm Copyright 2012 - 2016 Qore Technologies, sro
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
// minimum required Qore version
// assume local var scope, do not use "$" for vars, members, and method calls
/* see release notes below for version history
*/
class CsvHelper {
public:
private :
const C_OPT1 = 0x1;
const C_OPT2 = 0x2;
const Types = (
"int": True,
"*int": True,
"float": True,
"*float": True,
"number": True,
"*number": True,
"string": True,
"*string": True,
"date": True,
"*date": True,
);
const FieldAttrs = ("type", "format", "timezone", "code", "header");
bool tolwr = False;
string date_format;
hash m_specs;
string errname;
// reorder data according headers set by options.headers or read from CsvHeader
bool headerReorder = True;
public:
constructor (string n_errname);
private bool isMultiType();
private checkType(string fld_errs, string key, string value);
// get spec from options.fields for old Csv. Check spec param for new Csv
private hash getSpec(*hash fields, string fld_errs, int C_OPTx);
private hash getSpec1(*hash fields);
private hash getSpec2(hash spec);
private list adjustFieldsFromHeaders(string type, *list headers);
}; // class CsvHelper
namespace CsvUtil {
const EOL_UNIX = "\n";
const EOL_WIN = "\r\n";
const EOL_MACINTOSH = "\r";
// helper list of end of line values
const EOLS = (EOL_UNIX, EOL_WIN, EOL_MACINTOSH, );
const CSV_TYPE_UNKNOWN = "<unknown>";
const CSV_TYPE_SINGLE = "<single>";
class AbstractCsvIterator : public Qore::AbstractIterator, private CsvHelper {
public:
private :
const Options = (
"date_format": C_OPT1|C_OPT2,
"date-format": C_OPT1|C_OPT2,
"encoding": C_OPT1|C_OPT2,
"eol": C_OPT1|C_OPT2,
"extended_record": C_OPT2,
"fields": C_OPT1,
"header-lines": C_OPT1|C_OPT2,
"header_lines": C_OPT1|C_OPT2,
"header-names": C_OPT1|C_OPT2,
"header_names": C_OPT1|C_OPT2,
"header_reorder": C_OPT1|C_OPT2,
"headers": C_OPT1,
"ignore-empty": C_OPT1|C_OPT2,
"ignore_empty": C_OPT1|C_OPT2,
"ignore-whitespace": C_OPT1|C_OPT2,
"ignore_whitespace": C_OPT1|C_OPT2,
"quote": C_OPT1|C_OPT2,
"separator": C_OPT1|C_OPT2,
"timezone": C_OPT1|C_OPT2,
"tolwr": C_OPT1|C_OPT2,
"verify-columns": C_OPT1|C_OPT2,
"verify_columns": C_OPT1|C_OPT2,
);
// field separator
string separator = ",";
// field content delimiter
string quote = "\"";
// number of header lines
softint headerLines = 0;
// flag to use string names from the first header row if possible
bool headerNames = False;
// True if empty lines should be ignored
bool ignoreEmptyLines = True;
// Flag to trim the field content (trim leading and trailing whitespace) from unquoted fields
bool ignoreWhitespace = True;
// the @ref Qore::TimeZone to use when parsing dates (default: current time zone)
*TimeZone timezone;
// verify the column count for every row; if a row does not match, then throw a \c CSVFILEITERATOR-DATA-ERROR exception
bool checkElementCounts = False;
// getRecord/getValue returns extended hash
bool extendedRecord = False;
// column count for verifying column counts
int cc;
// current record count for the index() method
int rc = 0;
// to resolve record type by rules
hash m_resolve_by_rule;
// to resolve record type by number of fields
hash m_resolve_by_count;
// list of idx to field transformarions, in order of spec
hash m_resolve_by_idx;
public:
constructor(*hash opts);
// NOTE: when declared as *hash then always calls this constructor
constructor(hash spec, hash opts);
private processCommonOptions(*hash opts, int C_OPTx);
private processSpec(hash spec);
private prepareFieldsFromHeaders(*list headers);
private *string getDataName();
private abstract int lineNumberImpl();
private abstract string getLineValueImpl();
private abstract bool nextLineImpl();
bool next();
any memberGate(string name);
hash getValue();
hash getRecord(bool extended);
hash getRecord();
any getRecordList();
string getSeparator();
string getQuote();
*list getHeaders();
*list getHeaders(string type);
int index();
int lineNumber();
private any handleType(hash fh, *string val);
private list getLineAndSplit();
string identifyType(list rec);
private *string identifyTypeImpl(list rec);
private hash parseLine();
};
class CsvFileIterator : public CsvUtil::AbstractCsvIterator,public Qore::FileLineIterator {
public:
constructor(string path, *hash opts) ;
constructor(string path, hash spec, hash opts) ;
any memberGate(string name);
private *string getDataName();
private int lineNumberImpl();
private string getLineValueImpl();
private bool nextLineImpl();
}; // CsvFileIterator class
class CsvDataIterator : public CsvUtil::AbstractCsvIterator,public DataLineIterator {
public:
constructor(string data, *hash opts) ;
constructor(string data, hash spec, hash opts) ;
any memberGate(string name);
private int lineNumberImpl();
private string getLineValueImpl();
private bool nextLineImpl();
};
class AbstractCsvWriter : private CsvHelper {
public:
private :
const Options = (
"block": C_OPT1|C_OPT2,
"datamap": C_OPT1,
"date_format": C_OPT1|C_OPT2,
"date-format": C_OPT1|C_OPT2,
"encoding": C_OPT1|C_OPT2,
"eol": C_OPT1|C_OPT2,
"fields": C_OPT1,
"headers": C_OPT1,
"header_reorder": C_OPT1,
"info_log": C_OPT1|C_OPT2,
"optimal_quotes": C_OPT1|C_OPT2,
"optimal-quotes": C_OPT1|C_OPT2,
"quote": C_OPT1|C_OPT2,
"quote_escape": C_OPT1|C_OPT2,
"separator": C_OPT1|C_OPT2,
"verify_columns": C_OPT1|C_OPT2,
"verify-columns": C_OPT1|C_OPT2,
"write_headers": C_OPT1|C_OPT2,
"write-headers": C_OPT1|C_OPT2,
);
string encoding;
string separator = ",";
string quote = "\"";
string m_quoteEscapeChar = "\\";
string eol = EOL_UNIX;
bool checkElementCounts = False;
int lineNo = 0;
int block = 1000;
string baseTemplate;
bool write_headers = True;
bool optimal_quotes = True;
*code info_log;
hash m_out_by_name;
hash m_out_by_idx;
public:
constructor(string n_errname, *hash n_opts);
constructor(string n_errname, hash spec, hash n_opts);
private processCommonOptions(*hash n_opts, int C_OPTx);
private processSpec();
private writeHeaders();
writeLine(list values);
writeLine(hash values);
writeLine(string type, list values);
writeLine(string type, hash values);
write(Qore::AbstractIterator iterator);
write(Qore::SQL::SQLStatement iterator);
write(list l);
abstract private writeRawLine(list values);
private string prepareRawLine(list values);
}; // AbstractCsvWriter class
class CsvFileWriter : public AbstractCsvWriter {
public:
private :
// a file to write
File file;
public:
constructor(string path, *hash opts) ;
constructor(string path, hash spec, hash opts) ;
private openFile(string path);
private writeRawLine(list values);
}; // CsvFileWriter
class CsvStringWriter : public AbstractCsvWriter {
public:
private :
// a csv content
string content;
public:
constructor(*hash opts) ;
constructor(hash spec, hash opts) ;
private initContent();
private writeRawLine(list values);
string write(Qore::AbstractIterator iterator);
string write(list l);
string getContent();
}; // CsvStringWriter
}; // CsvUtil namespace
CsvUtil::CsvFileWriter::writeRawLine
private writeRawLine(list values)
This method must be overridden in child classes to provide the output implementation.
CsvUtil::AbstractCsvWriter::write_headers
bool write_headers
this flag determines if any stored headers are output
Definition: CsvUtil.qm.dox.h:1155
CsvUtil::CsvDataIterator::constructor
constructor(string data, *hash opts)
Creates the CsvDataIterator with the input data and optionally an option hash.
CsvUtil::AbstractCsvWriter::m_out_by_name
hash m_out_by_name
mapping output field by name
Definition: CsvUtil.qm.dox.h:1164
CsvUtil::CsvStringWriter::constructor
constructor(*hash opts)
creates the CsvStringWriter single-type mode with content in the memory
CsvUtil::AbstractCsvWriter::prepareRawLine
private string prepareRawLine(list values)
Prepare a string (line with EOF) with formatting and escaping.
CsvUtil::AbstractCsvIterator::index
int index()
Returns the row index being iterated, which does not necessarily correspond to the line number when t...
CsvUtil::CsvStringWriter::write
string write(Qore::AbstractIterator iterator)
Stream iterator and return a CSV-formatted output string.
CsvUtil::AbstractCsvIterator
the AbstractCsvIterator class is an abstract base class that allows abstract CSV data to be iterated ...
Definition: CsvUtil.qm.dox.h:592
CsvUtil::AbstractCsvIterator::getDataName
private *string getDataName()
Returns the name of the input data.
CsvUtil::CsvDataIterator::nextLineImpl
private bool nextLineImpl()
Moves the current line / record position to the next line / record; returns False if there are no mor...
True
const True
CsvUtil::AbstractCsvIterator::Options
const Options
valid options for the object (a hash for quick lookups of valid keys)
Definition: CsvUtil.qm.dox.h:597
CsvUtil::AbstractCsvWriter::m_out_by_idx
hash m_out_by_idx
mapping output field by index
Definition: CsvUtil.qm.dox.h:1167
CsvUtil::AbstractCsvIterator::processCommonOptions
private processCommonOptions(*hash opts, int C_OPTx)
process common options and and assing internal fields
CsvUtil::AbstractCsvIterator::getQuote
string getQuote()
Returns the current quote string.
CsvUtil::AbstractCsvWriter::processSpec
private processSpec()
Process specification and set internal variable for mapping.
CsvUtil::AbstractCsvIterator::lineNumberImpl
abstract private int lineNumberImpl()
Returns the current line number.
CsvUtil::CsvStringWriter::writeRawLine
private writeRawLine(list values)
This method must be overridden in child classes to provide the output implementation.
CsvUtil::AbstractCsvWriter::checkElementCounts
bool checkElementCounts
verify the column count for every row; if a row does not match, then throw a CSVFILEITERATOR-DATA-ERR...
Definition: CsvUtil.qm.dox.h:1143
CsvUtil::CsvFileIterator::lineNumberImpl
private int lineNumberImpl()
Returns the current line number.
CsvUtil::CsvFileIterator::constructor
constructor(string path, *hash opts)
Creates the CsvFileIterator in single-type mode with the path of the file to read and an option hash...
CsvUtil::AbstractCsvIterator::parseLine
private hash parseLine()
Parses a line in the file and returns a processed list of the fields.
False
const False
CsvUtil::AbstractCsvIterator::getLineAndSplit
private list getLineAndSplit()
Read line split by separator/quote into list.
CsvUtil::AbstractCsvWriter::m_quoteEscapeChar
string m_quoteEscapeChar
quote escape character
Definition: CsvUtil.qm.dox.h:1137
CsvUtil::CsvFileIterator::getDataName
private *string getDataName()
Returns the name of the input data.
CsvUtil::AbstractCsvWriter::info_log
*code info_log
a closure/call reference for informational logging when using write(SQLStatement) ...
Definition: CsvUtil.qm.dox.h:1161
list
list list(...)
CsvUtil::AbstractCsvWriter::Options
const Options
valid options for the object (a hash for quick lookups of valid keys)
Definition: CsvUtil.qm.dox.h:1105
CsvUtil::AbstractCsvWriter::write
write(Qore::AbstractIterator iterator)
Stream an iterator into the output.
CsvUtil::CsvFileIterator
The CsvFileIterator class allows CSV files to be iterated on a record basis.
Definition: CsvUtil.qm.dox.h:908
CsvUtil::CsvStringWriter::getContent
string getContent()
Get the current in-memory content as a string.
CsvUtil::AbstractCsvWriter::eol
string eol
end of line sequence
Definition: CsvUtil.qm.dox.h:1140
CsvUtil::AbstractCsvIterator::getSeparator
string getSeparator()
Returns the current separator string.
Qore::SQL::SQLStatement
CsvUtil::AbstractCsvIterator::getRecordList
any getRecordList()
Returns the current record as a list.
CsvUtil::AbstractCsvWriter::processCommonOptions
private processCommonOptions(*hash n_opts, int C_OPTx)
Process options and set internal variables.
CsvUtil::CsvStringWriter
The CsvStringWriter class for in-memory string CSV creation.
Definition: CsvUtil.qm.dox.h:1334
CsvUtil::AbstractCsvWriter::separator
string separator
field separator
Definition: CsvUtil.qm.dox.h:1131
CsvUtil::AbstractCsvWriter::quote
string quote
field content delimiter
Definition: CsvUtil.qm.dox.h:1134
CsvUtil::AbstractCsvIterator::memberGate
any memberGate(string name)
Returns the given column value for the current row.
CsvUtil::AbstractCsvWriter
The AbstractCsvWriter class provides a parent for all CSV writers.
Definition: CsvUtil.qm.dox.h:1100
CsvUtil::AbstractCsvIterator::processSpec
private processSpec(hash spec)
process specification and assing internal data for resolving
CsvUtil::AbstractCsvWriter::writeHeaders
private writeHeaders()
Write csv headers.
CsvUtil::AbstractCsvWriter::baseTemplate
string baseTemplate
base template for value format
Definition: CsvUtil.qm.dox.h:1152
CsvUtil::EOL_MACINTOSH
const EOL_MACINTOSH
Old (pre-OSX) Macintosh end of line character sequence.
Definition: CsvUtil.qm.dox.h:342
CsvUtil::CSV_TYPE_UNKNOWN
const CSV_TYPE_UNKNOWN
Record type when non matching any type.
Definition: CsvUtil.qm.dox.h:348
CsvUtil::AbstractCsvIterator::getLineValueImpl
abstract private string getLineValueImpl()
Returns the current line.
type
string type(any arg)
CsvUtil::CsvFileIterator::nextLineImpl
private bool nextLineImpl()
Moves the current line / record position to the next line / record; returns False if there are no mor...
CsvUtil::EOL_UNIX
const EOL_UNIX
Unix end of line character sequence (for new OS X too)
Definition: CsvUtil.qm.dox.h:338
CsvUtil::AbstractCsvIterator::getHeaders
*list getHeaders()
Returns the current record headers or NOTHING if no headers have been detected or saved yet...
CsvUtil::AbstractCsvIterator::identifyTypeImpl
private *string identifyTypeImpl(list rec)
Identify a input record, given the raw line string. This method performs a lookup to a precalculated ...
CsvUtil::AbstractCsvWriter::constructor
constructor(string n_errname, *hash n_opts)
Creates the AbstractCsvWriter in single-type mode.
CsvUtil::AbstractCsvIterator::prepareFieldsFromHeaders
private prepareFieldsFromHeaders(*list headers)
match headers provided at Csv header or in options, never called for multi-type because header_names ...
CsvUtil::CsvDataIterator::lineNumberImpl
private int lineNumberImpl()
Returns the current line number; returns 0 if not pointing at any data.
CsvUtil::AbstractCsvWriter::lineNo
int lineNo
the latest line number
Definition: CsvUtil.qm.dox.h:1146
CsvUtil::AbstractCsvWriter::writeLine
writeLine(list values)
Write a line with a list of values; data are checked against column rules.
CsvUtil::AbstractCsvIterator::getValue
hash getValue()
Returns the current record as a hash.
CsvUtil::CsvDataIterator::getLineValueImpl
private string getLineValueImpl()
Returns the current line trimmed of the EOL character(s)
CsvUtil::EOL_WIN
const EOL_WIN
MS DOS/Windows end of line character sequence.
Definition: CsvUtil.qm.dox.h:340
CsvUtil::AbstractCsvIterator::constructor
constructor(*hash opts)
creates the AbstractCsvIterator with an option hash in single-type mode
CsvUtil::CsvDataIterator
The CsvDataIterator class allows arbitrary CSV string data to be iterated on a record basis...
Definition: CsvUtil.qm.dox.h:953
CsvUtil::CsvFileWriter::constructor
constructor(string path, *hash opts)
creates the CsvFileWriter in single-type mode with the path of the file to read and an optional optio...
Qore::FileLineIterator
CsvUtil::AbstractCsvWriter::block
int block
block size for bulk DML
Definition: CsvUtil.qm.dox.h:1149
CsvUtil::AbstractCsvIterator::identifyType
string identifyType(list rec)
Identify a fixed-length line type using identifyTypeImpl(); may be overridden if necessary.
CsvUtil::AbstractCsvWriter::optimal_quotes
bool optimal_quotes
stores the optimal quotes option
Definition: CsvUtil.qm.dox.h:1158
CsvUtil::AbstractCsvWriter::encoding
string encoding
output file character encoding
Definition: CsvUtil.qm.dox.h:1128
CsvUtil
the CsvUtil namespace contains all the objects in the CsvUtil module
Definition: CsvUtil.qm.dox.h:336
CsvUtil::AbstractCsvIterator::nextLineImpl
abstract private bool nextLineImpl()
Moves the current line / record position to the next line / record; returns False if there are no mor...
CsvUtil::CsvFileWriter
The CsvFileWriter class for safe CSV file creation.
Definition: CsvUtil.qm.dox.h:1290
hash
hash hash(object obj)
CsvUtil::CsvFileIterator::getLineValueImpl
private string getLineValueImpl()
Returns the current line trimmed of the EOL character(s)
CsvUtil::AbstractCsvWriter::writeRawLine
abstract private writeRawLine(list values)
This method must be overridden in child classes to provide the output implementation.
CsvUtil::AbstractCsvIterator::next
bool next()
Moves the current line / record position to the next line / record; returns False if there are no mor...
CsvUtil::CSV_TYPE_SINGLE
const CSV_TYPE_SINGLE
Record type when multi-type is disabled.
Definition: CsvUtil.qm.dox.h:350
Qore::AbstractIterator
CsvUtil::AbstractCsvIterator::getRecord
hash getRecord()
Returns the current record as a hash.
CsvUtil::AbstractCsvIterator::lineNumber
int lineNumber()
Returns the current iterator line number in the file (the first line is line 1) or 0 if not pointing ...