All Classes Namespaces Functions Variables Groups Pages
CsvUtil.qm.dox.h
// -*- mode: c++; indent-tabs-mode: nil -*-
// @file CsvUtil.qm Qore user module for working with CSV files
/* CsvUtil.qm Copyright 2012 - 2014 Qore Technologies, sro
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
// this module requires Qore 0.8.8 or better
/* Version History
* 2014-01-31 v1.2: David Nichols <david@qore.org>
+ fixed CsvDataIterator::next() when header_lines > 0 and working with empty input data
* 2013-10-21 v1.2: David Nichols <david@qore.org>
+ implemented support for the "*int", "*float", "*number", and "*date" types
+ implemented support for allowing subclasses of CsvFileIterator to implement support for other custom types
+ fixed "date" type handling with empty input; now returns 1970-01-01 (use "*date" to map empty input to NOTHING)
* 2013-10-09 v1.2: Petr Vanek <petr.vanek@qoretechnologies.com>
+ CsvStringWriter addon
+ AbstractCsvWriter addon
* 2013-09-29 v1.2: David Nichols <david@qore.org>:
+ if "headers" are not given in the CsvUtil::CsvFileIterator::constructor() but "fields" are, then set the headers from the field descriptions automatically
* 2013-06-28 v1.2: Petr Vanek <petr.vanek@qoretechnologies.com>
+ CsvFileWriter addon
* 2013-05-15 v1.1: David Nichols <david@qore.org>:
+ bug fixes to header and fields option processing
+ fixed CsvUtil::CsvFileIterator::index() to return the line index
+ added CsvUtil::CsvFileIterator::lineNumber() to return the current line number in the file
* 2012-10-13 v1.0: David Nichols <david@qore.org>:
+ initial version of module
*/
// private class used to iterate a list and skip elements without any value
class ListValueIterator : public ListIterator {
public:
constructor(*list l);
bool next();
};
class CsvHelper {
public:
private :
const Types = (
"int": True,
"*int": True,
"float": True,
"*float": True,
"number": True,
"*number": True,
"string": True,
"*string": True,
"date": True,
"*date": True,
);
const FieldAttrs = ("type", "format", "timezone", "code");
public:
private setFields();
checkType(string key, string value);
}; // class CsvHelper
namespace CsvUtil {
const EOL_UNIX = "\n";
const EOL_WIN = "\r\n";
const EOL_MACINTOSH = "\r";
// helper list of end of line values
const EOLS = (EOL_UNIX, EOL_WIN, EOL_MACINTOSH, );
class CsvAbstractIterator : public Qore::AbstractIterator, private CsvHelper {
public:
private :
const Options = (
"encoding": True,
"separator": True,
"quote": True,
"eol": True,
"ignore-empty": True,
"ignore-whitespace": True,
"header-lines": True,
"header-names": True,
"headers": True,
"verify-columns": True,
"fields": True,
"timezone": True,
);
public:
private :
// field separator
string separator = ",";
// field content delimiter
string quote = "\"";
// number of header lines
softint headerLines = 0;
// flag to use string names from the first header row if possible
bool headerNames = False;
// True if empty lines should be ignored
bool ignoreEmptyLines = True;
// Flag to trim the field content (trim leading and trailing whitespace) from unquoted fields
bool ignoreWhitespace = True;
// headers / column names for lines iterated
*softlist headers;
// hash of field information (types, formats, and possible code), hash key = column name or number (starting with 0)
*hash fields;
// list of field descriptions (from fields, ordered when headers are set)
*list fdesc;
// the @ref Qore::TimeZone to use when parsing dates (default: current time zone)
*TimeZone tz;
// verify the column count for every row; if a row does not match, then throw a \c CSVFILEITERATOR-DATA-ERROR exception
bool checkElementCounts = False;
// column count for verifying column counts
int cc;
// current record count for the index() method
int rc = 0;
public:
constructor(*hash opts);
private *string getDataName();
private abstract int lineNumberImpl();
private abstract string getLineValueImpl();
private abstract bool nextLineImpl();
bool next();
any memberGate(string name);
hash getValue();
hash getRecord();
list getRecordList();
string getSeparator();
string getQuote();
*list getHeaders();
int index();
int lineNumber();
private any handleType(hash fh, *string val);
private list parseLine();
};
class CsvFileIterator : public CsvUtil::CsvAbstractIterator,public Qore::FileLineIterator, private CsvHelper {
public:
constructor(string path, *hash opts);
private *string getDataName();
private int lineNumberImpl();
private string getLineValueImpl();
private bool nextLineImpl();
}; // CsvFileIterator class
class CsvDataIterator : public CsvUtil::CsvAbstractIterator, private CsvHelper {
public:
private :
string data;
*string eol;
*string line;
int pos = 0;
int lineno = 0;
bool valid = False;
public:
constructor(string data, *hash opts);
bool valid();
private int lineNumberImpl();
private string getLineValueImpl();
private bool nextLineImpl();
};
class AbstractCsvWriter : private CsvHelper {
public:
private :
const Options = (
"encoding": True,
"separator": True,
"quote": True,
"eol": True,
"verify-columns": True,
"fields": True,
"headers": True,
"date-format": True,
);
public:
private :
// TODO/FIXME
string encoding = get_default_encoding();
// field separator
string separator = ",";
// field content delimiter
string quote = "\"";
// end of line sequence
string eol = EOL_UNIX;
// default date->string format
string dateFormat = 'DD/MM/YYYY hh:mm:SS';
// headers / column names for lines iterated
*softlist headers;
// hash of field information (types, formats, and possible code), hash key = column name or number (starting with 0)
*hash fields;
// list of field descriptions (from fields, ordered when headers are set)
*list fdesc;
// verify the column count for every row; if a row does not match, then throw a \c CSVFILEITERATOR-DATA-ERROR exception
bool checkElementCounts = False;
// the latest line number
int lineNo = 0;
// base template for value format
string baseTemplate;
string errname;
public:
constructor(string errname, *hash opts);
writeLine(list values);
writeLine(hash values);
write(AbstractIterator iterator);
abstract private writeRawLine(list values);
private string prepareRawLine(list values);
private string dateFormat(int ix);
}; // AbstractCsvWriter class
class CsvFileWriter : private AbstractCsvWriter {
public:
private :
// a file to write
File file;
public:
constructor(string path, *hash opts);
private writeRawLine(list values);
}; // CsvFileWriter
class CsvStringWriter : private AbstractCsvWriter {
public:
private :
// a csv content
string content;
public:
constructor(*hash opts);
private writeRawLine(list values);
string getContent();
}; // CsvStringWriter
}; // CsvUtil namespace
CsvUtil::CsvFileWriter::writeRawLine
private writeRawLine(list values)
real write implementation. Without any checking.
CsvUtil::CsvAbstractIterator::getQuote
string getQuote()
returns the current quote string
get_default_encoding
string get_default_encoding()
CsvUtil::CsvAbstractIterator::getRecordList
list getRecordList()
returns the current record as a list
CsvUtil::CsvDataIterator::constructor
constructor(string data, *hash opts)
creates the CsvDataIterator with the input data and optionally an option hash
CsvUtil::CsvStringWriter::constructor
constructor(*hash opts)
creates the CsvStringWriter with content in the memory
CsvUtil::AbstractCsvWriter::prepareRawLine
private string prepareRawLine(list values)
Prepare a string (line with EOF) with formatting and escaping.
CsvUtil::CsvAbstractIterator::parseLine
private list parseLine()
parses a line in the file and returns a processed list of the fields
CsvUtil::CsvAbstractIterator
the CsvAbstractIterator class is an abstract base class that allows abstract CSV data to be iterated ...
Definition: CsvUtil.qm.dox.h:340
CsvUtil::CsvAbstractIterator::getHeaders
*list getHeaders()
returns the current column headers or NOTHING if no headers have been detected or saved yet ...
CsvUtil::AbstractCsvWriter::write
write(AbstractIterator iterator)
stream iterator into the file.
CsvUtil::CsvAbstractIterator::getDataName
private *string getDataName()
Returns the name of the input data.
CsvUtil::CsvDataIterator::eol
*string eol
EOL marker.
Definition: CsvUtil.qm.dox.h:606
CsvUtil::CsvAbstractIterator::lineNumber
int lineNumber()
returns the current iterator line number in the file (the first line is line 1) or 0 if not pointing ...
CsvUtil::CsvAbstractIterator::getValue
hash getValue()
returns the current record as a hash
CsvUtil::AbstractCsvWriter::constructor
constructor(string errname, *hash opts)
creates the AbstractCsvWriter
CsvUtil::CsvAbstractIterator::getRecord
hash getRecord()
returns the current record as a hash
CsvUtil::CsvAbstractIterator::next
bool next()
Moves the current line / record position to the next line / record; returns False if there are no mor...
CsvUtil::CsvDataIterator::nextLineImpl
private bool nextLineImpl()
Moves the current line / record position to the next line / record; returns False if there are no mor...
True
const True
CsvUtil::CsvStringWriter::writeRawLine
private writeRawLine(list values)
real write implementation. Without any checking.
CsvUtil::CsvAbstractIterator::Options
const Options
valid options for the object (a hash for quick lookups of valid keys)
Definition: CsvUtil.qm.dox.h:345
CsvUtil::CsvDataIterator::lineno
int lineno
current line number
Definition: CsvUtil.qm.dox.h:612
CsvUtil::CsvDataIterator::pos
int pos
current byte pos
Definition: CsvUtil.qm.dox.h:610
CsvUtil::CsvFileIterator::lineNumberImpl
private int lineNumberImpl()
Returns the current line number.
CsvUtil::CsvFileIterator::constructor
constructor(string path, *hash opts)
creates the CsvFileIterator with the path of the file to read and optionally an option hash ...
False
const False
CsvUtil::CsvFileIterator::getDataName
private *string getDataName()
Returns the name of the input data.
list
list list(...)
CsvUtil::AbstractCsvWriter::Options
const Options
valid options for the object (a hash for quick lookups of valid keys)
Definition: CsvUtil.qm.dox.h:715
CsvUtil::CsvAbstractIterator::getLineValueImpl
abstract private string getLineValueImpl()
Returns the current line.
CsvUtil::CsvFileIterator
the CsvFileIterator class allows CSV files to be iterated on a record basis
Definition: CsvUtil.qm.dox.h:564
CsvUtil::CsvStringWriter::getContent
string getContent()
Get the current in-memory content as a string.
CsvUtil::CsvAbstractIterator::constructor
constructor(*hash opts)
creates the CsvAbstractIterator with an option hash
CsvUtil::CsvStringWriter
the CsvStringWriter class for in-memory string CSV creation
Definition: CsvUtil.qm.dox.h:851
CsvUtil::AbstractCsvWriter
the AbstractCsvWriter class provides a parent for all CSV writers
Definition: CsvUtil.qm.dox.h:710
CsvUtil::EOL_MACINTOSH
const EOL_MACINTOSH
Old (pre-OSX) Macintosh end of line character sequence.
Definition: CsvUtil.qm.dox.h:182
CsvUtil::CsvAbstractIterator::memberGate
any memberGate(string name)
returns the given column value for the current row
CsvUtil::CsvFileIterator::nextLineImpl
private bool nextLineImpl()
Moves the current line / record position to the next line / record; returns False if there are no mor...
CsvUtil::EOL_UNIX
const EOL_UNIX
Unix end of line character sequence (for new OS X too)
Definition: CsvUtil.qm.dox.h:178
CsvUtil::CsvDataIterator::line
*string line
current line
Definition: CsvUtil.qm.dox.h:608
CsvUtil::CsvDataIterator::lineNumberImpl
private int lineNumberImpl()
Returns the current line number; returns 0 if not pointing at any data.
CsvUtil::AbstractCsvWriter::writeLine
writeLine(list values)
write a line with list of values. Data are checked against column rules.
CsvUtil::CsvAbstractIterator::index
int index()
returns the row index being iterated, which does not necessarily correspond to the line number when t...
CsvUtil::CsvAbstractIterator::getSeparator
string getSeparator()
returns the current separator string
CsvUtil::CsvDataIterator::getLineValueImpl
private string getLineValueImpl()
Returns the current line trimmed of the EOL character(s)
CsvUtil::EOL_WIN
const EOL_WIN
MS DOS/Windows end of line character sequence.
Definition: CsvUtil.qm.dox.h:180
CsvUtil::CsvDataIterator
the CsvDataIterator class allows arbitrary CSV string data to be iterated on a record basis ...
Definition: CsvUtil.qm.dox.h:599
CsvUtil::CsvDataIterator::valid
bool valid()
returns True if the iterator is currently pointing at a valid element, False if not ...
CsvUtil::CsvFileWriter::constructor
constructor(string path, *hash opts)
creates the CsvFileWriter with the path of the file to read with an options
CsvUtil::CsvAbstractIterator::nextLineImpl
abstract private bool nextLineImpl()
Moves the current line / record position to the next line / record; returns False if there are no mor...
Qore::FileLineIterator
CsvUtil::CsvAbstractIterator::lineNumberImpl
abstract private int lineNumberImpl()
Returns the current line number.
CsvUtil::CsvDataIterator::data
string data
input data
Definition: CsvUtil.qm.dox.h:604
CsvUtil::CsvFileWriter
the CsvFileWriter class for easy and safe CSV file creation
Definition: CsvUtil.qm.dox.h:823
hash
hash hash(object obj)
CsvUtil::CsvFileIterator::getLineValueImpl
private string getLineValueImpl()
Returns the current line trimmed of the EOL character(s)
CsvUtil::AbstractCsvWriter::writeRawLine
abstract private writeRawLine(list values)
real write implementation. Without any checking.
Qore::AbstractIterator