CsvUtil.qm.dox.h
// -*- mode: c++; indent-tabs-mode: nil -*-
// @file CsvUtil.qm Qore user module for working with CSV files
/* CsvUtil.qm Copyright 2012 - 2017 Qore Technologies, s.r.o.
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
// minimum required Qore version
// assume local var scope, do not use "$" for vars, members, and method calls
/* see release notes below for version history
*/
class CsvHelper {
public:
private :
const C_OPT1 = 0x1;
const C_OPT2 = 0x2;
const Types = (
"int": True,
"*int": True,
"float": True,
"*float": True,
"number": True,
"*number": True,
"string": True,
"*string": True,
"date": True,
"*date": True,
);
const FieldAttrs = ("type", "format", "timezone", "code", "header");
bool tolwr = False;
string date_format;
hash m_specs;
string errname;
// reorder data according headers set by options.headers or read from CsvHeader
bool headerReorder = True;
public:
constructor (string n_errname);
private:
bool isMultiType();
public:
private:
checkType(string fld_errs, string key, string value);
public:
// get spec from options.fields for old Csv. Check spec param for new Csv
private:
hash getSpec(*hash fields, string fld_errs, int C_OPTx);
public:
private:
hash getSpec1(*hash fields);
public:
private:
hash getSpec2(hash spec);
public:
private:
list adjustFieldsFromHeaders(string type, *list headers, bool check = False);
public:
}; // class CsvHelper
namespace CsvUtil {
const EOL_UNIX = "\n";
const EOL_WIN = "\r\n";
const EOL_MACINTOSH = "\r";
// helper list of end of line values
const EOLS = (EOL_UNIX, EOL_WIN, EOL_MACINTOSH, );
const CSV_TYPE_UNKNOWN = "<unknown>";
const CSV_TYPE_SINGLE = "<single>";
class AbstractCsvIterator : public Qore::AbstractIterator, private CsvHelper {
public:
private :
const Options = (
"date_format": C_OPT1|C_OPT2,
"date-format": C_OPT1|C_OPT2,
"encoding": C_OPT1|C_OPT2,
"eol": C_OPT1|C_OPT2,
"extended_record": C_OPT2,
"fields": C_OPT1,
"header-lines": C_OPT1|C_OPT2,
"header_lines": C_OPT1|C_OPT2,
"header-names": C_OPT1|C_OPT2,
"header_names": C_OPT1|C_OPT2,
"header_reorder": C_OPT1|C_OPT2,
"headers": C_OPT1,
"ignore-empty": C_OPT1|C_OPT2,
"ignore_empty": C_OPT1|C_OPT2,
"ignore-whitespace": C_OPT1|C_OPT2,
"ignore_whitespace": C_OPT1|C_OPT2,
"quote": C_OPT1|C_OPT2,
"separator": C_OPT1|C_OPT2,
"timezone": C_OPT1|C_OPT2,
"tolwr": C_OPT1|C_OPT2,
"verify-columns": C_OPT1|C_OPT2,
"verify_columns": C_OPT1|C_OPT2,
);
// field separator
string separator = ",";
// field content delimiter
string quote = "\"";
// number of header lines
softint headerLines = 0;
// flag to use string names from the first header row if possible
bool headerNames = False;
// True if empty lines should be ignored
bool ignoreEmptyLines = True;
// Flag to trim the field content (trim leading and trailing whitespace) from unquoted fields
bool ignoreWhitespace = True;
// the @ref Qore::TimeZone to use when parsing dates (default: current time zone)
*TimeZone timezone;
// verify the column count for every row; if a row does not match, then throw a \c CSVFILEITERATOR-DATA-ERROR exception
bool checkElementCounts = False;
// getRecord/getValue returns extended hash
bool extendedRecord = False;
// column count for verifying column counts
int cc;
// current record count for the index() method
int rc = 0;
// to resolve record type by rules
hash m_resolve_by_rule;
// to resolve record type by number of fields
hash m_resolve_by_count;
// list of idx to field transformarions, in order of spec
hash m_resolve_by_idx;
// fake specs based on the first non-header row
bool fakeHeaderNames;
// data source iterator
AbstractLineIterator lineIterator;
public:
constructor(AbstractLineIterator li, *hash opts);
// NOTE: when declared as *hash then always calls this constructor
constructor(AbstractLineIterator li, hash spec, hash opts);
private:
processCommonOptions(*hash opts, int C_OPTx);
public:
private:
processSpec(hash spec);
public:
private:
prepareFieldsFromHeaders(*list headers);
public:
bool valid();
bool next();
auto memberGate(string name);
hash getValue();
hash getRecord(bool extended);
hash getRecord();
auto getRecordList();
string getSeparator();
string getQuote();
*list getHeaders();
*list getHeaders(string type);
int index();
int lineNumber();
private:
auto handleType(hash fh, *string val);
public:
private:
list getLineAndSplit();
public:
string identifyType(list rec);
private:
*string identifyTypeImpl(list rec);
public:
private:
hash parseLine();
public:
};
class CsvIterator : public CsvUtil::AbstractCsvIterator {
public:
constructor(Qore::AbstractLineIterator li, *hash opts) ;
constructor(Qore::AbstractLineIterator li, hash spec, hash opts) ;
constructor(Qore::InputStream input, string encoding = "UTF-8", *hash opts) ;
constructor(Qore::InputStream input, string encoding = "UTF-8", hash spec, hash opts) ;
auto memberGate(string name);
};
class CsvFileIterator : public CsvUtil::AbstractCsvIterator {
public:
private :
string m_file_path;
public:
constructor(string path, *hash opts) ;
constructor(string path, hash spec, hash opts) ;
auto memberGate(string name);
string getEncoding();
string getFileName();
hash<Qore::StatInfo> hstat();
list stat();
}; // CsvFileIterator class
class CsvDataIterator : public CsvUtil::AbstractCsvIterator {
public:
constructor(string data, *hash opts) ;
constructor(string data, hash spec, hash opts) ;
auto memberGate(string name);
};
class AbstractCsvWriter : private CsvHelper {
public:
private :
const Options = (
"block": C_OPT1|C_OPT2,
"datamap": C_OPT1,
"date_format": C_OPT1|C_OPT2,
"date-format": C_OPT1|C_OPT2,
"encoding": C_OPT1|C_OPT2,
"eol": C_OPT1|C_OPT2,
"fields": C_OPT1,
"headers": C_OPT1,
"header_reorder": C_OPT1,
"info_log": C_OPT1|C_OPT2,
"optimal_quotes": C_OPT1|C_OPT2,
"optimal-quotes": C_OPT1|C_OPT2,
"quote": C_OPT1|C_OPT2,
"quote_escape": C_OPT1|C_OPT2,
"separator": C_OPT1|C_OPT2,
"verify_columns": C_OPT1|C_OPT2,
"verify-columns": C_OPT1|C_OPT2,
"write_headers": C_OPT1|C_OPT2,
"write-headers": C_OPT1|C_OPT2,
);
string encoding;
string separator = ",";
string quote = "\"";
string m_quoteEscapeChar = "\\";
string eol = EOL_UNIX;
bool checkElementCounts = False;
int lineNo = 0;
int block = 1000;
string baseTemplate;
bool write_headers = True;
bool optimal_quotes = True;
*code info_log;
hash m_out_by_name;
hash m_out_by_idx;
public:
constructor(string n_errname, *hash n_opts);
constructor(string n_errname, hash spec, hash n_opts);
private:
processCommonOptions(*hash n_opts, int C_OPTx);
public:
private:
processSpec();
public:
private:
writeHeaders();
public:
writeLine(list values);
writeLine(hash values);
writeLine(string type, list values);
writeLine(string type, hash values);
write(Qore::AbstractIterator iterator);
write(Qore::SQL::SQLStatement iterator);
write(list l);
private:
abstract writeRawLine(list values);
public:
private:
string prepareRawLine(list values);
public:
private:
string prepareRawLineIntern(list values);
public:
}; // AbstractCsvWriter class
class CsvWriter : public AbstractCsvWriter {
public:
private :
StreamWriter output;
public:
constructor(Qore::OutputStream output, *hash opts) ;
constructor(Qore::OutputStream output, hash spec, hash opts) ;
private:
writeRawLine(list values);
public:
};
class CsvFileWriter : public AbstractCsvWriter {
public:
private :
Qore::File file;
public:
constructor(string path, *hash opts) ;
constructor(string path, hash spec, hash opts) ;
private:
openFile(string path);
public:
private:
writeRawLine(list values);
public:
}; // CsvFileWriter
class CsvStringWriter : public AbstractCsvWriter {
public:
private :
// a csv content
string content;
public:
constructor(*hash opts) ;
constructor(hash spec, hash opts) ;
private:
initContent();
public:
private:
writeRawLine(list values);
public:
string write(Qore::AbstractIterator iterator);
string write(list l);
string getContent();
}; // CsvStringWriter
}; // CsvUtil namespace
CsvUtil::AbstractCsvWriter::m_out_by_name
hash m_out_by_name
mapping output field by name
Definition: CsvUtil.qm.dox.h:1284
stat
*list stat(string path)
CsvUtil::AbstractCsvIterator
the AbstractCsvIterator class is an abstract base class that allows abstract CSV data to be iterated ...
Definition: CsvUtil.qm.dox.h:641
CsvUtil::CsvFileWriter::file
Qore::File file
the file to write
Definition: CsvUtil.qm.dox.h:1480
True
const True
CsvUtil::AbstractCsvWriter::m_out_by_idx
hash m_out_by_idx
mapping output field by index
Definition: CsvUtil.qm.dox.h:1287
False
const False
CsvUtil::AbstractCsvWriter::info_log
*code info_log
a closure/call reference for informational logging when using write(SQLStatement) ...
Definition: CsvUtil.qm.dox.h:1281
list
list list(...)
hstat
*hash< StatInfo > hstat(string path)
CsvUtil::CsvFileIterator
The CsvFileIterator class allows CSV files to be iterated on a record basis.
Definition: CsvUtil.qm.dox.h:1029
index
int index(softstring str, softstring substr, softint pos=0)
Qore::SQL::SQLStatement
Qore::AbstractLineIterator
CsvUtil::CsvStringWriter
The CsvStringWriter class for in-memory string CSV creation.
Definition: CsvUtil.qm.dox.h:1527
CsvUtil::AbstractCsvWriter
The AbstractCsvWriter class provides a parent for all CSV writers.
Definition: CsvUtil.qm.dox.h:1220
CsvUtil::AbstractCsvWriter::baseTemplate
string baseTemplate
base template for value format
Definition: CsvUtil.qm.dox.h:1272
CsvUtil::EOL_MACINTOSH
const EOL_MACINTOSH
Old (pre-OSX) Macintosh end of line character sequence.
Definition: CsvUtil.qm.dox.h:392
CsvUtil::CSV_TYPE_UNKNOWN
const CSV_TYPE_UNKNOWN
Record type when non matching any type.
Definition: CsvUtil.qm.dox.h:398
Qore::OutputStream
CsvUtil::EOL_UNIX
const EOL_UNIX
Unix end of line character sequence (for new OS X too)
Definition: CsvUtil.qm.dox.h:388
CsvUtil::CsvWriter::output
StreamWriter output
the output stream for the CSV data
Definition: CsvUtil.qm.dox.h:1437
CsvUtil::CsvWriter
The CsvWriter class for safe CSV data creation.
Definition: CsvUtil.qm.dox.h:1432
CsvUtil::CsvIterator
The CsvIterator class allows CSV sources to be iterated on a record basis. The source of the input da...
Definition: CsvUtil.qm.dox.h:974
type
string type(auto arg)
CsvUtil::EOL_WIN
const EOL_WIN
MS DOS/Windows end of line character sequence.
Definition: CsvUtil.qm.dox.h:390
CsvUtil::CsvDataIterator
The CsvDataIterator class allows arbitrary CSV string data to be iterated on a record basis...
Definition: CsvUtil.qm.dox.h:1084
CsvUtil::CsvFileIterator::m_file_path
string m_file_path
the path of the file being iterated
Definition: CsvUtil.qm.dox.h:1034
CsvUtil::AbstractCsvWriter::encoding
string encoding
output file character encoding
Definition: CsvUtil.qm.dox.h:1248
Qore::File
CsvUtil
the CsvUtil namespace contains all the objects in the CsvUtil module
Definition: CsvUtil.qm.dox.h:386
CsvUtil::CsvFileWriter
The CsvFileWriter class for safe CSV file creation.
Definition: CsvUtil.qm.dox.h:1475
Qore::InputStream
hash
hash hash(object obj)
CsvUtil::CSV_TYPE_SINGLE
const CSV_TYPE_SINGLE
Record type when multi-type is disabled.
Definition: CsvUtil.qm.dox.h:400
Qore::AbstractIterator