All Classes Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
QoreEncoding.h
Go to the documentation of this file.
/* -*- mode: c++; indent-tabs-mode: nil -*- */
/*
QoreEncoding.h
Qore Programming Language
Copyright 2003 - 2013 David Nichols
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef _QORE_CHARSET_H
#define _QORE_CHARSET_H
#include <qore/common.h>
#include <qore/QoreThreadLock.h>
#include <strings.h>
#include <string.h>
#include <map>
#include <string>
typedef qore_size_t (*mbcs_length_t)(const char* str, const char* end, bool &invalid);
typedef qore_size_t (*mbcs_end_t)(const char* str, const char* end, qore_size_t num_chars, bool &invalid);
typedef qore_size_t (*mbcs_pos_t)(const char* str, const char* ptr, bool &invalid);
typedef qore_size_t (*mbcs_charlen_t)(const char* str, qore_size_t valid_len);
class ExceptionSink;
class QoreEncoding {
private:
std::string code;
std::string desc;
mbcs_length_t flength;
mbcs_end_t fend;
mbcs_pos_t fpos;
mbcs_charlen_t fcharlen;
unsigned char maxwidth;
public:
DLLLOCAL QoreEncoding(const char* n_code, const char* n_desc = 0, unsigned char n_maxwidth = 1, mbcs_length_t l = 0, mbcs_end_t e = 0, mbcs_pos_t p = 0, mbcs_charlen_t c = 0) : code(n_code), desc(n_desc ? n_desc : ""), flength(l), fend(e), fpos(p), fcharlen(c), maxwidth(n_maxwidth) {
}
DLLLOCAL ~QoreEncoding() {
}
DLLLOCAL qore_size_t getLength(const char* p, const char* end, bool &invalid) const {
return flength ? flength(p, end, invalid) : strlen(p);
}
DLLLOCAL qore_size_t getLength(const char* p, const char* end, ExceptionSink* xsink) const;
DLLLOCAL qore_size_t getByteLen(const char* p, const char* end, qore_size_t c, bool& invalid) const {
return fend ? fend(p, end, c, invalid) : c;
}
DLLLOCAL qore_size_t getByteLen(const char* p, const char* end, qore_size_t c, ExceptionSink* xsink) const;
DLLLOCAL qore_size_t getCharPos(const char* p, const char* end, bool& invalid) const {
return fpos ? fpos(p, end, invalid) : end - p;
}
DLLLOCAL qore_size_t getCharPos(const char* p, const char* end, ExceptionSink* xsink) const;
DLLLOCAL qore_size_t getCharLen(const char* p, qore_size_t valid_len) const {
return fcharlen ? fcharlen(p, valid_len) : 1;
}
DLLLOCAL bool isMultiByte() const {
return (bool)flength;
}
DLLLOCAL const char* getCode() const {
return code.c_str();
}
DLLLOCAL const char* getDesc() const {
return desc.empty() ? "<no description available>" : desc.c_str();
}
DLLLOCAL int getMaxCharWidth() const {
return maxwidth;
}
};
// case-insensitive maps for encodings
typedef std::map<const char*, QoreEncoding*, class ltcstrcase> encoding_map_t;
typedef std::map<const char*, const QoreEncoding*, class ltcstrcase> const_encoding_map_t;
class QoreString;
class QoreEncodingManager {
private:
DLLLOCAL static encoding_map_t emap;
DLLLOCAL static const_encoding_map_t amap;
DLLLOCAL static class QoreThreadLock mutex;
DLLLOCAL static const QoreEncoding* addUnlocked(const char* code, const char* desc, unsigned char maxwidth = 1, mbcs_length_t l = 0, mbcs_end_t e = 0, mbcs_pos_t p = 0, mbcs_charlen_t = 0);
DLLLOCAL static const QoreEncoding* findUnlocked(const char* name);
public:
DLLEXPORT static void addAlias(const QoreEncoding* qcs, const char* alias);
DLLEXPORT static const QoreEncoding* findCreate(const char* name);
DLLEXPORT static const QoreEncoding* findCreate(const QoreString* str);
DLLEXPORT static void showEncodings();
DLLEXPORT static void showAliases();
DLLEXPORT static const QoreEncoding* add(const char* code, const char* desc = 0, unsigned char maxwidth = 1, mbcs_length_t l = 0, mbcs_end_t e = 0, mbcs_pos_t p = 0, mbcs_charlen_t = 0);
DLLLOCAL static void init(const char* def);
DLLLOCAL QoreEncodingManager();
DLLLOCAL ~QoreEncodingManager();
};
DLLEXPORT qore_size_t q_get_byte_len(const QoreEncoding* enc, const char* p, const char* end, qore_size_t c, ExceptionSink* xsink);
DLLEXPORT qore_size_t q_get_char_len(const QoreEncoding* enc, const char* p, qore_size_t valid_len, ExceptionSink* xsink);
DLLEXPORT extern QoreEncodingManager QEM;
// builtin character encodings
DLLEXPORT extern const QoreEncoding* QCS_DEFAULT,
*QCS_USASCII,
*QCS_UTF8,
*QCS_ISO_8859_1,
*QCS_ISO_8859_2,
*QCS_ISO_8859_3,
*QCS_ISO_8859_4,
*QCS_ISO_8859_5,
*QCS_ISO_8859_6,
*QCS_ISO_8859_7,
*QCS_ISO_8859_8,
*QCS_ISO_8859_9,
*QCS_ISO_8859_10,
*QCS_ISO_8859_11,
*QCS_ISO_8859_13,
*QCS_ISO_8859_14,
*QCS_ISO_8859_15,
*QCS_ISO_8859_16,
*QCS_KOI8_R,
*QCS_KOI8_U,
*QCS_KOI7;
DLLEXPORT qore_size_t q_UTF8_get_char_len(const char* p, qore_size_t valid_len);
#endif // _QORE_CHARSET_H
QoreEncoding::getDesc
DLLLOCAL const char * getDesc() const
returns the description for the encoding
Definition: QoreEncoding.h:166
QCS_UTF8
DLLEXPORT const QoreEncoding * QCS_UTF8
UTF-8 multi-byte encoding (the only multi-byte encoding, all others are single-byte encodings) ...
QoreEncoding::getMaxCharWidth
DLLLOCAL int getMaxCharWidth() const
returns the maximum character width in bytes for the encoding
Definition: QoreEncoding.h:171
QoreEncoding
defines string encoding functions in Qore
Definition: QoreEncoding.h:72
QCS_ISO_8859_8
DLLEXPORT const QoreEncoding * QCS_ISO_8859_8
Hebrew character set.
QCS_ISO_8859_1
DLLEXPORT const QoreEncoding * QCS_ISO_8859_1
latin-1, Western European encoding
QCS_DEFAULT
DLLEXPORT const QoreEncoding * QCS_DEFAULT
the default encoding for the Qore library
QoreEncoding::getLength
DLLLOCAL qore_size_t getLength(const char *p, const char *end, bool &invalid) const
gives the length of the string in characters
Definition: QoreEncoding.h:95
QoreEncodingManager::addAlias
static DLLEXPORT void addAlias(const QoreEncoding *qcs, const char *alias)
adds an alias for an encoding
QCS_ISO_8859_2
DLLEXPORT const QoreEncoding * QCS_ISO_8859_2
latin-2, Central European encoding
QCS_ISO_8859_11
DLLEXPORT const QoreEncoding * QCS_ISO_8859_11
Thai character set.
QoreEncodingManager
manages encodings in Qore
Definition: QoreEncoding.h:185
QCS_ISO_8859_3
DLLEXPORT const QoreEncoding * QCS_ISO_8859_3
latin-3, Southern European character set
qore_size_t
size_t qore_size_t
used for sizes (same range as a pointer)
Definition: common.h:62
mbcs_charlen_t
qore_size_t(* mbcs_charlen_t)(const char *str, qore_size_t valid_len)
for multi-byte encodings: gives the number of total bytes for the character given one or more charact...
Definition: QoreEncoding.h:57
QCS_ISO_8859_4
DLLEXPORT const QoreEncoding * QCS_ISO_8859_4
latin-4, Northern European character set
QoreEncodingManager::showEncodings
static DLLEXPORT void showEncodings()
prints out all valid encodings to stdout
QCS_USASCII
DLLEXPORT const QoreEncoding * QCS_USASCII
ascii encoding
QoreEncoding::getCode
DLLLOCAL const char * getCode() const
returns the string code (ex: &quot;UTF-8&quot;) for the encoding
Definition: QoreEncoding.h:161
QoreEncoding::getByteLen
DLLLOCAL qore_size_t getByteLen(const char *p, const char *end, qore_size_t c, bool &invalid) const
gives the number of bytes for the number of chars in the string or up to the end of the string ...
Definition: QoreEncoding.h:114
QCS_ISO_8859_10
DLLEXPORT const QoreEncoding * QCS_ISO_8859_10
latin-6, Nordic character set
QoreString
Qore&#39;s string type supported by the QoreEncoding class.
Definition: QoreString.h:42
QCS_KOI8_U
DLLEXPORT const QoreEncoding * QCS_KOI8_U
Ukrainian: Kod Obmena Informatsiey, 8 bit.
mbcs_length_t
qore_size_t(* mbcs_length_t)(const char *str, const char *end, bool &invalid)
for multi-byte character set encodings: gives the length of the string in characters ...
Definition: QoreEncoding.h:44
QEM
DLLEXPORT QoreEncodingManager QEM
the QoreEncodingManager object
mbcs_end_t
qore_size_t(* mbcs_end_t)(const char *str, const char *end, qore_size_t num_chars, bool &invalid)
for multi-byte character set encodings: gives the number of bytes for the number of chars ...
Definition: QoreEncoding.h:47
QCS_ISO_8859_9
DLLEXPORT const QoreEncoding * QCS_ISO_8859_9
latin-5, Turkish character set
QoreEncodingManager::showAliases
static DLLEXPORT void showAliases()
prints out all aliases to stdout
QoreEncodingManager::findCreate
static DLLEXPORT const QoreEncoding * findCreate(const char *name)
finds an encoding if it exists (also looks up against alias names) and creates a new one if it doesn&#39;...
QCS_KOI7
DLLEXPORT const QoreEncoding * QCS_KOI7
Russian: Kod Obmena Informatsiey, 7 bit characters.
mbcs_pos_t
qore_size_t(* mbcs_pos_t)(const char *str, const char *ptr, bool &invalid)
for multi-byte character set encodings: gives the character position of the ptr
Definition: QoreEncoding.h:50
ExceptionSink
container for holding Qore-language exception information and also for registering a &quot;thread_exit&quot; ca...
Definition: ExceptionSink.h:35
QCS_ISO_8859_14
DLLEXPORT const QoreEncoding * QCS_ISO_8859_14
latin-8, Celtic character set
QoreEncoding::getCharPos
DLLLOCAL qore_size_t getCharPos(const char *p, const char *end, bool &invalid) const
gives the character position (number of characters) starting from the first pointer to the second ...
Definition: QoreEncoding.h:133
QCS_ISO_8859_6
DLLEXPORT const QoreEncoding * QCS_ISO_8859_6
Arabic character set.
QoreEncoding::isMultiByte
DLLLOCAL bool isMultiByte() const
returns true if the encoding is a multi-byte encoding
Definition: QoreEncoding.h:156
q_UTF8_get_char_len
DLLEXPORT qore_size_t q_UTF8_get_char_len(const char *p, qore_size_t valid_len)
returns the length of the next UTF-8 character or 0 for an encoding error or a negative number if the...
QCS_ISO_8859_5
DLLEXPORT const QoreEncoding * QCS_ISO_8859_5
Cyrillic character set.
common.h
QoreThreadLock
provides a mutually-exclusive thread lock
Definition: QoreThreadLock.h:41
QoreEncoding::getCharLen
DLLLOCAL qore_size_t getCharLen(const char *p, qore_size_t valid_len) const
gives the number of total bytes for the character given one or more characters
Definition: QoreEncoding.h:151
QoreEncodingManager::add
static DLLEXPORT const QoreEncoding * add(const char *code, const char *desc=0, unsigned char maxwidth=1, mbcs_length_t l=0, mbcs_end_t e=0, mbcs_pos_t p=0, mbcs_charlen_t=0)
adds a new encoding to the list
QCS_ISO_8859_16
DLLEXPORT const QoreEncoding * QCS_ISO_8859_16
latin-10, Southeast European character set
QCS_ISO_8859_15
DLLEXPORT const QoreEncoding * QCS_ISO_8859_15
latin-9, Western European with euro symbol
QCS_KOI8_R
DLLEXPORT const QoreEncoding * QCS_KOI8_R
Russian: Kod Obmena Informatsiey, 8 bit.
QCS_ISO_8859_7
DLLEXPORT const QoreEncoding * QCS_ISO_8859_7
Greek character set.
QCS_ISO_8859_13
DLLEXPORT const QoreEncoding * QCS_ISO_8859_13
latin-7, Baltic rim character set