#include <QDataStream>
#include <QMutex>
#include <QRegularExpression>
+#include <QtMath>
typedef unsigned char uint8_t;
#include "rplmodules.hpp"
#include "rplcore/rplexception.hpp"
#include "rplcore/rplcontainer.hpp"
#include "rplcore/rplstring.hpp"
+#include "rplcore/rplqstring.hpp"
#include "rplcore/rplconfigurator.hpp"
#include "rplcore/rplconfig.hpp"
#include "rplcore/rplterminator.hpp"
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+
+#include "rplcore/rplcore.hpp"
+
+
+/**
+ * @brief Determines the length and vlaue of an integer.
+ *
+ * @param text the number as text
+ * @param start the first index to inspect
+ * @param radix the base of the number sytem: 8 (octal), 10 or 16
+ * @param pValue OUT: the value of the integer. May be NULL
+ *
+ * @return <=0: no integer found
+ * otherwise: the length of the integer
+ */
+int RplQString::lengthOfUInt64(const QString& text, int start,
+ int radix, quint64* pValue)
+{
+ int inputLength = text.size();
+ qint64 value = 0;
+ int ix = start;
+ int cc;
+ if (radix == 10){
+ while (ix < inputLength){
+ if ( (cc = text[ix].unicode()) >= '0' && cc <= '9')
+ value = value * 10 + cc - '0';
+ else
+ break;
+ ix++;
+ }
+ } else if (radix == 16){
+ while (ix < inputLength){
+ if ( (cc = text[ix].unicode()) >= '0' && cc <= '9')
+ value = value * 16 + cc - '0';
+ else if (cc >= 'A' && cc <= 'F')
+ value = value * 16 + cc - 'A' + 10;
+ else if (cc >= 'a' && cc <= 'f')
+ value = value * 16 + cc - 'a' + 10;
+ else
+ break;
+ ix++;
+ }
+ } else if (radix == 8){
+ while (ix < inputLength){
+ if ( (cc = text[ix].unicode()) >= '0' && cc <= '7')
+ value = value * 8 + cc - '0';
+ else
+ break;
+ ix++;
+ }
+ } else {
+ throw RplException("RplQString::lengthOfInt(): wrong radix: %d", radix);
+ }
+ if (pValue != NULL)
+ *pValue = value;
+ return ix - start;
+}
+/**
+ * @brief Determines the length and value of an unsigned integer.
+ *
+ * @param text the number as text
+ * @param start the first index to inspect
+ * @param radix the base of the number sytem: 8 (octal), 10 or 16
+ * @param pValue OUT: the value of the integer. May be NULL
+ *
+ * @return 0: no integer found
+ * otherwise: the length of the integer
+ */
+int RplQString::lengthOfUInt(const QString& text, int start,
+ int radix, uint* pValue)
+{
+ quint64 value;
+ int rc = lengthOfUInt64(text, start, radix, &value);
+ if (pValue != NULL)
+ *pValue = (uint) value;
+ return rc;
+}
+
+/**
+ * @brief Determines the length and value of a floting point number.
+ *
+ * @param text the number as text
+ * @param start the first index to inspect
+ * @param pValue OUT: the value of the integer. May be NULL
+ *
+ * @return <=0: no real number found
+ * otherwise: the length of the floating point number
+ */
+int RplQString::lengthOfReal(const QString& text, int start, qreal* pValue)
+{
+ int inputLength = text.size();
+ qreal value = 0.0;
+ int cc;
+ int ix = start;
+ while (ix < inputLength){
+ if ( (cc = text[ix].unicode()) >= '0' && cc <= '9')
+ value = value * 10 + (cc - '0');
+ else
+ break;
+ ix++;
+ }
+ // found: a digit has been found (in front of or behind the '.'
+ bool found = ix > start;
+ if (ix < inputLength && text[ix].unicode() == '.'){
+ ix++;
+ }
+ if (ix < inputLength && text[ix].isDigit()){
+ found = true;
+ qreal divisor = 1;
+ qreal precision = 0;
+ while ( ix < inputLength && (cc = text[ix].unicode()) >= '0' && cc <= '9'){
+ divisor *= 10;
+ precision = precision*10 + cc - '0';
+ ix++;
+ }
+ value += precision / divisor;
+ } else if (! found){
+ ix = start;
+ }
+ if (found && ix + 1 < inputLength && toupper(text[ix].unicode()) == 'E'){
+ int savePoint = ix;
+ ix++;
+ bool negative = false;
+ if ( (cc = text[ix].unicode()) == '+')
+ ix++;
+ else if (cc == '-'){
+ ix++;
+ negative = true;
+ }
+ if (ix >= inputLength || ! text[ix].isDigit())
+ ix = savePoint;
+ else{
+ int exponent = 0;
+ while (ix < inputLength && text[ix].isDigit()){
+ exponent = exponent * 10 + text[ix].unicode() - '0';
+ ix++;
+ }
+ if (negative)
+ value /= qPow(10, exponent);
+ else
+ value *= qPow(10, exponent);
+ }
+ }
+ if (pValue)
+ *pValue = value;
+ return found ? ix - start : 0;
+}
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+
+#ifndef RPLQSTRING_HPP
+#define RPLQSTRING_HPP
+
+class RplQString
+{
+public:
+ static int lengthOfUInt64(const QString& text, int start = 0,
+ int radix = 10, quint64* value = NULL);
+ static int lengthOfUInt(const QString& text, int start, int radix,
+ uint* pValue);
+ static int lengthOfReal(const QString& text, int start = 0,
+ qreal* value = NULL);
+ /**
+ * @brief Returns the value of a hexadecimal digit.
+ *
+ * @param digit a (unicode) character
+ * @return -1: not a hexadecimal digit<br>
+ * otherwise: the value, e.g. 10 for 'a'
+ */
+ inline static int valueOfHexDigit(int digit){
+ return digit >= '0' && digit <= '9' ? digit - '0'
+ : digit >= 'A' && digit <= 'F' ? digit - 'A' + 10
+ : digit >= 'a' && digit <= 'f' ? digit - 'a' + 10 : -1;
+ }
+};
+
+#endif // RPLQSTRING_HPP
return expected == current;
}
+/**
+ * Tests the equality of two values.
+ *
+ * Differences will be logged.
+ *
+ * @param expected the expected value
+ * @param current the current value
+ * @param file the file containing the test
+ * @param lineNo the line number containing the test
+ * @return true: equal
+ */
+bool RplTest::assertEquals(qint64 expected, qint64 current, const char* file,
+ int lineNo) {
+ if(expected != current)
+ error("%s-%d: error: %lld != %lld / %llx != %llx)", file, lineNo,
+ expected, current, (quint64) expected, (quint64) current);
+ return expected == current;
+}
+
+
/**
* Tests the equality of two values.
*
return expected == current;
}
+/**
+ * @brief Tests the equality of two values.
+ *
+ * Differences will be logged.
+ *
+ * @param expected the expected value
+ * @param current the current value
+ * @param file the file containing the test
+ * @param lineNo the line number containing the test
+ * @return true: equal
+ */
+bool RplTest::assertEquals(const char* expected, const QString& current,
+ const char* file, int lineNo) {
+ bool equal = assertEquals(expected, current.toUtf8().constData(), file,
+ lineNo);
+ return equal;
+}
+
+/**
+ * @brief Tests the equality of two values.
+ *
+ * Differences will be logged.
+ *
+ * @param expected the expected value
+ * @param current the current value
+ * @param file the file containing the test
+ * @param lineNo the line number containing the test
+ * @return true: equal
+ */
+bool RplTest::assertEquals(const QString& expected, const QString& current,
+ const char* file, int lineNo) {
+ bool equal = assertEquals(expected.toUtf8().constData(),
+ current.toUtf8().constData(), file, lineNo);
+ return equal;
+}
+
+
/**
* @brief Tests the equality of two values.
*
RplTest& operator =(const RplTest& source);
public:
bool assertEquals(int expected, int current, const char* file, int lineNo);
+ bool assertEquals(qint64 expected, qint64 current, const char* file, int lineNo);
bool assertEquals(qreal expected, qreal current, const char* file, int lineNo);
+ bool assertEquals(const char* expected, const QString& current,
+ const char* file, int lineNo);
+ bool assertEquals(const QString& expected, const QString& current,
+ const char* file, int lineNo);
bool assertEquals(const char* expected, const char* current,
const char* file, int lineNo);
bool assertEquals(const QByteArray& expected, const QByteArray& current,
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+
+#ifndef RPLEXPR_HPP
+#define RPLEXPR_HPP
+
+#include <QStack>
+#include <QRegularExpression>
+#include <QFile>
+#include <QTextStream>
+#include <QDir>
+#include <QtAlgorithms>
+
+#include "rplexpr/rplsource.hpp"
+#include "rplexpr/rpllexer.hpp"
+
+#endif // RPLEXPR_HPP
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+#include "rplcore/rplcore.hpp"
+#include "rplexpr/rplexpr.hpp"
+
+#define CHAR_INFO_SIZE (int(sizeof m_charInfo / sizeof m_charInfo[0]))
+/** @class RplToken rpllexer.hpp "rplexpr/rpllexer.hpp"
+ *
+ * @brief Implements a token which is the smallest unit for a parser.
+ *
+ */
+
+/**
+ * @brief RplLexException::RplLexException
+ * @param message
+ */
+/**
+ * @brief Constructor.
+ *
+ * @param position describes the position of the error/warning
+ * @param format the reason of the exception
+ * @param ... the values for the placeholders in the format.
+ */
+RplLexException::RplLexException(RplSourcePosition& position,
+ const char* format, ...) :
+ RplException("")
+{
+ char buffer[64000];
+ m_message = position.toString().toUtf8();
+ va_list ap;
+ va_start(ap, format);
+ vsnprintf(buffer, sizeof buffer, format, ap);
+ va_end(ap);
+ m_message += buffer;
+}
+
+/**
+ * @brief Constructor.
+ * @param type token type
+ */
+RplToken::RplToken(RplTokenType type) :
+ m_tokenType(type),
+ m_string(),
+ m_rawString()
+ // m_value
+{
+ memset(&m_value, 0, sizeof m_value);
+}
+
+/**
+ * @brief Destructor.
+ */
+RplToken::~RplToken()
+{
+}
+/**
+ * @brief Copy constructor.
+ *
+ * @param source source to copy
+ */
+RplToken::RplToken(const RplToken& source) :
+ m_tokenType(source.m_tokenType),
+ m_string(source.m_string),
+ m_rawString(source.m_rawString),
+ m_value(source.m_value)
+{
+}
+/**
+ * @brief Assignment operator.
+ *
+ * @param source source to copy
+ * @return
+ */
+RplToken& RplToken::operator =(const RplToken& source)
+{
+ m_tokenType = source.m_tokenType;
+ m_string = source.m_string;
+ m_value = source.m_value;
+ return *this;
+}
+
+/**
+ * @brief Returns the string representation of the instance
+ * @return a string representing the instance
+ */
+const QString& RplToken::toString()
+{
+ QString& rc = m_string;
+ switch(m_tokenType)
+ {
+ case TOKEN_STRING:
+ rc = m_rawString;
+ break;
+ case TOKEN_KEYWORD:
+ case TOKEN_NUMBER:
+ case TOKEN_OPERATOR:
+ case TOKEN_ID:
+ case TOKEN_COMMENT_REST_OF_LINE:
+ case TOKEN_COMMENT_START:
+ case TOKEN_COMMENT_END:
+ case TOKEN_SPACE:
+ default:
+ break;
+ }
+ return rc;
+}
+
+/**
+ * @brief Returns the integer value of the token
+ *
+ * Only relevant if a TOKEN_NUMBER.
+ *
+ * @return the value of the token as integer
+ */
+int RplToken::asInteger() const
+{
+ return (int) m_value.m_integer;
+}
+
+/**
+ * @brief Returns the integer value of the token
+ *
+ * Only relevant if a TOKEN_NUMBER.
+ *
+ * @return the value of the token as unsigned integer (64 bit)
+ */
+quint64 RplToken::asUInt64() const
+{
+ return m_value.m_integer;
+}
+
+/**
+ * @brief Returns the floating point value of the token
+ *
+ * Only relevant if a TOKEN_REAL.
+ *
+ * @return the value of the token as floating point value
+ */
+qreal RplToken::asReal() const
+{
+ return m_value.m_real;
+}
+
+/**
+ * @brief Returns the floating point value of the token
+ *
+ * Only relevant if a TOKEN_NUMBER.
+ *
+ * @return the value of the token as floating point value
+ */
+const QString& RplToken::rawString() const
+{
+ return m_rawString;
+}
+/**
+ * @brief Returns the id of the token.
+ *
+ * Ids are more handy than string, e.g. allowing switch statements.
+ *
+ * Only relevant for TOKEN_KEYWORD and TOKEN_OPERATOR.
+ *
+ * @return the id of the token
+ */
+int RplToken::id() const
+{
+ return m_value.m_id;
+}
+/**
+ * @brief Returns the token type.
+ * @return the token type
+ */
+RplTokenType RplToken::tokenType() const
+{
+ return m_tokenType;
+}
+
+/**
+ * @brief Makes all members undefined.
+ */
+void RplToken::clear()
+{
+ m_string.clear();
+ m_rawString.clear();
+ m_tokenType = TOKEN_UNDEF;
+ m_value.m_integer = 0;
+}
+
+
+/** @class RplLexer rpllexer.hpp "rplexpr/rpllexer.hpp"
+ *
+ * @brief Implements a lexical analyser.
+ *
+ * A lexical analyser reads a text source and separates the tokens for a parser.
+ * Tokens are the smallest elements of a parsing process.
+ *
+ */
+
+static void itemsToVector(const char* items, QVector<QString>& vector,
+ int firstCharFlag, int secondCharFlag,
+ int thirdCharFlag, int restCharFlag,
+ int charInfo[])
+{
+ QByteArray array2(items);
+ QList<QByteArray> list = array2.split(' ');
+ QList<QByteArray>::iterator it;
+ int id = 0;
+ for (it = list.begin(); it < list.end(); it++){
+ QByteArray& item2 = *it;
+ QString item(item2);
+ id++;
+ item += QChar(' ');
+ item += QChar(id);
+ vector.push_back(item);
+ unsigned char cc = item2.at(0);
+ if (cc < 128)
+ charInfo[cc] |= firstCharFlag;
+ if(item2.size() > 1){
+ cc = item2.at(1);
+ if (cc < 128)
+ charInfo[cc] |= secondCharFlag;
+ }
+ if(item2.size() > 2){
+ cc = item2.at(2);
+ if (cc < 128)
+ charInfo[cc] |= thirdCharFlag;
+ }
+ if(item2.size() > 3){
+ const char* ptr = item2.constData() + 3;
+ while( (cc = *ptr++) != '\0'){
+ if (cc < 128)
+ charInfo[cc] |= restCharFlag;
+ }
+ }
+ }
+ qSort(vector.begin(), vector.end(), qLess<QString>());
+}
+
+static void charClassToCharInfo(const char* charClass, int flag,
+ int charInfo[])
+{
+ for (int ix = 0; charClass[ix] != '\0'; ix++){
+ unsigned char cc = (unsigned char) charClass[ix];
+ if (cc < 128)
+ charInfo[cc] |= flag;
+ if (charClass[ix+1] == '-'){
+ unsigned char ubound = charClass[ix+2];
+ if (ubound == '\0')
+ charInfo['-'] |= flag;
+ else if (cc >= ubound)
+ throw new RplException("wrong character class range: %c-%c (%s)",
+ cc, ubound, charClass);
+ else {
+ for (int ii = cc + 1; ii <= ubound; ii++){
+ charInfo[ii] |= flag;
+ }
+ }
+ ix += 2;
+ }
+ }
+}
+
+/**
+ * @brief Constructor.
+ *
+ * @param source the input source handler
+ * @param keywords a string with all keywords delimited by ' '.
+ * Example: "if then else fi while do done"
+ * @param operators a string with all operators delimited by ' '
+ * @param comments a string with pairs of comment begin and end delimited
+ * by ' '. The comment end can be '\n' for line end.
+ * Example: "/ * * / // \n" (ignore the blank in "* /")
+ * @param firstCharsId string with the characters which are allowed as first
+ * characters of an id
+ * @param restCharsId string with the characters which are allowed as non
+ * first characters of an id
+ * @param numericTypes bit mask of allowed numeric,
+ * e.g. NUMTYPE_DECIMAL | NUMTYPE_HEXADECIMAL
+ * @param stringFeatures bit mask of the string features,
+ * e.g. SF_QUOTE | SF_TICK
+ * @param storageFlags describes the things which should be stored, e.g.
+ * S_ORG_STRINGS | S_COMMENTS | S_BLANKS
+ */
+RplLexer::RplLexer(RplSource* source,
+ const char* keywords, const char* operators, const char* comments,
+ const char* firstCharsId, const char* restCharsId, int numericTypes,
+ int stringFeatures, int storageFlags) :
+ m_source(source),
+ m_keywords(),
+ m_operators(),
+ m_commentStarts(),
+ m_commentEnds(),
+ //m_charInfo()
+ m_idFirstRare(),
+ m_idRestRare(),
+ m_numericTypes(numericTypes),
+ m_idRest2(),
+ m_currentToken(&m_token1),
+ m_waitingToken(NULL),
+ m_token1(TOKEN_UNDEF),
+ m_token2(TOKEN_UNDEF),
+ m_currentPosition(&m_position1),
+ m_position1(RplSourcePosition()),
+ m_position2(RplSourcePosition()),
+ m_maxTokenLength(64),
+ m_input(),
+ m_currentCol(0),
+ m_hasMoreInput(false),
+ m_stringFeatures(stringFeatures),
+ m_storageFlags(storageFlags)
+{
+ m_currentPosition->setSourceUnit(source->currentReader()
+ ->currentSourceUnit());
+ memset(m_charInfo, 0, sizeof m_charInfo);
+ itemsToVector(keywords, m_keywords, CC_FIRST_KEYWORD, CC_2nd_KEYWORD,
+ CC_3rd_KEYWORD, CC_REST_KEYWORD, m_charInfo);
+ prepareOperators(operators);
+ charClassToCharInfo(firstCharsId, CC_FIRST_ID, m_charInfo);
+ charClassToCharInfo(restCharsId, CC_REST_ID, m_charInfo);
+ initializeComments(comments);
+ m_input.reserve(m_maxTokenLength*2);
+}
+/**
+ * @brief Destructor.
+ */
+RplLexer::~RplLexer()
+{
+}
+/**
+ * @brief Stores the operators in the internal members
+ *
+ * @param operators a string with the operators separated by blank
+ */
+void RplLexer::prepareOperators(const char* operators){
+ itemsToVector(operators, m_operators, CC_FIRST_OP, CC_2nd_OP, CC_3rd_OP,
+ CC_REST_OP, m_charInfo);
+ // m_operators is now sorted:
+ // test whether the successor of 1 char operators is starting with this char:
+ // if not this operator will be marked with CC_OP_1_ONLY:
+ for (int ix = 0; ix < m_operators.size() - 1; ix++){
+ // the entries of m_operators end with ' ' and id:
+ if (m_operators.at(ix).size() == 1 + 2
+ && m_operators.at(ix).at(0) != m_operators.at(ix+1).at(0)){
+ int id = m_operators[ix].at(2).unicode();
+ m_charInfo[id] |= CC_OP_1_ONLY;
+ }
+ }
+}
+
+void RplLexer::initializeComments(const char* comments)
+{
+ if (comments != NULL)
+ {
+ QByteArray starters;
+ QByteArray comments2(comments);
+ int ix = comments2.indexOf(" ");
+ if (ix >= 0)
+ throw RplException("more than one blank between comment pair(s): col %d %s",
+ ix + 1, comments + ix);
+ // the index of m_commentEnds is the position number: we need a dummy entry:
+ m_commentEnds.append("");
+
+ QList<QByteArray> items = comments2.split(' ');
+ QList<QByteArray>::iterator it;
+ ix = 0;
+ for (it = items.begin(); it != items.end(); it++, ix++){
+ if (ix % 2 == 0){
+ if (ix > 0)
+ starters += " ";
+ starters += *it;
+ }else{
+ m_commentEnds.append(*it);
+ }
+ }
+ if (ix % 2 != 0)
+ throw RplException("not only pairs in the comment list");
+ itemsToVector(starters, m_commentStarts, CC_FIRST_COMMENT_START,
+ CC_2nd_COMMENT_START, CC_3rd_COMMENT_START, CC_REST_COMMENT_START,
+ m_charInfo);
+ }
+}
+/**
+ * @brief Searches the prefix of <code>m_input</code> in the vector.
+ *
+ * @param tokenLength the length of the prefix in <code>m_input</code>
+ * @param vector the vector to search. Each element contains the id
+ * as last entry
+ * @param id the id of the entry in the vector. Only set if found
+ * @return
+ */
+int RplLexer::findInVector(int tokenLength, const QVector<QString>& vector)
+{
+ int id = 0;
+ int lbound = 0;
+ int ubound = vector.size() - 1;
+ while(lbound <= ubound){
+ int half = (ubound + lbound) / 2;
+ int compareRc = 0;
+ int ix = 0;
+ const QString& current = vector[half];
+ // vector items end with ' ' and id:
+ int currentLength = current.size() - 2;
+ while(ix < tokenLength && compareRc == 0){
+ if (ix >= currentLength)
+ // current is shorter:
+ compareRc = 1;
+ else
+ compareRc = m_input.at(ix).unicode()
+ - (int) current.at(ix).unicode();
+ ix++;
+ }
+ if (compareRc == 0 && current.at(ix).unicode() != ' ')
+ // token.size() < current.size():
+ compareRc = -1;
+ if (compareRc < 0)
+ ubound = half - 1;
+ else if (compareRc > 0)
+ lbound = half + 1;
+ else {
+ id = current[currentLength + 1].unicode();
+ break;
+ }
+ }
+ return id;
+}
+
+void RplLexer::startUnit(const char* unit)
+{
+ //m_source->
+}
+const char* RplLexer::nextText(int& length, bool &isLast)
+{
+ return NULL;
+}
+/**
+ * @brief Reads data until enough data are available for one token.
+ *
+ * Data will be read from the current source unit.
+ * If this unit does not contain more data the next source unit from the stack
+ * will be used until the stack is empty.
+ *
+ * @return false: no more input is available<br>
+ * true: data are available
+ */
+bool RplLexer::fillInput()
+{
+ if (m_hasMoreInput){
+ if (m_input.size() < m_maxTokenLength){
+ m_source->currentReader()->fillBuffer(m_maxTokenLength, m_input,
+ m_hasMoreInput);
+ }
+ }
+
+ while (m_input.size() == 0 && m_source->currentReader() != NULL){
+ if (m_source->currentReader()->nextLine(m_maxTokenLength,
+ m_input, m_hasMoreInput)){
+ m_currentCol = 0;
+ }
+ }
+ return m_input.size() > 0;
+}
+
+/**
+ * @brief Finds a token with an id: TOKEN_OP, TOKEN_KEYWORD, ...
+ *
+ * @param tokenType the token type
+ * @param flag2 the flag of the 2nd char
+ * @param names the vector with the names, sorted
+ * @return NULL: not found<br>
+ * otherwise: the token
+ */
+RplToken* RplLexer::findTokenWithId(RplTokenType tokenType, int flag2,
+ QVector<QString>& names)
+{
+ int length = 1;
+ int inputLength = m_input.size();
+ int cc;
+ if (inputLength > 1){
+ cc = m_input[1].unicode();
+ if (cc < CHAR_INFO_SIZE && (m_charInfo[cc] & flag2)){
+ length++;
+ if (inputLength > 2){
+ cc = m_input[2].unicode();
+ // the 3rd char flag is the "successor" of the 2nd char flag:
+ int flag = (flag2 << 1);
+ if (cc < CHAR_INFO_SIZE && (m_charInfo[cc] & flag)){
+ length++;
+ // the rest char flag is the "successor" of the 3nd char flag:
+ flag <<= 1;
+ while (length < inputLength){
+ cc = m_input[length].unicode();
+ if (cc < CHAR_INFO_SIZE && (m_charInfo[cc] & flag))
+ length++;
+ else
+ break;
+ }
+ }
+ }
+ }
+ }
+ RplToken* rc = NULL;
+ if (! (tokenType == TOKEN_KEYWORD && length < inputLength
+ && (cc = m_input[length].unicode()) < CHAR_INFO_SIZE
+ && (m_charInfo[cc] & CC_REST_ID))) {
+ int id;
+ // the length could be too long: the CC_2nd_.. flag could be ambigous
+ while(length >= 1 && (id = findInVector(length, names)) <= 0){
+ length--;
+ }
+ if (id > 0){
+ rc = m_currentToken;
+ rc->m_tokenType = tokenType;
+ rc->m_value.m_id = id;
+ m_input.remove(0, length);
+ m_currentCol += length;
+ }
+ }
+ return rc;
+}
+
+/**
+ * @brief Converts the number into a token.
+ *
+ * @return the token with the number
+ */
+RplToken* RplLexer::scanNumber()
+{
+ int inputLength = m_input.size();
+ int cc;
+ int length;
+ quint64 value = 0;
+ if ( (cc = m_input[0].unicode()) == '0' && inputLength > 1
+ && (m_numericTypes & NUMTYPE_HEXADECIMAL)
+ && (m_input[1].unicode() == 'x' || m_input[1].unicode() == 'X')){
+ length = RplQString::lengthOfUInt64(m_input, 2, 16, &value);
+ if (length > 0)
+ length += 2;
+ else
+ throw RplException("invalid hexadecimal number: no digit behind 'x");
+ } else if (cc == '0' && (m_numericTypes & NUMTYPE_OCTAL)
+ && inputLength > 1){
+ length = 1;
+ while (length < inputLength){
+ if ( (cc = m_input[length].unicode()) >= '0' && cc <= '7')
+ value = value * 8 + cc - '0';
+ else if (cc >= '8' && cc <= '9')
+ throw RplLexException(*m_currentPosition,
+ "invalid octal digit: %c", cc);
+ else
+ break;
+ length++;
+ }
+ } else {
+ length = 1;
+ value = cc - '0';
+ while (length < inputLength){
+ if ( (cc = m_input[length].unicode()) >= '0' && cc <= '9')
+ value = value * 10 + cc - '0';
+ else
+ break;
+ length++;
+ }
+ }
+ m_currentToken->m_value.m_integer = value;
+ m_currentToken->m_tokenType = TOKEN_NUMBER;
+ if (length + 1 < inputLength
+ && ((cc = m_input[length].unicode()) == '.' || toupper(cc) == 'E')){
+ qreal realValue;
+ int realLength = RplQString::lengthOfReal(m_input, 0, &realValue);
+ if (realLength > length){
+ m_currentToken->m_tokenType = TOKEN_REAL;
+ m_currentToken->m_value.m_real = realValue;
+ length = realLength;
+ }
+ }
+ m_input.remove(0, length);
+ m_currentCol += length;
+ return m_currentToken;
+}
+
+/**
+ * @brief Reads a string into the internal data.
+ * @return the token with the string
+ */
+RplToken*RplLexer::scanString()
+{
+ int delim = m_input[0].unicode();
+ int inputLength = m_input.size();
+ int cc;
+ int length = 1;
+ m_currentToken->m_string.append(m_input[0]);
+ bool again = false;
+ do {
+ while(length < inputLength && (cc = m_input[length].unicode()) != delim){
+ length++;
+ if (cc != '\\'
+ || (m_stringFeatures
+ & (SF_C_ESCAPING | SF_C_HEX_CHARS | SF_C_SPECIAL)) == 0){
+ m_currentToken->m_rawString.append(QChar(cc));
+ } else {
+ if (length >= inputLength)
+ throw RplLexException(*m_currentPosition,
+ "backslash without following character");
+ cc = m_input[length++].unicode();
+ if ( (m_stringFeatures & SF_C_HEX_CHARS) && toupper(cc) == 'X'){
+ if (length >= inputLength)
+ throw RplLexException(*m_currentPosition,
+ "missing hexadecimal digit behind \\x");
+ cc = m_input[length++].unicode();
+ int hexVal = RplQString::valueOfHexDigit(cc);
+ if (hexVal < 0)
+ throw RplLexException(*m_currentPosition,
+ "not a hexadecimal digit behind \\x: %lc",
+ QChar(cc));
+ if (length < inputLength){
+ cc = m_input[length].unicode();
+ int nibble = RplQString::valueOfHexDigit(cc);
+ if (nibble >= 0){
+ length++;
+ hexVal = hexVal * 16 + nibble;
+ }
+ }
+ m_currentToken->m_rawString.append(QChar(hexVal));
+ } else if ( (m_stringFeatures & SF_C_SPECIAL)){
+ switch(cc){
+ case 'r':
+ cc = '\r';
+ break;
+ case 'n':
+ cc = '\n';
+ break;
+ case 't':
+ cc = '\t';
+ break;
+ case 'a':
+ cc = '\a';
+ break;
+ case 'v':
+ cc = '\v';
+ break;
+ default:
+ break;
+ }
+ m_currentToken->m_rawString.append(QChar(cc));
+ } else {
+ m_currentToken->m_rawString.append(QChar(cc));
+ }
+ }
+ }
+ if (cc == delim){
+ length++;
+ }
+ if ( (m_stringFeatures & SF_DOUBLE_DELIM) && length < inputLength
+ && m_input[length].unicode() == delim){
+ m_currentToken->m_rawString.append(delim);
+ length++;
+ again = true;
+ }
+ }
+ while(again);
+ if (m_storageFlags & STORE_ORG_STRING)
+ m_currentToken->m_string.append(m_input.mid(0, length));
+ m_input.remove(0, length);
+ m_currentCol += length;
+ return m_currentToken;
+}
+
+/**
+ * @brief Reads a comment into the internal data.
+ *
+ * precondition: the current token is prepared yet
+ */
+void RplLexer::scanComment()
+{
+ int inputLength = m_input.size();
+ int cc;
+ int length = 1;
+ QString commentEnd = m_commentEnds[m_currentToken->id()];
+ int ix;
+ if (commentEnd[0].unicode() == '\n'){
+ // single line comment:
+ if (m_storageFlags & STORE_COMMENT)
+ m_currentToken->m_string.append(m_input);
+ length = inputLength;
+ } else {
+ // multiline comment:
+ while( (ix = m_input.indexOf(commentEnd)) < 0){
+ if (m_storageFlags & STORE_COMMENT)
+ m_currentToken->m_string.append(m_input);
+ if (! fillInput())
+ throw RplLexException(*m_currentPosition,
+ "comment end not found");
+ }
+ if (m_storageFlags & STORE_COMMENT)
+ m_currentToken->m_rawString.append(m_input.mid(0, ix));
+ length = ix + commentEnd.size();
+ }
+ m_input.remove(0, length);
+ m_currentCol += length;
+}
+
+/**
+ * @brief Returns the next token.
+ *
+ * @return the next token
+ */
+RplToken* RplLexer::nextToken()
+{
+ RplToken* rc = NULL;
+ int ix;
+ if (m_waitingToken != NULL){
+ rc = m_waitingToken;
+ m_waitingToken = NULL;
+ m_currentPosition = m_currentPosition == &m_position1
+ ? &m_position2 : &m_position1;
+ } else {
+ m_currentToken->clear();
+ m_currentPosition->setLineNo(m_source->currentReader()
+ ->currentSourceUnit()->lineNo());
+ m_currentPosition->setColumn(m_currentCol);
+ if (! fillInput()){
+ m_currentToken->m_tokenType = TOKEN_END_OF_SOURCE;
+ } else {
+ QChar cc = m_input.at(0);
+ if (cc.isSpace()){
+ m_currentToken->m_tokenType = TOKEN_SPACE;
+ ix = 1;
+ while(ix < m_input.size() && m_input.at(ix).isSpace())
+ ix++;
+ if (m_storageFlags & STORE_BLANK){
+ m_currentToken->m_string.append(m_input.mid(0, ix));
+ }
+ m_currentCol += ix;
+ m_input.remove(0, ix);
+ rc = m_currentToken;
+ } else if (cc.isDigit()){
+ rc = scanNumber();
+ } else if (cc == '"' || cc == '\''){
+ rc = scanString();
+ } else {
+ int cc2 = cc.unicode();
+ if (cc2 >= CHAR_INFO_SIZE)
+ throw RplLexException(*m_currentPosition,
+ "no lexical symbol can start with this char: %ls",
+ QChar(cc2));
+ else
+ {
+ if (rc == NULL && (m_charInfo[cc2] & CC_FIRST_COMMENT_START)){
+ rc = findTokenWithId(TOKEN_COMMENT_START,
+ CC_2nd_COMMENT_START, m_commentStarts);
+ if (rc != NULL)
+ scanComment();
+ }
+
+ if (rc == NULL && (m_charInfo[cc2] & CC_FIRST_OP)){
+ if ( (m_charInfo[cc2] & CC_OP_1_ONLY) == 0){
+ rc = findTokenWithId(TOKEN_OPERATOR,
+ CC_2nd_OP, m_operators);
+ } else {
+ rc = m_currentToken;
+ rc->m_tokenType = TOKEN_OPERATOR;
+ rc->m_value.m_id = findInVector(1, m_operators);
+ m_input.remove(0, 1);
+ m_currentCol += 1;
+ }
+ }
+ if (rc == NULL && (m_charInfo[cc2] & CC_FIRST_KEYWORD)){
+ rc = findTokenWithId(TOKEN_KEYWORD,
+ CC_2nd_KEYWORD, m_keywords);
+ }
+
+ }
+ }
+ }
+ }
+ if (rc == NULL || rc->tokenType() == TOKEN_UNDEF){
+ if (m_input.size() == 0){
+ rc = m_currentToken;
+ rc->m_tokenType = TOKEN_END_OF_SOURCE;
+ } else {
+ QString symbol = m_input.mid(0, qMin(20, m_input.size()-1));
+ throw RplLexException(*m_currentPosition,
+ "unknown lexical symbol: %s", symbol.toUtf8().constData());
+ }
+ }
+ return rc;
+}
+/**
+ * @brief Returns the maximal length of a token
+ * @return the maximal length of a token
+ */
+size_t RplLexer::maxTokenLength() const
+{
+ return m_maxTokenLength;
+}
+
+/**
+ * @brief Sets the maximal length of a token
+ *
+ * @param maxTokenLength the new maximal length of a token
+ */
+void RplLexer::setMaxTokenLength(size_t maxTokenLength)
+{
+ m_maxTokenLength = maxTokenLength;
+}
+
+/**
+ * @brief RplLexer::nextNonSpaceToken
+ * @return
+ */
+RplToken* RplLexer::nextNonSpaceToken()
+{
+ RplToken* rc = NULL;
+ RplTokenType type;
+ do{
+ rc = nextToken();
+ } while( (type = m_currentToken->tokenType()) == TOKEN_SPACE
+ || type == TOKEN_COMMENT_START || type == TOKEN_COMMENT_END
+ || type == TOKEN_COMMENT_REST_OF_LINE);
+ return rc;
+}
+
+/**
+ * @brief Starts a new source unit.
+ *
+ * Saves the current source position onto the top of stack.
+ * Pushes the source unit onto the top of stack.
+ *
+ * @param unit
+ */
+void RplLexer::startUnit(const QString& unit)
+{
+ // m_source->startUnit(unit, new RplSourcePosition(
+ // m_source->currentReader()));
+}
+/**
+ * @brief Returns the source of the instance.
+ *
+ * @return the source of the instance
+ */
+RplSource*RplLexer::source()
+{
+ return m_source;
+}
+
+
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+
+#ifndef RPLLEXER_HPP
+#define RPLLEXER_HPP
+
+
+enum RplTokenType {
+ TOKEN_UNDEF,
+ TOKEN_STRING,
+ TOKEN_NUMBER,
+ TOKEN_REAL,
+ TOKEN_KEYWORD,
+ TOKEN_OPERATOR,
+ TOKEN_ID,
+ TOKEN_COMMENT_REST_OF_LINE,
+ TOKEN_COMMENT_START,
+ TOKEN_COMMENT_END,
+ TOKEN_SPACE,
+ TOKEN_END_OF_SOURCE,
+ TOKEN_COUNT
+};
+
+class RplLexException : public RplException {
+public:
+ RplLexException(RplSourcePosition& position, const char* message, ...);
+};
+
+class RplLexer;
+
+class RplToken {
+public:
+ RplToken(RplTokenType type);
+ ~RplToken();
+ RplToken(const RplToken& source);
+ RplToken& operator =(const RplToken& source);
+public:
+ friend class RplLexer;
+ const QString& toString();
+ bool isInteger();
+ int asInteger() const;
+ quint64 asUInt64() const;
+ qreal asReal() const;
+ const QString& rawString() const;
+ int id() const;
+ RplTokenType tokenType() const;
+ void clear();
+protected:
+ RplTokenType m_tokenType;
+ QString m_string;
+ // only for TOKEN_STRING: copy from source but without escaped chars like "\\n"
+ QString m_rawString;
+ union {
+ // only for TOKEN_KEYWORD and TOKEN_OPERATOR
+ int m_id;
+ quint64 m_integer;
+ qreal m_real;
+ } m_value;
+};
+
+
+class RplSource;
+class RplLexer
+{
+public:
+ enum NumericType {
+ NUMTYPE_UNDEF,
+ NUMTYPE_DECIMAL = 1 << 0,
+ NUMTYPE_OCTAL = 1 << 1,
+ NUMTYPE_HEXADECIMAL = 1 << 2,
+ NUMTYPE_FLOAT = 1 << 3,
+ ///
+ NUMTYPE_ALL_INTEGER = NUMTYPE_DECIMAL | NUMTYPE_OCTAL |NUMTYPE_HEXADECIMAL,
+ NUMTYPE_ALL = NUMTYPE_ALL_INTEGER | NUMTYPE_FLOAT
+ };
+ enum CharClassTag {
+ CC_UNDEF = 0,
+ /// this char is possible as first char of an id
+ CC_FIRST_ID = 1 << 0,
+ /// this char is possible as 2nd char of an id
+ CC_2nd_ID = 1 << 1,
+ /// this char is possible as 3rd char of an id
+ CC_3rd_ID = 1 << 2,
+ /// this char is possible as 4th... char of an id
+ CC_REST_ID = 1 << 3,
+ /// this char can start a comment
+ CC_FIRST_COMMENT_START = 1 << 4,
+ /// this char can be the 2nd char of a comment start
+ CC_2nd_COMMENT_START = 1 << 5,
+ /// this char can be the 3rd char of a comment start
+ CC_3rd_COMMENT_START = 1 << 6,
+ /// this char can be the 4th ... of a comment start
+ CC_REST_COMMENT_START = 1 << 7,
+ /// this char can start a keyword
+ CC_FIRST_KEYWORD = 1 << 8,
+ /// this char can be the 2nd char of a keyword
+ CC_2nd_KEYWORD = 1 << 9,
+ /// this char can be the 3rd char of a keyword
+ CC_3rd_KEYWORD = 1 << 10,
+ /// this char can be the 4th... char of a keyword
+ CC_REST_KEYWORD = 1 << 11,
+ /// this char can be the 1st char of an operator
+ CC_FIRST_OP = 1 << 12,
+ /// this char can be the 2nd char of an operator
+ CC_2nd_OP = 1 << 13,
+ /// this char can be the 3rd char of an operator
+ CC_3rd_OP = 1 << 14,
+ /// this char can be the 4th... char of an operator
+ CC_REST_OP = 1 << 15,
+ /// there is an operator with exactly this char
+ /// and there is no other operator starting with this char
+ CC_OP_1_ONLY = 1 << 16
+ };
+ enum StringFeatures {
+ SF_UNDEF,
+ /// ' can be a string delimiter
+ SF_TICK = 1 << 1,
+ /// " can be a string delimiter
+ SF_QUOTE = 1 << 2,
+ /// character escaping like in C: \<char> is <char>
+ SF_C_ESCAPING = 1 << 3,
+ /// special characters like in C: \r \f \r \t
+ SF_C_SPECIAL = 1 << 4,
+ /// characters can be written in hexadecimal notation: \x20 is ' '
+ SF_C_HEX_CHARS = 1 << 5,
+ /// A delimiter inside a string must be doubled (like in Pascal)
+ SF_DOUBLE_DELIM = 1 << 6
+ };
+ enum StorageFlags {
+ S_UNDEF,
+ /// the original string will be stored in m_string too
+ /// (not only m_rawString)
+ STORE_ORG_STRING = 1 << 1,
+ /// comments will be stored in m_string
+ STORE_COMMENT = 1 << 2,
+ /// blanks will be stored in m_string
+ STORE_BLANK = 1 << 3,
+ /// redefinitions for better reading:
+ STORE_NOTHING = 0,
+ STORE_ALL = STORE_ORG_STRING | STORE_COMMENT | STORE_BLANK
+ };
+
+public:
+ RplLexer(RplSource* source,
+ const char* keywords, const char* operators,
+ const char* comments,
+ const char* firstCharsId = "a-zA-Z_",
+ const char* restCharsId = "a-zA-Z0-9_",
+ int numericTypes = NUMTYPE_DECIMAL | NUMTYPE_HEXADECIMAL | NUMTYPE_FLOAT,
+ int stringFeatures = SF_TICK | SF_QUOTE | SF_C_ESCAPING | SF_C_SPECIAL
+ | SF_C_HEX_CHARS,
+ int storageFlags = STORE_NOTHING);
+ virtual ~RplLexer();
+public:
+ void startUnit(const char* unit);
+ RplToken* nextToken();
+ RplToken* peekNonSpaceToken();
+ RplToken* nextNonSpaceToken();
+ size_t maxTokenLength() const;
+ void setMaxTokenLength(size_t maxTokenLength);
+ void startUnit(const QString& unit);
+ RplSource* source();
+private:
+ void prepareOperators(const char* operators);
+ void initializeComments(const char* comments);
+ bool fillInput();
+ int findInVector(int tokenLength, const QVector<QString>& vector);
+ const char* nextText(int& length, bool &isLast);
+ RplToken* findTokenWithId(RplTokenType tokenType, int flag2,
+ QVector<QString>& names);
+ RplToken* scanNumber();
+ RplToken* scanString();
+ void scanComment();
+protected:
+ RplSource* m_source;
+ /// sorted, string ends with the id of the keyword
+ QVector<QString> m_keywords;
+ // sorted, string ends with the id of the operator
+ QVector<QString> m_operators;
+ // sorted, string ends with the id of the comment start
+ QVector<QString> m_commentStarts;
+ // index: id content: comment_end
+ QVector<QString> m_commentEnds;
+ // index: ord(char) content: a sum of CharClassTags
+ int m_charInfo[128];
+ // a list of QChars with ord(cc) > 127 and which can be the first char
+ QString m_idFirstRare;
+ // index: ord(char) content: chr(ix) can be the non first char of an id
+ QString m_idRestRare;
+ // a list of QChars with ord(cc) > 127 and which can be the first char
+ int m_numericTypes;
+ QString m_idRest2;
+ RplToken* m_currentToken;
+ RplToken* m_waitingToken;
+ RplToken m_token1;
+ RplToken m_token2;
+ RplSourcePosition* m_currentPosition;
+ RplSourcePosition m_position1;
+ RplSourcePosition m_position2;
+ int m_maxTokenLength;
+ QString m_input;
+ int m_currentCol;
+ bool m_hasMoreInput;
+ int m_stringFeatures;
+ int m_storageFlags;
+};
+
+
+#endif // RPLLEXER_HPP
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+#include "rplcore/rplcore.hpp"
+#include "rplexpr/rplexpr.hpp"
+
+
+/** @class RplSource rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Stores all source positions.
+ *
+ */
+
+/** @class RplSourceUnit rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Implements the base class of input source units.
+ *
+ * A source unit is set of input lines with a name, e.g. a file.
+ */
+
+/**
+ * @brief Constructor
+ * @param name name of the unit
+ * @param reader the reader which can read the unit
+ */
+RplSourceUnit::RplSourceUnit(const QString& name, RplReader* reader) :
+ m_name(name),
+ m_lineNo(0),
+ m_reader(reader) {
+}
+
+/**
+ * @brief Destructor.
+ */
+RplSourceUnit::~RplSourceUnit() {
+}
+
+/**
+ * @brief Returns the name.
+ * @return the name
+ */
+const QString& RplSourceUnit::name() const {
+ return m_name;
+}
+
+/**
+ * @brief Returns the line number.
+ *
+ * @return the line number
+ */
+int RplSourceUnit::lineNo() const {
+ return m_lineNo;
+}
+
+/**
+ * @brief Sets the line number.
+ *
+ * @param lineNo the new line number
+ */
+void RplSourceUnit::setLineNo(int lineNo) {
+ m_lineNo = lineNo;
+}
+/**
+ * @brief Returns the reader of the instance.
+ *
+ * @return the reader belonging to the instance
+ */
+RplReader* RplSourceUnit::reader() const {
+ return m_reader;
+}
+
+/** @class RplSourcePosition rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Stores a precise position in the input source.
+ *
+ * The input source contains normally a lot of nested source units.
+ *
+ * Each source unit owns a name and a sequence of lines.
+ *
+ * The mostly used source unit is a text file.
+ *
+ * A precice position is the stack of source unit positions.
+ */
+
+/**
+ * @brief Constructor.
+ */
+RplSourcePosition::RplSourcePosition() :
+ m_sourceUnit(NULL),
+ m_lineNo(0),
+ m_column(0),
+ m_caller(NULL)
+{
+}
+
+/**
+ * @brief Constructor.
+ *
+ * @param unit name of the input source (normally a file)
+ * @param lineNo line number inside the input source
+ * @param colNo distance to the line start
+ * @param source parent: the source
+ */
+RplSourcePosition::RplSourcePosition(RplSourceUnit* unit, int lineNo,
+ int colNo) :
+ m_sourceUnit(unit),
+ m_lineNo(lineNo),
+ m_column(colNo),
+ m_caller(unit->reader()->source().sourcePositionStack().top())
+{
+}
+/**
+ * @brief Destructor
+ */
+RplSourcePosition::~ RplSourcePosition(){
+}
+
+/**
+ * @brief Copy constructor.
+ *
+ * @param source instance to copy
+ */
+RplSourcePosition::RplSourcePosition(const RplSourcePosition& source) :
+ m_sourceUnit(source.m_sourceUnit),
+ m_lineNo(source.m_lineNo),
+ m_column(source.m_column),
+ m_caller(source.m_caller)
+{
+}
+
+/**
+ * @brief Assignment operator.
+ *
+ * @param source instance to copy
+ * @return the instance itself
+ */
+RplSourcePosition&RplSourcePosition::operator=(const RplSourcePosition& source)
+{
+ m_sourceUnit = source.m_sourceUnit;
+ m_lineNo = source.m_lineNo;
+ m_column = source.m_column;
+ m_caller = source.m_caller;
+ return *this;
+}
+/**
+ * @brief Returns a description of the source position: "<unit>-<lineNo> (<col>):".
+ *
+ * @return a description of the instance
+ */
+QString RplSourcePosition::toString() const
+{
+ QString rc = m_sourceUnit->name();
+ QTextStream stream(&rc);
+ stream << "-" << m_lineNo << " (" << m_column << "): ";
+ return rc;
+}
+
+/**
+ * @brief Sets the line number.
+ *
+ * @param lineNo the new lineNo
+ */
+void RplSourcePosition::setLineNo(int lineNo)
+{
+ m_lineNo = lineNo;
+}
+/**
+ * @brief Returns the column.
+ *
+ * @return the column of instance.
+ */
+int RplSourcePosition::column() const
+{
+ return m_column;
+}
+
+/**
+ * @brief Sets the column.
+ *
+ * @param column the new column
+ */
+void RplSourcePosition::setColumn(int column)
+{
+ m_column = column;
+}
+
+/**
+ * @brief Returns the source unit belonging to the instance.
+ *
+ * @return the source unit of the instance
+ */
+RplSourceUnit* RplSourcePosition::sourceUnit() const
+{
+ return m_sourceUnit;
+}
+
+/**
+ * @brief Sets the source unit.
+ *
+ * @param sourceUnit the new source unit of the instance
+ */
+void RplSourcePosition::setSourceUnit(RplSourceUnit* sourceUnit)
+{
+ m_sourceUnit = sourceUnit;
+}
+
+
+/** @class RplReader rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Implements a base class for readers of different media.
+ */
+
+
+/**
+ * @brief Constructor.
+ *
+ * @param source the parent
+ */
+RplReader::RplReader(RplSource& source) :
+ m_currentSourceUnit(NULL),
+ m_units(),
+ m_source(source) {
+}
+
+/**
+ * @brief Destructor.
+ */
+RplReader::~RplReader() {
+ QMap<QString, RplSourceUnit*>::iterator it;
+ for(it = m_units.begin(); it != m_units.end(); it++) {
+ RplStringSourceUnit* unit = (RplStringSourceUnit*)(*it);
+ delete unit;
+ }
+}
+
+/**
+ * @brief Returns the source of the reader.
+ *
+ * @return the parent, a source instance
+ */
+RplSource&RplReader::source()
+{
+ return m_source;
+}
+/**
+ * @brief Returns the current source unit.
+ *
+ * @return the source unit
+ */
+RplSourceUnit* RplReader::currentSourceUnit() const {
+ return m_currentSourceUnit;
+}
+
+/**
+ * @brief Sets the current source unit.
+ *
+ * @param sourceUnit the name of the new source unit
+ */
+bool RplReader::setCurrentSourceUnit(const QString& sourceUnit) {
+ bool rc = m_units.contains(sourceUnit);
+ if(rc) {
+ m_currentSourceUnit = m_units.value(sourceUnit);
+ m_source.pushSourceUnit(m_currentSourceUnit);
+ }
+ return rc;
+}
+
+/**
+ * @brief Removes the "latest" sourceUnit.
+ */
+void RplReader::removeSourceUnit() {
+
+ m_currentSourceUnit = m_source.popSourceUnit(this);;
+}
+
+
+/** @class RplSource rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Administrates a set of input sources with different readers.
+ *
+ * An input stream can be built by some different resources, e.g. files
+ * and memory buffers. The RplSource can administrate these resources.
+ */
+/**
+ * @brief Constructor.
+ */
+RplSource::RplSource() :
+ m_sourcePositionStack(),
+ m_sourcePositions(),
+ m_readers(),
+ m_sourceUnits(),
+ m_unitStack(),
+ m_currentReader(NULL)
+{
+ // the stack should never be empty:
+ m_sourcePositionStack.push(NULL);
+}
+
+/**
+ * @brief Destructor.
+ */
+RplSource::~RplSource() {
+}
+
+/**
+ * @brief Returns the permanently valid source unit name.
+ *
+ * If unit names can be local objects (not string constants
+ * like __FILE__) then this method must be overridden by
+ * a method which builds a permanently valid string.
+ *
+ * @param unit unit to find
+ * @return a permanently valid unit name
+ */
+const char* RplSource::permanentUnitName(const char* unit) {
+ return unit;
+}
+
+/**
+ * @brief Returns the stack with the positions of the open input resources.
+ *
+ * @return the stack
+ */
+QStack<const RplSourcePosition*> RplSource::sourcePositionStack() const {
+ return m_sourcePositionStack;
+}
+
+/**
+ * @brief Returns the source unit stack.
+ * @return the stack of the source units
+ */
+QStack<RplSourceUnit*>& RplSource::sourceUnitStack()
+{
+ return m_unitStack;
+}
+
+/**
+ * @brief Adds a source reader.
+ *
+ * @param reader the new reader. Will be freed in the destructor
+ */
+void RplSource::addReader(RplReader* reader) {
+ m_readers.push_back(reader);
+ if (m_currentReader == NULL)
+ m_currentReader = reader;
+}
+
+/**
+ * @brief Adds a source unit.
+ *
+ * @param unit the new unit. Will be freed in the destructor
+ */
+void RplSource::addSourceUnit(RplSourceUnit* unit) {
+ m_sourceUnits.push_back(unit);
+}
+
+/**
+ * @brief Starts a new source unit.
+ *
+ * Saves the current source position onto the top of stack.
+ * Pushes the source unit onto the top of stack.
+ *
+ * @param unit
+ */
+bool RplSource::startUnit(const QString& unit,
+ const RplSourcePosition* sourcePosition) {
+ m_sourcePositionStack.push_back(sourcePosition);
+ RplReader* reader = NULL;
+ QList<RplReader*>::iterator it;
+ for(it = m_readers.begin();
+ reader == NULL && it != m_readers.end();
+ it++) {
+ RplReader* current = *it;
+ if(current->openSourceUnit(unit)) {
+ reader = current;
+ }
+ }
+ return reader != NULL;
+}
+
+/**
+ * @brief Pushes a source unit onto the stack.
+ *
+ * @param unit the source unit
+ */
+void RplSource::pushSourceUnit(RplSourceUnit* unit) {
+ m_unitStack.push(unit);
+}
+
+/**
+ * @brief Removes the latest source unit from the stack.
+ *
+ * @param reader the current reader
+ * @return NULL: the current reader does not have an open source unit<br>
+ * otherwise: the last entry from the source unit stack
+ */
+RplSourceUnit* RplSource::popSourceUnit(RplReader* reader) {
+ if(m_unitStack.size() > 0)
+ m_unitStack.pop();
+ m_currentReader = m_unitStack.size() <= 0
+ ? NULL : m_unitStack.top()->reader();
+ RplSourceUnit* rc = NULL;
+ if(m_currentReader == reader)
+ rc = m_unitStack.top();
+ else {
+ for(int ix = m_unitStack.size() - 2; ix >= 0; ix--){
+ if(m_unitStack[ix]->reader() == reader){
+ rc = m_unitStack[ix];
+ break;
+ }
+ }
+ }
+ return rc;
+}
+
+/**
+ * @brief Returns the reader of the current source unit.
+ *
+ * @return NULL: no reader active<br>
+ * otherwise: the current reader
+ */
+RplReader* RplSource::currentReader() {
+ return m_currentReader;
+}
+
+/** @class RplStringSourceUnit rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Stores the state of a string based source unit.
+ *
+ */
+
+/**
+ * @brief Constructor.
+ *
+ * @param name name of the unit
+ * @param content content of the unit
+ * @param reader the parent
+ */
+RplStringSourceUnit::RplStringSourceUnit(const QString& name,
+ const QString& content, RplStringReader* reader) :
+ RplSourceUnit(name, reader),
+ m_currentPosition(0),
+ m_content(content) {
+}
+
+/**
+ * @brief Destructor.
+ */
+RplStringSourceUnit::~RplStringSourceUnit() {
+}
+/**
+ * @brief Returns the current read position.
+ *
+ * @return the offset (count of QChars) of the end of the last read block
+ * inside m_content
+ */
+int RplStringSourceUnit::currentPosition() const {
+ return m_currentPosition;
+}
+
+/**
+ * @brief Sets the current read position.
+ *
+ * @param currentPosition the offset (count of QChars) of the end of
+ * the last read block inside <code>m_content</code>
+ */
+void RplStringSourceUnit::setCurrentPosition(int currentPosition) {
+ m_currentPosition = currentPosition;
+}
+/**
+ * @brief Returns the content of the source unit.
+ *
+ * @return the content
+ */
+const QString& RplStringSourceUnit::content() const {
+ return m_content;
+}
+
+/** @class RplStringReader rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Implements a source which provides reading from memory based buffers.
+ *
+ * Examples for usage: interactive input
+ */
+
+/**
+ * @brief Constructor.
+ *
+ * @param source the parent
+ */
+RplStringReader::RplStringReader(RplSource& source) :
+ RplReader(source) {
+}
+
+/**
+ * @brief Destructor.
+ */
+RplStringReader::~RplStringReader() {
+}
+
+/**
+ * @brief Opens a new source unit.
+ *
+ * @param unit name of the source
+ * @return NULL: unknown source<br>
+ * otherwise: an instance of a sub class of
+ * <code>RplSourceUnit</code>
+ */
+RplSourceUnit* RplStringReader::openSourceUnit(const QString& unit) {
+ RplSourceUnit* rc = NULL;
+ if(setCurrentSourceUnit(unit)) {
+ rc = m_currentSourceUnit;
+ ((RplStringSourceUnit*) rc)->setCurrentPosition(0);
+ }
+ return rc;
+}
+/**
+ * @brief Delivers the next line from the input medium or the first part of it.
+ *
+ * @param maxSize if the line length is longer than this value, only the
+ * first part of the line will be returned
+ * @param buffer OUT: the line will be put here
+ * @param hasMore true: the line was longer than <code>maxSize</code>, only
+ * the first part of the line is put into the <code>buffer</code>
+ * @return false: no more input available<br>
+ * true: success
+ */
+bool RplStringReader::nextLine(int maxSize, QString& buffer,
+ bool& hasMore) {
+ m_currentSourceUnit->setLineNo(m_currentSourceUnit->lineNo() + 1);
+ bool rc = fillBuffer(maxSize, buffer, hasMore);
+ return rc;
+}
+
+/**
+ * @brief Delivers the next part of a long line.
+ *
+ * @param maxSize if the remaining line length is longer than this value,
+ * only the a part of the line will be returned
+ * @param buffer OUT: the part of line will be put here
+ * @param hasMore true: the line was longer than <code>maxSize</code>, only
+ * the first part of the line is put into the <code>buffer</code>
+ * @return false: no more input available<br>
+ * true: success
+ */
+bool RplStringReader::fillBuffer(int maxSize, QString& buffer,
+ bool& hasMore) {
+ RplStringSourceUnit* unit = (RplStringSourceUnit*) m_currentSourceUnit;
+ int start = unit->currentPosition();
+ const QString& content = unit->content();
+ int end = content.indexOf("\n", start);
+ hasMore = false;
+ if(end < 0) {
+ end = content.size() - 1;
+ }
+ int size = end - start + 1;
+ hasMore = false;
+ if(size > maxSize) {
+ size = maxSize;
+ hasMore = true;
+ }
+ if(size > 0) {
+ buffer += content.mid(start, size);
+ unit->setCurrentPosition(start + size);
+ } else {
+ removeSourceUnit();
+ }
+ return size > 0;
+}
+
+/**
+ * @brief Adds a source buffer to the reader
+ *
+ * @param name name of the medium
+ * @param content
+ */
+void RplStringReader::addSource(const QString& name,
+ const QString& content) {
+ // Deletion in the destructor of the base class RplReader
+ RplStringSourceUnit* unit = new RplStringSourceUnit(name, content, this);
+ m_units.insert(m_units.begin(), name, unit);
+ m_currentSourceUnit = unit;
+}
+
+/** @class RplFileSourceUnit rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Stores the state of a file based source unit.
+ *
+ * This is the mostly used implementation of the RplSourceUnit/RplReader.
+ */
+
+/**
+ * @brief Constructor.
+ *
+ * @param name name of the unit, the filename
+ * @param reader the parent
+ */
+RplFileSourceUnit::RplFileSourceUnit(const QDir& dir,
+ RplFileReader* reader) :
+ RplSourceUnit(dir.absolutePath(), reader),
+ m_currentPosition(0),
+ m_file(dir.absolutePath()),
+ m_textStream(&m_file),
+ m_line() {
+}
+
+/**
+ * @brief Destructor.
+ */
+RplFileSourceUnit::~RplFileSourceUnit() {
+}
+/** @class RplFileReader rplsource.hpp "rplexpr/rplsource.hpp"
+ *
+ * @brief Implements a source which provides reading from memory based buffers.
+ *
+ * Examples for usage: interactive input
+ */
+
+/**
+ * @brief Constructor.
+ */
+RplFileReader::RplFileReader(RplSource& source) :
+ RplReader(source) {
+}
+
+/**
+ * @brief Destructor.
+ */
+RplFileReader::~RplFileReader() {
+}
+
+/**
+ * @brief Opens a new source unit.
+ *
+ * @param unit name of the source
+ * @return NULL: unknown source<br>
+ * otherwise: an instance of a sub class of
+ * <code>RplSourceUnit</code>
+ */
+RplSourceUnit* RplFileReader::openSourceUnit(const QString& unit) {
+ RplSourceUnit* rc = NULL;
+ if(m_units.contains(unit)) {
+ rc = *m_units.find(unit);
+ m_currentUnit = (RplFileSourceUnit*) rc;
+ }
+ return rc;
+}
+/**
+ * @brief Delivers the next line from the input medium or the first part of it.
+ *
+ * @param maxSize if the line length is longer than this value, only the
+ * first part of the line will be returned
+ * @param buffer OUT: the line will be put here
+ * @param hasMore true: the line was longer than <code>maxSize</code>, only
+ * the first part of the line is put into the <code>buffer</code>
+ * @return false: no more input available<br>
+ * true: success
+ */
+bool RplFileReader::nextLine(int maxSize, QString& buffer,
+ bool& hasMore) {
+ RplFileSourceUnit* unit = (RplFileSourceUnit*) m_currentUnit;
+ bool rc = ! unit->m_textStream.atEnd();
+ if(rc) {
+ unit->m_line = unit->m_textStream.readLine();
+ rc = fillBuffer(maxSize, buffer, hasMore);
+ }
+ return rc;
+}
+
+/**
+ * @brief Delivers the next part of a long line.
+ *
+ * @param maxSize if the remaining line length is longer than this value,
+ * only the a part of the line will be returned
+ * @param buffer OUT: the part of line will be put here
+ * @param hasMore true: the line was longer than <code>maxSize</code>, only
+ * the first part of the line is put into the <code>buffer</code>
+ * @return false: no more input available<br>
+ * true: success
+ */
+bool RplFileReader::fillBuffer(int maxSize, QString& buffer,
+ bool& hasMore) {
+ RplFileSourceUnit* unit = (RplFileSourceUnit*) m_currentUnit;
+ int start = unit->m_currentPosition;
+ const QString& content = unit->m_line;
+ int size = content.size() - start;
+ if(size > maxSize)
+ size = maxSize;
+ buffer += content.mid(start, size);
+ unit->m_currentPosition = (start += size);
+ hasMore = start < content.length();
+ return size > 0;
+}
+
+/**
+ * @brief Adds a source file to the reader
+ *
+ * @param dirEntry the file
+ */
+void RplFileReader::addSource(const QDir& dirEntry) {
+ QString name = dirEntry.absolutePath();
+ // Deleting in ~RplSourceUnit():
+ RplFileSourceUnit* unit = new RplFileSourceUnit(dirEntry, this);
+ m_units.insert(m_units.begin(), name, unit);
+}
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+
+#ifndef RPLSOURCE_HPP
+#define RPLSOURCE_HPP
+
+class RplSource;
+class RplReader;
+
+class RplSourceUnit {
+public:
+ RplSourceUnit(const QString& name, RplReader* reader);
+ virtual ~RplSourceUnit();
+public:
+ const QString& name() const;
+ int lineNo() const;
+ void setLineNo(int lineNo);
+ RplReader* reader() const;
+
+protected:
+ QString m_name;
+ int m_lineNo;
+ RplReader* m_reader;
+};
+
+class RplSourcePosition {
+public:
+ RplSourcePosition();
+ RplSourcePosition(RplSourceUnit* unit, int lineNo, int colNo);
+ ~ RplSourcePosition();
+ RplSourcePosition(const RplSourcePosition& source);
+ RplSourcePosition& operator=(const RplSourcePosition& source);
+public:
+ QString toString() const;
+ int lineNo() const;
+ void setLineNo(int lineNo);
+
+ int column() const;
+ void setColumn(int column);
+
+ RplSourceUnit* sourceUnit() const;
+ void setSourceUnit(RplSourceUnit* sourceUnit);
+
+private:
+ RplSourceUnit* m_sourceUnit;
+ int m_lineNo;
+ int m_column;
+ const RplSourcePosition* m_caller;
+};
+
+class RplReader {
+public:
+ RplReader(RplSource& source);
+ ~RplReader();
+public:
+ /**
+ * @brief Prepares the reading from a given source unit.
+ *
+ * @param unit name of the unit
+ * @return NULL: unit not known<br>
+ * otherwise: an instance with the state of the reader
+ * for the source. This is normally a sub class of
+ * <code>RplSourceUnit</code>
+ */
+ virtual RplSourceUnit* openSourceUnit(const QString& unit) = 0;
+ /**
+ * @brief Reads the first part of the next line into a given buffer.
+ *
+ * @param maxSize the maximum length of the read input.
+ * If a line is longer the next part must be read
+ * by <code>fillBuffer()</code>
+ * @param buffer IN/OUT: the read input will be appended here
+ * @param hasMore OUT: true: the line is longer than maxSize
+ * @return true: the read was successful<br>
+ * false: no more input is available
+ */
+ virtual bool nextLine(int maxSize, QString& buffer, bool& hasMore) = 0;
+ /**
+ * @brief Reads the next part of the current line into a given buffer.
+ *
+ * @param maxSize the maximum length of the read input.
+ * @param buffer IN/OUT: the read input will be appended here
+ * @param hasMore OUT: true: the rest of line is longer than maxSize
+ * @return true: the read was successful<br>
+ * false: no more input is available
+ */
+ virtual bool fillBuffer(int maxSize, QString& buffer, bool& hasMore) = 0;
+public:
+ RplSource& source();
+ RplSourceUnit* currentSourceUnit() const;
+ bool setCurrentSourceUnit(const QString& currentSourceUnit);
+protected:
+ void removeSourceUnit();
+
+protected:
+ RplSourceUnit* m_currentSourceUnit;
+ /// name -> source
+ QMap<QString, RplSourceUnit*> m_units;
+ RplSource& m_source;
+};
+
+
+class RplSource {
+public:
+ RplSource();
+ virtual ~RplSource();
+public:
+ virtual const char* permanentUnitName(const char* unit);
+ void finishSourceUnit();
+ void addReader(RplReader* reader);
+ void addSourceUnit(RplSourceUnit* unit);
+ QStack<const RplSourcePosition*> sourcePositionStack() const;
+ QStack<RplSourceUnit*>& sourceUnitStack();
+
+ bool startUnit(const QString& unit, const RplSourcePosition*);
+ void pushSourceUnit(RplSourceUnit* unit);
+ RplSourceUnit* popSourceUnit(RplReader* reader);
+ RplReader* currentReader();
+protected:
+ // stack of the info about the stacked (open) source units:
+ QStack<const RplSourcePosition*> m_sourcePositionStack;
+ QList<const RplSourcePosition> m_sourcePositions;
+ QList<RplReader*> m_readers;
+ QList<RplSourceUnit*> m_sourceUnits;
+ // setCurrentSourceUnit() pushes one entry, removeSourceUnit() pops it
+ // (when end of input has been reached).
+ QStack<RplSourceUnit*> m_unitStack;
+ RplReader* m_currentReader;
+};
+
+class RplStringReader;
+
+class RplStringSourceUnit : public RplSourceUnit {
+ friend class RplStringReader;
+public:
+ RplStringSourceUnit(const QString& name,
+ const QString& content, RplStringReader* reader);
+ virtual ~RplStringSourceUnit();
+public:
+ int currentPosition() const;
+ void setCurrentPosition(int currentPosition);
+ const QString& content() const;
+
+private:
+ int m_currentPosition;
+ QString m_content;
+};
+
+class RplStringReader : public RplReader{
+public:
+ RplStringReader(RplSource& source);
+ virtual ~RplStringReader();
+ // RplReader interface
+public:
+ virtual RplSourceUnit* openSourceUnit(const QString& unit);
+ virtual bool nextLine(int maxSize, QString& buffer, bool& hasMore);
+ virtual bool fillBuffer(int maxSize, QString& buffer, bool& hasMore);
+public:
+ void addSource(const QString& name, const QString& content);
+};
+
+class RplFileReader;
+
+class RplFileSourceUnit : public RplSourceUnit {
+ friend class RplFileReader;
+public:
+ RplFileSourceUnit(const QDir& dir, RplFileReader* reader);
+ virtual ~RplFileSourceUnit();
+private:
+ int m_currentPosition;
+ QFile m_file;
+ QTextStream m_textStream;
+ QString m_line;
+};
+
+class RplFileReader : public RplReader{
+public:
+ RplFileReader(RplSource& source);
+ virtual ~RplFileReader();
+ // RplReader interface
+public:
+ virtual RplSourceUnit* openSourceUnit(const QString& unit);
+ virtual bool nextLine(int maxSize, QString& buffer, bool& hasMore);
+ virtual bool fillBuffer(int maxSize, QString& buffer, bool& hasMore);
+public:
+ void addSource(const QDir& dirEntry);
+private:
+ /// name -> source
+ QMap<QString, RplFileSourceUnit*> m_units;
+ RplFileSourceUnit* m_currentUnit;
+};
+
+
+#endif // RPLSOURCE_HPP
../rplnet/rpltcpclient.cpp \
../rplnet/rpltcppeer.cpp \
../rplnet/rpltcpserver.cpp \
+ ../rplexpr/rpllexer.cpp \
+ ../rplexpr/rplsource.cpp \
+ ../rplcore/rplqstring.cpp
HEADERS += ../rplmodules.hpp \
../rplcore/rplconfig.hpp \
../rplnet/rpltcpclient.hpp \
../rplnet/rpltcppeer.hpp \
../rplnet/rpltcpserver.hpp \
+ ../rplexpr/rpllexer.hpp \
+ ../rplexpr/rplexpr.hpp \
+ ../rplexpr/rplsource.hpp \
+ ../rplcore/rplqstring.hpp
unix:!symbian {
maemo5 {
#include <QCoreApplication>
+void testCore(){
+ extern void testRplQString();
+ testRplQString();
+
+ extern void testRplString();
+ testRplString();
+
+ extern void testRplException();
+ testRplException();
+}
+
+void testExpr(){
+ extern void testRplSource();
+ testRplSource();
+
+}
+
+void testStandard(){
+ extern void testRplMatrix();
+ testRplMatrix();
+
+ testExpr();
+ testCore();
+}
+
int main(int argc, char *argv[])
{
if (argc > 1)
printf("not used: %s\n", argv[1]);
- extern void testRplMatrix();
- testRplMatrix();
+ extern void testRplLexer();
+ testRplLexer();
- extern void testRplString();
- testRplString();
+ testStandard();
- extern void testRplException();
- testRplException();
}
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+#include "rplcore/rplcore.hpp"
+#include "rplexpr/rplexpr.hpp"
+#include "rplcore/rpltest.hpp"
+
+class TestRplLexer : public RplTest, public RplToken{
+public:
+ TestRplLexer() :
+ RplTest("RplLexer"),
+ RplToken(TOKEN_ID)
+ {}
+
+public:
+ void testRplToken(){
+ // test constructor values:
+ checkE(TOKEN_ID, tokenType());
+ checkE(0, m_value.m_id);
+ checkT(m_string.isEmpty());
+ checkT(m_rawString.isEmpty());
+
+ m_value.m_id = 7422;
+ checkE(7422, RplToken::id());
+ m_string = "Wow!";
+ checkE("Wow!", RplToken::toString());
+ m_rawString = "GooGoo";
+ checkE("GooGoo", rawString());
+ m_tokenType = TOKEN_NUMBER;
+ checkE(TOKEN_NUMBER, tokenType());
+
+ clear();
+ checkE(TOKEN_UNDEF, tokenType());
+ checkE(0, m_value.m_id);
+ checkT(m_string.isEmpty());
+ checkT(m_rawString.isEmpty());
+
+ m_value.m_integer = 773322;
+ checkE(773322, asInteger());
+ m_value.m_real = 0.25;
+ checkE(0.25, asReal());
+ }
+
+ RplToken* checkToken(RplToken* token, RplTokenType type, int id = 0,
+ const char* string = NULL){
+ checkE(type, token->tokenType());
+ if (id != 0)
+ checkE(id, token->id());
+ if (string != NULL)
+ checkE(string, token->toString());
+ return token;
+ }
+ enum { KEY_UNDEF, KEY_IF, KEY_THEN, KEY_ELSE, KEY_FI
+ };
+# define KEYWORDS "if then else fi"
+ enum { OP_UNDEF, OP_PLUS, OP_TIMES, OP_DIV, OP_ASSIGN, OP_GT,
+ OP_LT, OP_GE, OP_LE, OP_EQ
+ };
+# define OPERATORS "+ * / = > < >= <= =="
+ enum { COMMENT_UNDEF, COMMENT_1
+ };
+# define COMMENTS "# \n"
+ void testSpace(){
+ RplSource source;
+ RplStringReader reader(source);
+# define BLANKS1 "\t\t \n"
+# define BLANKS2 " \n"
+ reader.addSource("<main>", BLANKS1 BLANKS2);
+ source.addReader(&reader);
+ RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS,
+ "A-Za-z_",
+ "A-Za-z0-9_",
+ RplLexer::NUMTYPE_DECIMAL,
+ RplLexer::SF_TICK, RplLexer::STORE_ALL);
+ checkToken(lex.nextToken(), TOKEN_SPACE, 0, BLANKS1);
+ checkToken(lex.nextToken(), TOKEN_SPACE, 0, BLANKS2);
+ }
+ void testNumeric(){
+ RplSource source;
+ RplStringReader reader(source);
+ const char* blanks = "321 0x73 7.8e+5";
+ reader.addSource("<main>", blanks);
+ source.addReader(&reader);
+ RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS,
+ "A-Za-z_",
+ "A-Za-z0-9_",
+ RplLexer::NUMTYPE_ALL,
+ RplLexer::SF_TICK, RplLexer::STORE_ALL);
+ RplToken* token = checkToken(lex.nextToken(), TOKEN_NUMBER);
+ checkE(321, token->asInteger());
+ token = checkToken(lex.nextNonSpaceToken(), TOKEN_NUMBER);
+ checkE(0x73, token->asInteger());
+ token = checkToken(lex.nextNonSpaceToken(), TOKEN_REAL);
+ checkE(7.8e+5, token->asReal());
+ }
+
+ void testOperators(){
+ RplSource source;
+ RplStringReader reader(source);
+ const char* ops = "<< < <<< <= == = ( ) [ ]";
+ reader.addSource("<main>", ops);
+ source.addReader(&reader);
+ enum { UNDEF, SHIFT, LT, SHIFT2, LE, EQ, ASSIGN,
+ LPARENT, RPARENT, LBRACKET, RBRACKET };
+ RplLexer lex(&source, KEYWORDS, ops, COMMENTS,
+ "A-Za-z_",
+ "A-Za-z0-9_",
+ RplLexer::NUMTYPE_ALL,
+ RplLexer::SF_TICK, RplLexer::STORE_ALL);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, SHIFT);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LT);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, SHIFT2);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LE);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, EQ);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, ASSIGN);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LPARENT);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, RPARENT);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LBRACKET);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, RBRACKET);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_END_OF_SOURCE);
+ reader.addSource("<buffer2>", "(([[");
+ lex.startUnit("<buffer2>");
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LPARENT);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LPARENT);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LBRACKET);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LBRACKET);
+ checkToken(lex.nextNonSpaceToken(), TOKEN_END_OF_SOURCE);
+ }
+
+ void testBasic(){
+ RplSource source;
+ RplStringReader reader(source);
+ reader.addSource("<main>", "if i>1 then i=1+2*_x9 fi");
+ RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS,
+ "A-Za-z_",
+ "A-Za-z0-9_"
+ );
+ checkToken(lex.nextToken(), TOKEN_KEYWORD, KEY_IF);
+ checkToken(lex.nextToken(), TOKEN_SPACE, 0);
+ checkToken(lex.nextToken(), TOKEN_ID, 0, "i");
+ checkToken(lex.nextToken(), TOKEN_OPERATOR, OP_GT);
+ checkToken(lex.nextToken(), TOKEN_NUMBER, 0, "1");
+ checkToken(lex.nextToken(), TOKEN_SPACE, 0);
+ checkToken(lex.nextToken(), TOKEN_KEYWORD, KEY_THEN);
+ checkToken(lex.nextToken(), TOKEN_SPACE, 0);
+
+ }
+
+ virtual void doIt(void) {
+ testOperators();
+ testNumeric();
+ testSpace();
+ testBasic();
+ testRplToken();
+ }
+};
+void testRplLexer() {
+ TestRplLexer test;
+ test.run();
+}
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+#include "rplcore/rplcore.hpp"
+#include "rplcore/rpltest.hpp"
+
+class TestRplQString : public RplTest {
+public:
+ TestRplQString() :
+ RplTest("RplQString")
+ {}
+
+public:
+ void testLengthOfUInt64(){
+ quint64 value = -3;
+ checkE(1, RplQString::lengthOfUInt64(QString("0"), 0, 10, &value));
+ checkE(0LL, value);
+ checkE(3, RplQString::lengthOfUInt64("x432", 1, 10, &value));
+ checkE(432LL, value);
+ checkE(3, RplQString::lengthOfUInt64("x432 x", 1, 10, &value));
+ checkE(432LL, value);
+ checkE(3, RplQString::lengthOfUInt64("x432fabc x", 1, 10, &value));
+ checkE(432LL, value);
+ checkE(16, RplQString::lengthOfUInt64("a1234567890123567", 1, 10, &value));
+ checkE(1234567890123567LL, value);
+ checkE(10, RplQString::lengthOfUInt64("x1234abcdef", 1, 16, &value));
+ checkE(0x1234abcdefLL, value);
+ checkE(3, RplQString::lengthOfUInt64("432", 0, 8, &value));
+ checkE(0432LL, value);
+ checkE(6, RplQString::lengthOfUInt64(" 765432 ", 1, 8, &value));
+ checkE(0765432LL, value);
+
+ checkE(0, RplQString::lengthOfUInt64("1 ", 1, 8, &value));
+ checkE(0, RplQString::lengthOfUInt64("", 1, 8, &value));
+ }
+ void testLengthOfUInt(){
+ uint value = 3;
+ checkE(1, RplQString::lengthOfUInt(QString("0"), 0, 10, &value));
+ checkE(0, value);
+ checkE(3, RplQString::lengthOfUInt("x432", 1, 10, &value));
+ checkE(432, value);
+ checkE(3, RplQString::lengthOfUInt("x432 x", 1, 10, &value));
+ checkE(432, value);
+ checkE(3, RplQString::lengthOfUInt("x432fabc x", 1, 10, &value));
+ checkE(432, value);
+ checkE(3, RplQString::lengthOfUInt("432", 0, 8, &value));
+ checkE(0432, value);
+ checkE(6, RplQString::lengthOfUInt(" 765432 ", 1, 8, &value));
+ checkE(0765432, value);
+
+ checkE(0, RplQString::lengthOfUInt("1 ", 1, 8, &value));
+ checkE(0, RplQString::lengthOfUInt("", 1, 8, &value));
+ }
+ void testLengthOfReal(){
+ qreal value;
+ checkE(4, RplQString::lengthOfReal(QString("0.25"), 0, &value));
+ checkE(0.25, value);
+ checkE(3, RplQString::lengthOfReal(QString("X.25"), 1, &value));
+ checkE(0.25, value);
+ checkE(1, RplQString::lengthOfReal(QString(" 0"), 1, &value));
+ checkE(0.0, value);
+ checkE(17, RplQString::lengthOfReal(QString("X12345678901234567"), 1, &value));
+ checkE(12345678901234567.0, value);
+ checkE(2, RplQString::lengthOfReal(QString(".5"), 0, &value));
+ checkE(0.5, value);
+ checkE(5, RplQString::lengthOfReal(QString("2.5e2x"), 0, &value));
+ checkE(250.0, value);
+ checkE(6, RplQString::lengthOfReal(QString("2.5e+2"), 0, &value));
+ checkE(250.0, value);
+ checkE(7, RplQString::lengthOfReal(QString("2.5E-33"), 0, &value));
+ checkE(2.5e-33, value);
+
+ checkE(3, RplQString::lengthOfReal(QString("2.5E"), 0, &value));
+ checkE(2.5, value);
+ checkE(3, RplQString::lengthOfReal(QString("2.5E+"), 0, &value));
+ checkE(2.5, value);
+ checkE(3, RplQString::lengthOfReal(QString("2.5E-a"), 0, &value));
+ checkE(2.5, value);
+ }
+
+ void testValueOfHexDigit(){
+ checkE(0, RplQString::valueOfHexDigit('0'));
+ checkE(9, RplQString::valueOfHexDigit('9'));
+ checkE(10, RplQString::valueOfHexDigit('a'));
+ checkE(15, RplQString::valueOfHexDigit('f'));
+ checkE(10, RplQString::valueOfHexDigit('A'));
+ checkE(15, RplQString::valueOfHexDigit('F'));
+
+ checkE(-1, RplQString::valueOfHexDigit('0' - 1));
+ checkE(-1, RplQString::valueOfHexDigit('9' + 1));
+ checkE(-1, RplQString::valueOfHexDigit('A' - 1));
+ checkE(-1, RplQString::valueOfHexDigit('F' + 1));
+ checkE(-1, RplQString::valueOfHexDigit('a' - 1));
+ checkE(-1, RplQString::valueOfHexDigit('f' + 1));
+ }
+
+ virtual void doIt(void) {
+ testLengthOfUInt64();
+ testLengthOfUInt();
+ testLengthOfReal();
+ testValueOfHexDigit();
+ }
+};
+void testRplQString() {
+ TestRplQString test;
+ test.run();
+}
+
--- /dev/null
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+#include "rplcore/rplcore.hpp"
+#include "rplexpr/rplexpr.hpp"
+#include "rplcore/rpltest.hpp"
+
+class TestRplSource : public RplTest{
+public:
+ TestRplSource() : RplTest("TestRplSource") {}
+
+private:
+ QString m_content1_1;
+ QString m_content1_2;
+ QString m_content2;
+ RplSource m_source;
+
+protected:
+ void init(){
+ m_content1_1 = "# test\nimport source2\n";
+ m_content1_2 = "a=1;\nveeeeeeeeery looooooooooooooong\n";
+ m_content2 = "x=2";
+ }
+
+ void testRplStringSourceUnit(){
+ RplStringReader reader(m_source);
+ QString content("a=1;\nveeeeeeeeery looooooooooooooong\n");
+ RplStringSourceUnit unit(QString("test"), content, &reader);
+ unit.setLineNo(144);
+ checkE(144, unit.lineNo());
+ checkE("test", unit.name());
+ }
+ void checkOne(int maxSize, RplReader& reader){
+ QString total;
+ QString buffer;
+ int lineNo = 0;
+ bool hasMore;
+ checkF(reader.openSourceUnit("unknownSource"));
+ checkT(reader.openSourceUnit("source1"));
+ while(reader.nextLine(maxSize, buffer, hasMore)){
+ lineNo++;
+ total += buffer;
+ buffer.clear();
+ while(hasMore && reader.fillBuffer(maxSize, buffer, hasMore)){
+ total += buffer;
+ buffer.clear();
+ }
+ bool isImport = total.endsWith("source2\n");
+ if (isImport){
+ reader.openSourceUnit("source2");
+ checkE("source2", reader.currentSourceUnit()->name());
+ while(reader.nextLine(maxSize, buffer, hasMore)){
+ lineNo++;
+ while(hasMore && reader.fillBuffer(maxSize, buffer, hasMore)){
+ total += buffer;
+ buffer.clear();
+ }
+ }
+ checkE("source1", reader.currentSourceUnit()->name());
+ }
+ }
+ checkE(5, lineNo);
+ checkE(m_content1_1 + m_content2 + m_content1_2, total);
+
+ }
+
+ void testRplStringReader(){
+ RplStringReader reader(m_source);
+ reader.addSource("source1", m_content1_1 + m_content1_2);
+ reader.addSource("source2", m_content2);
+ RplSourceUnit* unit = reader.openSourceUnit("source1");
+ checkE("source1", unit->name());
+ checkE(0, unit->lineNo());
+ checkOne(6, reader);
+ checkOne(100, reader);
+ }
+
+public:
+ virtual void doIt(void) {
+ init();
+ testRplStringSourceUnit();
+ testRplStringReader();
+ }
+};
+void testRplSource() {
+ TestRplSource test;
+ test.run();
+}
+
+
+
SOURCES += main.cpp \
rplmatrix_test.cpp \
- ../rplmath/rplmatrix.cpp \
../rplcore/rpllogger.cpp \
../rplcore/rpltest.cpp \
../rplcore/rplstring.cpp \
../rplcore/rplexception.cpp \
+ ../rplmath/rplmatrix.cpp \
+ ../rplexpr/rplsource.cpp \
+ ../rplexpr/rpllexer.cpp \
rplexception_test.cpp \
- rplstring_test.cpp
+ rplstring_test.cpp \
+ rplsource_test.cpp \
+ rpllexer_test.cpp \
+ rplqstring_test.cpp \
+ ../rplcore/rplqstring.cpp
projects / reqt / commitdiff