From df881a655a51aa17291097be1792ed36174c12af Mon Sep 17 00:00:00 2001 From: hama Date: Wed, 11 Jun 2014 01:15:35 +0200 Subject: [PATCH] dayly work --- rplcore/rplcore.hpp | 2 + rplcore/rplqstring.cpp | 155 +++++++ rplcore/rplqstring.hpp | 36 ++ rplcore/rpltest.cpp | 57 +++ rplcore/rpltest.hpp | 5 + rplexpr/rplexpr.hpp | 23 + rplexpr/rpllexer.cpp | 851 ++++++++++++++++++++++++++++++++++ rplexpr/rpllexer.hpp | 215 +++++++++ rplexpr/rplsource.cpp | 709 ++++++++++++++++++++++++++++ rplexpr/rplsource.hpp | 200 ++++++++ rplstatic/rplstatic.pro | 7 + unittests/main.cpp | 34 +- unittests/rpllexer_test.cpp | 165 +++++++ unittests/rplqstring_test.cpp | 112 +++++ unittests/rplsource_test.cpp | 96 ++++ unittests/unittests.pro | 10 +- 16 files changed, 2669 insertions(+), 8 deletions(-) create mode 100644 rplcore/rplqstring.cpp create mode 100644 rplcore/rplqstring.hpp create mode 100644 rplexpr/rplexpr.hpp create mode 100644 rplexpr/rpllexer.cpp create mode 100644 rplexpr/rpllexer.hpp create mode 100644 rplexpr/rplsource.cpp create mode 100644 rplexpr/rplsource.hpp create mode 100644 unittests/rpllexer_test.cpp create mode 100644 unittests/rplqstring_test.cpp create mode 100644 unittests/rplsource_test.cpp diff --git a/rplcore/rplcore.hpp b/rplcore/rplcore.hpp index f630fd7..515e442 100644 --- a/rplcore/rplcore.hpp +++ b/rplcore/rplcore.hpp @@ -28,6 +28,7 @@ #include #include #include +#include typedef unsigned char uint8_t; #include "rplmodules.hpp" @@ -35,6 +36,7 @@ typedef unsigned char uint8_t; #include "rplcore/rplexception.hpp" #include "rplcore/rplcontainer.hpp" #include "rplcore/rplstring.hpp" +#include "rplcore/rplqstring.hpp" #include "rplcore/rplconfigurator.hpp" #include "rplcore/rplconfig.hpp" #include "rplcore/rplterminator.hpp" diff --git a/rplcore/rplqstring.cpp b/rplcore/rplqstring.cpp new file mode 100644 index 0000000..f685ddb --- /dev/null +++ b/rplcore/rplqstring.cpp @@ -0,0 +1,155 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + + +#include "rplcore/rplcore.hpp" + + +/** + * @brief Determines the length and vlaue of an integer. + * + * @param text the number as text + * @param start the first index to inspect + * @param radix the base of the number sytem: 8 (octal), 10 or 16 + * @param pValue OUT: the value of the integer. May be NULL + * + * @return <=0: no integer found + * otherwise: the length of the integer + */ +int RplQString::lengthOfUInt64(const QString& text, int start, + int radix, quint64* pValue) +{ + int inputLength = text.size(); + qint64 value = 0; + int ix = start; + int cc; + if (radix == 10){ + while (ix < inputLength){ + if ( (cc = text[ix].unicode()) >= '0' && cc <= '9') + value = value * 10 + cc - '0'; + else + break; + ix++; + } + } else if (radix == 16){ + while (ix < inputLength){ + if ( (cc = text[ix].unicode()) >= '0' && cc <= '9') + value = value * 16 + cc - '0'; + else if (cc >= 'A' && cc <= 'F') + value = value * 16 + cc - 'A' + 10; + else if (cc >= 'a' && cc <= 'f') + value = value * 16 + cc - 'a' + 10; + else + break; + ix++; + } + } else if (radix == 8){ + while (ix < inputLength){ + if ( (cc = text[ix].unicode()) >= '0' && cc <= '7') + value = value * 8 + cc - '0'; + else + break; + ix++; + } + } else { + throw RplException("RplQString::lengthOfInt(): wrong radix: %d", radix); + } + if (pValue != NULL) + *pValue = value; + return ix - start; +} +/** + * @brief Determines the length and value of an unsigned integer. + * + * @param text the number as text + * @param start the first index to inspect + * @param radix the base of the number sytem: 8 (octal), 10 or 16 + * @param pValue OUT: the value of the integer. May be NULL + * + * @return 0: no integer found + * otherwise: the length of the integer + */ +int RplQString::lengthOfUInt(const QString& text, int start, + int radix, uint* pValue) +{ + quint64 value; + int rc = lengthOfUInt64(text, start, radix, &value); + if (pValue != NULL) + *pValue = (uint) value; + return rc; +} + +/** + * @brief Determines the length and value of a floting point number. + * + * @param text the number as text + * @param start the first index to inspect + * @param pValue OUT: the value of the integer. May be NULL + * + * @return <=0: no real number found + * otherwise: the length of the floating point number + */ +int RplQString::lengthOfReal(const QString& text, int start, qreal* pValue) +{ + int inputLength = text.size(); + qreal value = 0.0; + int cc; + int ix = start; + while (ix < inputLength){ + if ( (cc = text[ix].unicode()) >= '0' && cc <= '9') + value = value * 10 + (cc - '0'); + else + break; + ix++; + } + // found: a digit has been found (in front of or behind the '.' + bool found = ix > start; + if (ix < inputLength && text[ix].unicode() == '.'){ + ix++; + } + if (ix < inputLength && text[ix].isDigit()){ + found = true; + qreal divisor = 1; + qreal precision = 0; + while ( ix < inputLength && (cc = text[ix].unicode()) >= '0' && cc <= '9'){ + divisor *= 10; + precision = precision*10 + cc - '0'; + ix++; + } + value += precision / divisor; + } else if (! found){ + ix = start; + } + if (found && ix + 1 < inputLength && toupper(text[ix].unicode()) == 'E'){ + int savePoint = ix; + ix++; + bool negative = false; + if ( (cc = text[ix].unicode()) == '+') + ix++; + else if (cc == '-'){ + ix++; + negative = true; + } + if (ix >= inputLength || ! text[ix].isDigit()) + ix = savePoint; + else{ + int exponent = 0; + while (ix < inputLength && text[ix].isDigit()){ + exponent = exponent * 10 + text[ix].unicode() - '0'; + ix++; + } + if (negative) + value /= qPow(10, exponent); + else + value *= qPow(10, exponent); + } + } + if (pValue) + *pValue = value; + return found ? ix - start : 0; +} diff --git a/rplcore/rplqstring.hpp b/rplcore/rplqstring.hpp new file mode 100644 index 0000000..53c8d41 --- /dev/null +++ b/rplcore/rplqstring.hpp @@ -0,0 +1,36 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + + +#ifndef RPLQSTRING_HPP +#define RPLQSTRING_HPP + +class RplQString +{ +public: + static int lengthOfUInt64(const QString& text, int start = 0, + int radix = 10, quint64* value = NULL); + static int lengthOfUInt(const QString& text, int start, int radix, + uint* pValue); + static int lengthOfReal(const QString& text, int start = 0, + qreal* value = NULL); + /** + * @brief Returns the value of a hexadecimal digit. + * + * @param digit a (unicode) character + * @return -1: not a hexadecimal digit
+ * otherwise: the value, e.g. 10 for 'a' + */ + inline static int valueOfHexDigit(int digit){ + return digit >= '0' && digit <= '9' ? digit - '0' + : digit >= 'A' && digit <= 'F' ? digit - 'A' + 10 + : digit >= 'a' && digit <= 'f' ? digit - 'a' + 10 : -1; + } +}; + +#endif // RPLQSTRING_HPP diff --git a/rplcore/rpltest.cpp b/rplcore/rpltest.cpp index 1b9adfd..7cce846 100644 --- a/rplcore/rpltest.cpp +++ b/rplcore/rpltest.cpp @@ -80,6 +80,26 @@ bool RplTest::assertEquals(int expected, int current, const char* file, return expected == current; } +/** + * Tests the equality of two values. + * + * Differences will be logged. + * + * @param expected the expected value + * @param current the current value + * @param file the file containing the test + * @param lineNo the line number containing the test + * @return true: equal + */ +bool RplTest::assertEquals(qint64 expected, qint64 current, const char* file, + int lineNo) { + if(expected != current) + error("%s-%d: error: %lld != %lld / %llx != %llx)", file, lineNo, + expected, current, (quint64) expected, (quint64) current); + return expected == current; +} + + /** * Tests the equality of two values. * @@ -99,6 +119,43 @@ bool RplTest::assertEquals(qreal expected, qreal current, const char* file, return expected == current; } +/** + * @brief Tests the equality of two values. + * + * Differences will be logged. + * + * @param expected the expected value + * @param current the current value + * @param file the file containing the test + * @param lineNo the line number containing the test + * @return true: equal + */ +bool RplTest::assertEquals(const char* expected, const QString& current, + const char* file, int lineNo) { + bool equal = assertEquals(expected, current.toUtf8().constData(), file, + lineNo); + return equal; +} + +/** + * @brief Tests the equality of two values. + * + * Differences will be logged. + * + * @param expected the expected value + * @param current the current value + * @param file the file containing the test + * @param lineNo the line number containing the test + * @return true: equal + */ +bool RplTest::assertEquals(const QString& expected, const QString& current, + const char* file, int lineNo) { + bool equal = assertEquals(expected.toUtf8().constData(), + current.toUtf8().constData(), file, lineNo); + return equal; +} + + /** * @brief Tests the equality of two values. * diff --git a/rplcore/rpltest.hpp b/rplcore/rpltest.hpp index 536bdbf..e5cbde5 100644 --- a/rplcore/rpltest.hpp +++ b/rplcore/rpltest.hpp @@ -24,7 +24,12 @@ private: RplTest& operator =(const RplTest& source); public: bool assertEquals(int expected, int current, const char* file, int lineNo); + bool assertEquals(qint64 expected, qint64 current, const char* file, int lineNo); bool assertEquals(qreal expected, qreal current, const char* file, int lineNo); + bool assertEquals(const char* expected, const QString& current, + const char* file, int lineNo); + bool assertEquals(const QString& expected, const QString& current, + const char* file, int lineNo); bool assertEquals(const char* expected, const char* current, const char* file, int lineNo); bool assertEquals(const QByteArray& expected, const QByteArray& current, diff --git a/rplexpr/rplexpr.hpp b/rplexpr/rplexpr.hpp new file mode 100644 index 0000000..a9d4d8d --- /dev/null +++ b/rplexpr/rplexpr.hpp @@ -0,0 +1,23 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + + +#ifndef RPLEXPR_HPP +#define RPLEXPR_HPP + +#include +#include +#include +#include +#include +#include + +#include "rplexpr/rplsource.hpp" +#include "rplexpr/rpllexer.hpp" + +#endif // RPLEXPR_HPP diff --git a/rplexpr/rpllexer.cpp b/rplexpr/rpllexer.cpp new file mode 100644 index 0000000..c4482f8 --- /dev/null +++ b/rplexpr/rpllexer.cpp @@ -0,0 +1,851 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + +#include "rplcore/rplcore.hpp" +#include "rplexpr/rplexpr.hpp" + +#define CHAR_INFO_SIZE (int(sizeof m_charInfo / sizeof m_charInfo[0])) +/** @class RplToken rpllexer.hpp "rplexpr/rpllexer.hpp" + * + * @brief Implements a token which is the smallest unit for a parser. + * + */ + +/** + * @brief RplLexException::RplLexException + * @param message + */ +/** + * @brief Constructor. + * + * @param position describes the position of the error/warning + * @param format the reason of the exception + * @param ... the values for the placeholders in the format. + */ +RplLexException::RplLexException(RplSourcePosition& position, + const char* format, ...) : + RplException("") +{ + char buffer[64000]; + m_message = position.toString().toUtf8(); + va_list ap; + va_start(ap, format); + vsnprintf(buffer, sizeof buffer, format, ap); + va_end(ap); + m_message += buffer; +} + +/** + * @brief Constructor. + * @param type token type + */ +RplToken::RplToken(RplTokenType type) : + m_tokenType(type), + m_string(), + m_rawString() + // m_value +{ + memset(&m_value, 0, sizeof m_value); +} + +/** + * @brief Destructor. + */ +RplToken::~RplToken() +{ +} +/** + * @brief Copy constructor. + * + * @param source source to copy + */ +RplToken::RplToken(const RplToken& source) : + m_tokenType(source.m_tokenType), + m_string(source.m_string), + m_rawString(source.m_rawString), + m_value(source.m_value) +{ +} +/** + * @brief Assignment operator. + * + * @param source source to copy + * @return + */ +RplToken& RplToken::operator =(const RplToken& source) +{ + m_tokenType = source.m_tokenType; + m_string = source.m_string; + m_value = source.m_value; + return *this; +} + +/** + * @brief Returns the string representation of the instance + * @return a string representing the instance + */ +const QString& RplToken::toString() +{ + QString& rc = m_string; + switch(m_tokenType) + { + case TOKEN_STRING: + rc = m_rawString; + break; + case TOKEN_KEYWORD: + case TOKEN_NUMBER: + case TOKEN_OPERATOR: + case TOKEN_ID: + case TOKEN_COMMENT_REST_OF_LINE: + case TOKEN_COMMENT_START: + case TOKEN_COMMENT_END: + case TOKEN_SPACE: + default: + break; + } + return rc; +} + +/** + * @brief Returns the integer value of the token + * + * Only relevant if a TOKEN_NUMBER. + * + * @return the value of the token as integer + */ +int RplToken::asInteger() const +{ + return (int) m_value.m_integer; +} + +/** + * @brief Returns the integer value of the token + * + * Only relevant if a TOKEN_NUMBER. + * + * @return the value of the token as unsigned integer (64 bit) + */ +quint64 RplToken::asUInt64() const +{ + return m_value.m_integer; +} + +/** + * @brief Returns the floating point value of the token + * + * Only relevant if a TOKEN_REAL. + * + * @return the value of the token as floating point value + */ +qreal RplToken::asReal() const +{ + return m_value.m_real; +} + +/** + * @brief Returns the floating point value of the token + * + * Only relevant if a TOKEN_NUMBER. + * + * @return the value of the token as floating point value + */ +const QString& RplToken::rawString() const +{ + return m_rawString; +} +/** + * @brief Returns the id of the token. + * + * Ids are more handy than string, e.g. allowing switch statements. + * + * Only relevant for TOKEN_KEYWORD and TOKEN_OPERATOR. + * + * @return the id of the token + */ +int RplToken::id() const +{ + return m_value.m_id; +} +/** + * @brief Returns the token type. + * @return the token type + */ +RplTokenType RplToken::tokenType() const +{ + return m_tokenType; +} + +/** + * @brief Makes all members undefined. + */ +void RplToken::clear() +{ + m_string.clear(); + m_rawString.clear(); + m_tokenType = TOKEN_UNDEF; + m_value.m_integer = 0; +} + + +/** @class RplLexer rpllexer.hpp "rplexpr/rpllexer.hpp" + * + * @brief Implements a lexical analyser. + * + * A lexical analyser reads a text source and separates the tokens for a parser. + * Tokens are the smallest elements of a parsing process. + * + */ + +static void itemsToVector(const char* items, QVector& vector, + int firstCharFlag, int secondCharFlag, + int thirdCharFlag, int restCharFlag, + int charInfo[]) +{ + QByteArray array2(items); + QList list = array2.split(' '); + QList::iterator it; + int id = 0; + for (it = list.begin(); it < list.end(); it++){ + QByteArray& item2 = *it; + QString item(item2); + id++; + item += QChar(' '); + item += QChar(id); + vector.push_back(item); + unsigned char cc = item2.at(0); + if (cc < 128) + charInfo[cc] |= firstCharFlag; + if(item2.size() > 1){ + cc = item2.at(1); + if (cc < 128) + charInfo[cc] |= secondCharFlag; + } + if(item2.size() > 2){ + cc = item2.at(2); + if (cc < 128) + charInfo[cc] |= thirdCharFlag; + } + if(item2.size() > 3){ + const char* ptr = item2.constData() + 3; + while( (cc = *ptr++) != '\0'){ + if (cc < 128) + charInfo[cc] |= restCharFlag; + } + } + } + qSort(vector.begin(), vector.end(), qLess()); +} + +static void charClassToCharInfo(const char* charClass, int flag, + int charInfo[]) +{ + for (int ix = 0; charClass[ix] != '\0'; ix++){ + unsigned char cc = (unsigned char) charClass[ix]; + if (cc < 128) + charInfo[cc] |= flag; + if (charClass[ix+1] == '-'){ + unsigned char ubound = charClass[ix+2]; + if (ubound == '\0') + charInfo['-'] |= flag; + else if (cc >= ubound) + throw new RplException("wrong character class range: %c-%c (%s)", + cc, ubound, charClass); + else { + for (int ii = cc + 1; ii <= ubound; ii++){ + charInfo[ii] |= flag; + } + } + ix += 2; + } + } +} + +/** + * @brief Constructor. + * + * @param source the input source handler + * @param keywords a string with all keywords delimited by ' '. + * Example: "if then else fi while do done" + * @param operators a string with all operators delimited by ' ' + * @param comments a string with pairs of comment begin and end delimited + * by ' '. The comment end can be '\n' for line end. + * Example: "/ * * / // \n" (ignore the blank in "* /") + * @param firstCharsId string with the characters which are allowed as first + * characters of an id + * @param restCharsId string with the characters which are allowed as non + * first characters of an id + * @param numericTypes bit mask of allowed numeric, + * e.g. NUMTYPE_DECIMAL | NUMTYPE_HEXADECIMAL + * @param stringFeatures bit mask of the string features, + * e.g. SF_QUOTE | SF_TICK + * @param storageFlags describes the things which should be stored, e.g. + * S_ORG_STRINGS | S_COMMENTS | S_BLANKS + */ +RplLexer::RplLexer(RplSource* source, + const char* keywords, const char* operators, const char* comments, + const char* firstCharsId, const char* restCharsId, int numericTypes, + int stringFeatures, int storageFlags) : + m_source(source), + m_keywords(), + m_operators(), + m_commentStarts(), + m_commentEnds(), + //m_charInfo() + m_idFirstRare(), + m_idRestRare(), + m_numericTypes(numericTypes), + m_idRest2(), + m_currentToken(&m_token1), + m_waitingToken(NULL), + m_token1(TOKEN_UNDEF), + m_token2(TOKEN_UNDEF), + m_currentPosition(&m_position1), + m_position1(RplSourcePosition()), + m_position2(RplSourcePosition()), + m_maxTokenLength(64), + m_input(), + m_currentCol(0), + m_hasMoreInput(false), + m_stringFeatures(stringFeatures), + m_storageFlags(storageFlags) +{ + m_currentPosition->setSourceUnit(source->currentReader() + ->currentSourceUnit()); + memset(m_charInfo, 0, sizeof m_charInfo); + itemsToVector(keywords, m_keywords, CC_FIRST_KEYWORD, CC_2nd_KEYWORD, + CC_3rd_KEYWORD, CC_REST_KEYWORD, m_charInfo); + prepareOperators(operators); + charClassToCharInfo(firstCharsId, CC_FIRST_ID, m_charInfo); + charClassToCharInfo(restCharsId, CC_REST_ID, m_charInfo); + initializeComments(comments); + m_input.reserve(m_maxTokenLength*2); +} +/** + * @brief Destructor. + */ +RplLexer::~RplLexer() +{ +} +/** + * @brief Stores the operators in the internal members + * + * @param operators a string with the operators separated by blank + */ +void RplLexer::prepareOperators(const char* operators){ + itemsToVector(operators, m_operators, CC_FIRST_OP, CC_2nd_OP, CC_3rd_OP, + CC_REST_OP, m_charInfo); + // m_operators is now sorted: + // test whether the successor of 1 char operators is starting with this char: + // if not this operator will be marked with CC_OP_1_ONLY: + for (int ix = 0; ix < m_operators.size() - 1; ix++){ + // the entries of m_operators end with ' ' and id: + if (m_operators.at(ix).size() == 1 + 2 + && m_operators.at(ix).at(0) != m_operators.at(ix+1).at(0)){ + int id = m_operators[ix].at(2).unicode(); + m_charInfo[id] |= CC_OP_1_ONLY; + } + } +} + +void RplLexer::initializeComments(const char* comments) +{ + if (comments != NULL) + { + QByteArray starters; + QByteArray comments2(comments); + int ix = comments2.indexOf(" "); + if (ix >= 0) + throw RplException("more than one blank between comment pair(s): col %d %s", + ix + 1, comments + ix); + // the index of m_commentEnds is the position number: we need a dummy entry: + m_commentEnds.append(""); + + QList items = comments2.split(' '); + QList::iterator it; + ix = 0; + for (it = items.begin(); it != items.end(); it++, ix++){ + if (ix % 2 == 0){ + if (ix > 0) + starters += " "; + starters += *it; + }else{ + m_commentEnds.append(*it); + } + } + if (ix % 2 != 0) + throw RplException("not only pairs in the comment list"); + itemsToVector(starters, m_commentStarts, CC_FIRST_COMMENT_START, + CC_2nd_COMMENT_START, CC_3rd_COMMENT_START, CC_REST_COMMENT_START, + m_charInfo); + } +} +/** + * @brief Searches the prefix of m_input in the vector. + * + * @param tokenLength the length of the prefix in m_input + * @param vector the vector to search. Each element contains the id + * as last entry + * @param id the id of the entry in the vector. Only set if found + * @return + */ +int RplLexer::findInVector(int tokenLength, const QVector& vector) +{ + int id = 0; + int lbound = 0; + int ubound = vector.size() - 1; + while(lbound <= ubound){ + int half = (ubound + lbound) / 2; + int compareRc = 0; + int ix = 0; + const QString& current = vector[half]; + // vector items end with ' ' and id: + int currentLength = current.size() - 2; + while(ix < tokenLength && compareRc == 0){ + if (ix >= currentLength) + // current is shorter: + compareRc = 1; + else + compareRc = m_input.at(ix).unicode() + - (int) current.at(ix).unicode(); + ix++; + } + if (compareRc == 0 && current.at(ix).unicode() != ' ') + // token.size() < current.size(): + compareRc = -1; + if (compareRc < 0) + ubound = half - 1; + else if (compareRc > 0) + lbound = half + 1; + else { + id = current[currentLength + 1].unicode(); + break; + } + } + return id; +} + +void RplLexer::startUnit(const char* unit) +{ + //m_source-> +} +const char* RplLexer::nextText(int& length, bool &isLast) +{ + return NULL; +} +/** + * @brief Reads data until enough data are available for one token. + * + * Data will be read from the current source unit. + * If this unit does not contain more data the next source unit from the stack + * will be used until the stack is empty. + * + * @return false: no more input is available
+ * true: data are available + */ +bool RplLexer::fillInput() +{ + if (m_hasMoreInput){ + if (m_input.size() < m_maxTokenLength){ + m_source->currentReader()->fillBuffer(m_maxTokenLength, m_input, + m_hasMoreInput); + } + } + + while (m_input.size() == 0 && m_source->currentReader() != NULL){ + if (m_source->currentReader()->nextLine(m_maxTokenLength, + m_input, m_hasMoreInput)){ + m_currentCol = 0; + } + } + return m_input.size() > 0; +} + +/** + * @brief Finds a token with an id: TOKEN_OP, TOKEN_KEYWORD, ... + * + * @param tokenType the token type + * @param flag2 the flag of the 2nd char + * @param names the vector with the names, sorted + * @return NULL: not found
+ * otherwise: the token + */ +RplToken* RplLexer::findTokenWithId(RplTokenType tokenType, int flag2, + QVector& names) +{ + int length = 1; + int inputLength = m_input.size(); + int cc; + if (inputLength > 1){ + cc = m_input[1].unicode(); + if (cc < CHAR_INFO_SIZE && (m_charInfo[cc] & flag2)){ + length++; + if (inputLength > 2){ + cc = m_input[2].unicode(); + // the 3rd char flag is the "successor" of the 2nd char flag: + int flag = (flag2 << 1); + if (cc < CHAR_INFO_SIZE && (m_charInfo[cc] & flag)){ + length++; + // the rest char flag is the "successor" of the 3nd char flag: + flag <<= 1; + while (length < inputLength){ + cc = m_input[length].unicode(); + if (cc < CHAR_INFO_SIZE && (m_charInfo[cc] & flag)) + length++; + else + break; + } + } + } + } + } + RplToken* rc = NULL; + if (! (tokenType == TOKEN_KEYWORD && length < inputLength + && (cc = m_input[length].unicode()) < CHAR_INFO_SIZE + && (m_charInfo[cc] & CC_REST_ID))) { + int id; + // the length could be too long: the CC_2nd_.. flag could be ambigous + while(length >= 1 && (id = findInVector(length, names)) <= 0){ + length--; + } + if (id > 0){ + rc = m_currentToken; + rc->m_tokenType = tokenType; + rc->m_value.m_id = id; + m_input.remove(0, length); + m_currentCol += length; + } + } + return rc; +} + +/** + * @brief Converts the number into a token. + * + * @return the token with the number + */ +RplToken* RplLexer::scanNumber() +{ + int inputLength = m_input.size(); + int cc; + int length; + quint64 value = 0; + if ( (cc = m_input[0].unicode()) == '0' && inputLength > 1 + && (m_numericTypes & NUMTYPE_HEXADECIMAL) + && (m_input[1].unicode() == 'x' || m_input[1].unicode() == 'X')){ + length = RplQString::lengthOfUInt64(m_input, 2, 16, &value); + if (length > 0) + length += 2; + else + throw RplException("invalid hexadecimal number: no digit behind 'x"); + } else if (cc == '0' && (m_numericTypes & NUMTYPE_OCTAL) + && inputLength > 1){ + length = 1; + while (length < inputLength){ + if ( (cc = m_input[length].unicode()) >= '0' && cc <= '7') + value = value * 8 + cc - '0'; + else if (cc >= '8' && cc <= '9') + throw RplLexException(*m_currentPosition, + "invalid octal digit: %c", cc); + else + break; + length++; + } + } else { + length = 1; + value = cc - '0'; + while (length < inputLength){ + if ( (cc = m_input[length].unicode()) >= '0' && cc <= '9') + value = value * 10 + cc - '0'; + else + break; + length++; + } + } + m_currentToken->m_value.m_integer = value; + m_currentToken->m_tokenType = TOKEN_NUMBER; + if (length + 1 < inputLength + && ((cc = m_input[length].unicode()) == '.' || toupper(cc) == 'E')){ + qreal realValue; + int realLength = RplQString::lengthOfReal(m_input, 0, &realValue); + if (realLength > length){ + m_currentToken->m_tokenType = TOKEN_REAL; + m_currentToken->m_value.m_real = realValue; + length = realLength; + } + } + m_input.remove(0, length); + m_currentCol += length; + return m_currentToken; +} + +/** + * @brief Reads a string into the internal data. + * @return the token with the string + */ +RplToken*RplLexer::scanString() +{ + int delim = m_input[0].unicode(); + int inputLength = m_input.size(); + int cc; + int length = 1; + m_currentToken->m_string.append(m_input[0]); + bool again = false; + do { + while(length < inputLength && (cc = m_input[length].unicode()) != delim){ + length++; + if (cc != '\\' + || (m_stringFeatures + & (SF_C_ESCAPING | SF_C_HEX_CHARS | SF_C_SPECIAL)) == 0){ + m_currentToken->m_rawString.append(QChar(cc)); + } else { + if (length >= inputLength) + throw RplLexException(*m_currentPosition, + "backslash without following character"); + cc = m_input[length++].unicode(); + if ( (m_stringFeatures & SF_C_HEX_CHARS) && toupper(cc) == 'X'){ + if (length >= inputLength) + throw RplLexException(*m_currentPosition, + "missing hexadecimal digit behind \\x"); + cc = m_input[length++].unicode(); + int hexVal = RplQString::valueOfHexDigit(cc); + if (hexVal < 0) + throw RplLexException(*m_currentPosition, + "not a hexadecimal digit behind \\x: %lc", + QChar(cc)); + if (length < inputLength){ + cc = m_input[length].unicode(); + int nibble = RplQString::valueOfHexDigit(cc); + if (nibble >= 0){ + length++; + hexVal = hexVal * 16 + nibble; + } + } + m_currentToken->m_rawString.append(QChar(hexVal)); + } else if ( (m_stringFeatures & SF_C_SPECIAL)){ + switch(cc){ + case 'r': + cc = '\r'; + break; + case 'n': + cc = '\n'; + break; + case 't': + cc = '\t'; + break; + case 'a': + cc = '\a'; + break; + case 'v': + cc = '\v'; + break; + default: + break; + } + m_currentToken->m_rawString.append(QChar(cc)); + } else { + m_currentToken->m_rawString.append(QChar(cc)); + } + } + } + if (cc == delim){ + length++; + } + if ( (m_stringFeatures & SF_DOUBLE_DELIM) && length < inputLength + && m_input[length].unicode() == delim){ + m_currentToken->m_rawString.append(delim); + length++; + again = true; + } + } + while(again); + if (m_storageFlags & STORE_ORG_STRING) + m_currentToken->m_string.append(m_input.mid(0, length)); + m_input.remove(0, length); + m_currentCol += length; + return m_currentToken; +} + +/** + * @brief Reads a comment into the internal data. + * + * precondition: the current token is prepared yet + */ +void RplLexer::scanComment() +{ + int inputLength = m_input.size(); + int cc; + int length = 1; + QString commentEnd = m_commentEnds[m_currentToken->id()]; + int ix; + if (commentEnd[0].unicode() == '\n'){ + // single line comment: + if (m_storageFlags & STORE_COMMENT) + m_currentToken->m_string.append(m_input); + length = inputLength; + } else { + // multiline comment: + while( (ix = m_input.indexOf(commentEnd)) < 0){ + if (m_storageFlags & STORE_COMMENT) + m_currentToken->m_string.append(m_input); + if (! fillInput()) + throw RplLexException(*m_currentPosition, + "comment end not found"); + } + if (m_storageFlags & STORE_COMMENT) + m_currentToken->m_rawString.append(m_input.mid(0, ix)); + length = ix + commentEnd.size(); + } + m_input.remove(0, length); + m_currentCol += length; +} + +/** + * @brief Returns the next token. + * + * @return the next token + */ +RplToken* RplLexer::nextToken() +{ + RplToken* rc = NULL; + int ix; + if (m_waitingToken != NULL){ + rc = m_waitingToken; + m_waitingToken = NULL; + m_currentPosition = m_currentPosition == &m_position1 + ? &m_position2 : &m_position1; + } else { + m_currentToken->clear(); + m_currentPosition->setLineNo(m_source->currentReader() + ->currentSourceUnit()->lineNo()); + m_currentPosition->setColumn(m_currentCol); + if (! fillInput()){ + m_currentToken->m_tokenType = TOKEN_END_OF_SOURCE; + } else { + QChar cc = m_input.at(0); + if (cc.isSpace()){ + m_currentToken->m_tokenType = TOKEN_SPACE; + ix = 1; + while(ix < m_input.size() && m_input.at(ix).isSpace()) + ix++; + if (m_storageFlags & STORE_BLANK){ + m_currentToken->m_string.append(m_input.mid(0, ix)); + } + m_currentCol += ix; + m_input.remove(0, ix); + rc = m_currentToken; + } else if (cc.isDigit()){ + rc = scanNumber(); + } else if (cc == '"' || cc == '\''){ + rc = scanString(); + } else { + int cc2 = cc.unicode(); + if (cc2 >= CHAR_INFO_SIZE) + throw RplLexException(*m_currentPosition, + "no lexical symbol can start with this char: %ls", + QChar(cc2)); + else + { + if (rc == NULL && (m_charInfo[cc2] & CC_FIRST_COMMENT_START)){ + rc = findTokenWithId(TOKEN_COMMENT_START, + CC_2nd_COMMENT_START, m_commentStarts); + if (rc != NULL) + scanComment(); + } + + if (rc == NULL && (m_charInfo[cc2] & CC_FIRST_OP)){ + if ( (m_charInfo[cc2] & CC_OP_1_ONLY) == 0){ + rc = findTokenWithId(TOKEN_OPERATOR, + CC_2nd_OP, m_operators); + } else { + rc = m_currentToken; + rc->m_tokenType = TOKEN_OPERATOR; + rc->m_value.m_id = findInVector(1, m_operators); + m_input.remove(0, 1); + m_currentCol += 1; + } + } + if (rc == NULL && (m_charInfo[cc2] & CC_FIRST_KEYWORD)){ + rc = findTokenWithId(TOKEN_KEYWORD, + CC_2nd_KEYWORD, m_keywords); + } + + } + } + } + } + if (rc == NULL || rc->tokenType() == TOKEN_UNDEF){ + if (m_input.size() == 0){ + rc = m_currentToken; + rc->m_tokenType = TOKEN_END_OF_SOURCE; + } else { + QString symbol = m_input.mid(0, qMin(20, m_input.size()-1)); + throw RplLexException(*m_currentPosition, + "unknown lexical symbol: %s", symbol.toUtf8().constData()); + } + } + return rc; +} +/** + * @brief Returns the maximal length of a token + * @return the maximal length of a token + */ +size_t RplLexer::maxTokenLength() const +{ + return m_maxTokenLength; +} + +/** + * @brief Sets the maximal length of a token + * + * @param maxTokenLength the new maximal length of a token + */ +void RplLexer::setMaxTokenLength(size_t maxTokenLength) +{ + m_maxTokenLength = maxTokenLength; +} + +/** + * @brief RplLexer::nextNonSpaceToken + * @return + */ +RplToken* RplLexer::nextNonSpaceToken() +{ + RplToken* rc = NULL; + RplTokenType type; + do{ + rc = nextToken(); + } while( (type = m_currentToken->tokenType()) == TOKEN_SPACE + || type == TOKEN_COMMENT_START || type == TOKEN_COMMENT_END + || type == TOKEN_COMMENT_REST_OF_LINE); + return rc; +} + +/** + * @brief Starts a new source unit. + * + * Saves the current source position onto the top of stack. + * Pushes the source unit onto the top of stack. + * + * @param unit + */ +void RplLexer::startUnit(const QString& unit) +{ + // m_source->startUnit(unit, new RplSourcePosition( + // m_source->currentReader())); +} +/** + * @brief Returns the source of the instance. + * + * @return the source of the instance + */ +RplSource*RplLexer::source() +{ + return m_source; +} + + diff --git a/rplexpr/rpllexer.hpp b/rplexpr/rpllexer.hpp new file mode 100644 index 0000000..db6e8a1 --- /dev/null +++ b/rplexpr/rpllexer.hpp @@ -0,0 +1,215 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + + +#ifndef RPLLEXER_HPP +#define RPLLEXER_HPP + + +enum RplTokenType { + TOKEN_UNDEF, + TOKEN_STRING, + TOKEN_NUMBER, + TOKEN_REAL, + TOKEN_KEYWORD, + TOKEN_OPERATOR, + TOKEN_ID, + TOKEN_COMMENT_REST_OF_LINE, + TOKEN_COMMENT_START, + TOKEN_COMMENT_END, + TOKEN_SPACE, + TOKEN_END_OF_SOURCE, + TOKEN_COUNT +}; + +class RplLexException : public RplException { +public: + RplLexException(RplSourcePosition& position, const char* message, ...); +}; + +class RplLexer; + +class RplToken { +public: + RplToken(RplTokenType type); + ~RplToken(); + RplToken(const RplToken& source); + RplToken& operator =(const RplToken& source); +public: + friend class RplLexer; + const QString& toString(); + bool isInteger(); + int asInteger() const; + quint64 asUInt64() const; + qreal asReal() const; + const QString& rawString() const; + int id() const; + RplTokenType tokenType() const; + void clear(); +protected: + RplTokenType m_tokenType; + QString m_string; + // only for TOKEN_STRING: copy from source but without escaped chars like "\\n" + QString m_rawString; + union { + // only for TOKEN_KEYWORD and TOKEN_OPERATOR + int m_id; + quint64 m_integer; + qreal m_real; + } m_value; +}; + + +class RplSource; +class RplLexer +{ +public: + enum NumericType { + NUMTYPE_UNDEF, + NUMTYPE_DECIMAL = 1 << 0, + NUMTYPE_OCTAL = 1 << 1, + NUMTYPE_HEXADECIMAL = 1 << 2, + NUMTYPE_FLOAT = 1 << 3, + /// + NUMTYPE_ALL_INTEGER = NUMTYPE_DECIMAL | NUMTYPE_OCTAL |NUMTYPE_HEXADECIMAL, + NUMTYPE_ALL = NUMTYPE_ALL_INTEGER | NUMTYPE_FLOAT + }; + enum CharClassTag { + CC_UNDEF = 0, + /// this char is possible as first char of an id + CC_FIRST_ID = 1 << 0, + /// this char is possible as 2nd char of an id + CC_2nd_ID = 1 << 1, + /// this char is possible as 3rd char of an id + CC_3rd_ID = 1 << 2, + /// this char is possible as 4th... char of an id + CC_REST_ID = 1 << 3, + /// this char can start a comment + CC_FIRST_COMMENT_START = 1 << 4, + /// this char can be the 2nd char of a comment start + CC_2nd_COMMENT_START = 1 << 5, + /// this char can be the 3rd char of a comment start + CC_3rd_COMMENT_START = 1 << 6, + /// this char can be the 4th ... of a comment start + CC_REST_COMMENT_START = 1 << 7, + /// this char can start a keyword + CC_FIRST_KEYWORD = 1 << 8, + /// this char can be the 2nd char of a keyword + CC_2nd_KEYWORD = 1 << 9, + /// this char can be the 3rd char of a keyword + CC_3rd_KEYWORD = 1 << 10, + /// this char can be the 4th... char of a keyword + CC_REST_KEYWORD = 1 << 11, + /// this char can be the 1st char of an operator + CC_FIRST_OP = 1 << 12, + /// this char can be the 2nd char of an operator + CC_2nd_OP = 1 << 13, + /// this char can be the 3rd char of an operator + CC_3rd_OP = 1 << 14, + /// this char can be the 4th... char of an operator + CC_REST_OP = 1 << 15, + /// there is an operator with exactly this char + /// and there is no other operator starting with this char + CC_OP_1_ONLY = 1 << 16 + }; + enum StringFeatures { + SF_UNDEF, + /// ' can be a string delimiter + SF_TICK = 1 << 1, + /// " can be a string delimiter + SF_QUOTE = 1 << 2, + /// character escaping like in C: \ is + SF_C_ESCAPING = 1 << 3, + /// special characters like in C: \r \f \r \t + SF_C_SPECIAL = 1 << 4, + /// characters can be written in hexadecimal notation: \x20 is ' ' + SF_C_HEX_CHARS = 1 << 5, + /// A delimiter inside a string must be doubled (like in Pascal) + SF_DOUBLE_DELIM = 1 << 6 + }; + enum StorageFlags { + S_UNDEF, + /// the original string will be stored in m_string too + /// (not only m_rawString) + STORE_ORG_STRING = 1 << 1, + /// comments will be stored in m_string + STORE_COMMENT = 1 << 2, + /// blanks will be stored in m_string + STORE_BLANK = 1 << 3, + /// redefinitions for better reading: + STORE_NOTHING = 0, + STORE_ALL = STORE_ORG_STRING | STORE_COMMENT | STORE_BLANK + }; + +public: + RplLexer(RplSource* source, + const char* keywords, const char* operators, + const char* comments, + const char* firstCharsId = "a-zA-Z_", + const char* restCharsId = "a-zA-Z0-9_", + int numericTypes = NUMTYPE_DECIMAL | NUMTYPE_HEXADECIMAL | NUMTYPE_FLOAT, + int stringFeatures = SF_TICK | SF_QUOTE | SF_C_ESCAPING | SF_C_SPECIAL + | SF_C_HEX_CHARS, + int storageFlags = STORE_NOTHING); + virtual ~RplLexer(); +public: + void startUnit(const char* unit); + RplToken* nextToken(); + RplToken* peekNonSpaceToken(); + RplToken* nextNonSpaceToken(); + size_t maxTokenLength() const; + void setMaxTokenLength(size_t maxTokenLength); + void startUnit(const QString& unit); + RplSource* source(); +private: + void prepareOperators(const char* operators); + void initializeComments(const char* comments); + bool fillInput(); + int findInVector(int tokenLength, const QVector& vector); + const char* nextText(int& length, bool &isLast); + RplToken* findTokenWithId(RplTokenType tokenType, int flag2, + QVector& names); + RplToken* scanNumber(); + RplToken* scanString(); + void scanComment(); +protected: + RplSource* m_source; + /// sorted, string ends with the id of the keyword + QVector m_keywords; + // sorted, string ends with the id of the operator + QVector m_operators; + // sorted, string ends with the id of the comment start + QVector m_commentStarts; + // index: id content: comment_end + QVector m_commentEnds; + // index: ord(char) content: a sum of CharClassTags + int m_charInfo[128]; + // a list of QChars with ord(cc) > 127 and which can be the first char + QString m_idFirstRare; + // index: ord(char) content: chr(ix) can be the non first char of an id + QString m_idRestRare; + // a list of QChars with ord(cc) > 127 and which can be the first char + int m_numericTypes; + QString m_idRest2; + RplToken* m_currentToken; + RplToken* m_waitingToken; + RplToken m_token1; + RplToken m_token2; + RplSourcePosition* m_currentPosition; + RplSourcePosition m_position1; + RplSourcePosition m_position2; + int m_maxTokenLength; + QString m_input; + int m_currentCol; + bool m_hasMoreInput; + int m_stringFeatures; + int m_storageFlags; +}; + + +#endif // RPLLEXER_HPP diff --git a/rplexpr/rplsource.cpp b/rplexpr/rplsource.cpp new file mode 100644 index 0000000..e885db8 --- /dev/null +++ b/rplexpr/rplsource.cpp @@ -0,0 +1,709 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ +#include "rplcore/rplcore.hpp" +#include "rplexpr/rplexpr.hpp" + + +/** @class RplSource rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Stores all source positions. + * + */ + +/** @class RplSourceUnit rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Implements the base class of input source units. + * + * A source unit is set of input lines with a name, e.g. a file. + */ + +/** + * @brief Constructor + * @param name name of the unit + * @param reader the reader which can read the unit + */ +RplSourceUnit::RplSourceUnit(const QString& name, RplReader* reader) : + m_name(name), + m_lineNo(0), + m_reader(reader) { +} + +/** + * @brief Destructor. + */ +RplSourceUnit::~RplSourceUnit() { +} + +/** + * @brief Returns the name. + * @return the name + */ +const QString& RplSourceUnit::name() const { + return m_name; +} + +/** + * @brief Returns the line number. + * + * @return the line number + */ +int RplSourceUnit::lineNo() const { + return m_lineNo; +} + +/** + * @brief Sets the line number. + * + * @param lineNo the new line number + */ +void RplSourceUnit::setLineNo(int lineNo) { + m_lineNo = lineNo; +} +/** + * @brief Returns the reader of the instance. + * + * @return the reader belonging to the instance + */ +RplReader* RplSourceUnit::reader() const { + return m_reader; +} + +/** @class RplSourcePosition rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Stores a precise position in the input source. + * + * The input source contains normally a lot of nested source units. + * + * Each source unit owns a name and a sequence of lines. + * + * The mostly used source unit is a text file. + * + * A precice position is the stack of source unit positions. + */ + +/** + * @brief Constructor. + */ +RplSourcePosition::RplSourcePosition() : + m_sourceUnit(NULL), + m_lineNo(0), + m_column(0), + m_caller(NULL) +{ +} + +/** + * @brief Constructor. + * + * @param unit name of the input source (normally a file) + * @param lineNo line number inside the input source + * @param colNo distance to the line start + * @param source parent: the source + */ +RplSourcePosition::RplSourcePosition(RplSourceUnit* unit, int lineNo, + int colNo) : + m_sourceUnit(unit), + m_lineNo(lineNo), + m_column(colNo), + m_caller(unit->reader()->source().sourcePositionStack().top()) +{ +} +/** + * @brief Destructor + */ +RplSourcePosition::~ RplSourcePosition(){ +} + +/** + * @brief Copy constructor. + * + * @param source instance to copy + */ +RplSourcePosition::RplSourcePosition(const RplSourcePosition& source) : + m_sourceUnit(source.m_sourceUnit), + m_lineNo(source.m_lineNo), + m_column(source.m_column), + m_caller(source.m_caller) +{ +} + +/** + * @brief Assignment operator. + * + * @param source instance to copy + * @return the instance itself + */ +RplSourcePosition&RplSourcePosition::operator=(const RplSourcePosition& source) +{ + m_sourceUnit = source.m_sourceUnit; + m_lineNo = source.m_lineNo; + m_column = source.m_column; + m_caller = source.m_caller; + return *this; +} +/** + * @brief Returns a description of the source position: "- ():". + * + * @return a description of the instance + */ +QString RplSourcePosition::toString() const +{ + QString rc = m_sourceUnit->name(); + QTextStream stream(&rc); + stream << "-" << m_lineNo << " (" << m_column << "): "; + return rc; +} + +/** + * @brief Sets the line number. + * + * @param lineNo the new lineNo + */ +void RplSourcePosition::setLineNo(int lineNo) +{ + m_lineNo = lineNo; +} +/** + * @brief Returns the column. + * + * @return the column of instance. + */ +int RplSourcePosition::column() const +{ + return m_column; +} + +/** + * @brief Sets the column. + * + * @param column the new column + */ +void RplSourcePosition::setColumn(int column) +{ + m_column = column; +} + +/** + * @brief Returns the source unit belonging to the instance. + * + * @return the source unit of the instance + */ +RplSourceUnit* RplSourcePosition::sourceUnit() const +{ + return m_sourceUnit; +} + +/** + * @brief Sets the source unit. + * + * @param sourceUnit the new source unit of the instance + */ +void RplSourcePosition::setSourceUnit(RplSourceUnit* sourceUnit) +{ + m_sourceUnit = sourceUnit; +} + + +/** @class RplReader rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Implements a base class for readers of different media. + */ + + +/** + * @brief Constructor. + * + * @param source the parent + */ +RplReader::RplReader(RplSource& source) : + m_currentSourceUnit(NULL), + m_units(), + m_source(source) { +} + +/** + * @brief Destructor. + */ +RplReader::~RplReader() { + QMap::iterator it; + for(it = m_units.begin(); it != m_units.end(); it++) { + RplStringSourceUnit* unit = (RplStringSourceUnit*)(*it); + delete unit; + } +} + +/** + * @brief Returns the source of the reader. + * + * @return the parent, a source instance + */ +RplSource&RplReader::source() +{ + return m_source; +} +/** + * @brief Returns the current source unit. + * + * @return the source unit + */ +RplSourceUnit* RplReader::currentSourceUnit() const { + return m_currentSourceUnit; +} + +/** + * @brief Sets the current source unit. + * + * @param sourceUnit the name of the new source unit + */ +bool RplReader::setCurrentSourceUnit(const QString& sourceUnit) { + bool rc = m_units.contains(sourceUnit); + if(rc) { + m_currentSourceUnit = m_units.value(sourceUnit); + m_source.pushSourceUnit(m_currentSourceUnit); + } + return rc; +} + +/** + * @brief Removes the "latest" sourceUnit. + */ +void RplReader::removeSourceUnit() { + + m_currentSourceUnit = m_source.popSourceUnit(this);; +} + + +/** @class RplSource rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Administrates a set of input sources with different readers. + * + * An input stream can be built by some different resources, e.g. files + * and memory buffers. The RplSource can administrate these resources. + */ +/** + * @brief Constructor. + */ +RplSource::RplSource() : + m_sourcePositionStack(), + m_sourcePositions(), + m_readers(), + m_sourceUnits(), + m_unitStack(), + m_currentReader(NULL) +{ + // the stack should never be empty: + m_sourcePositionStack.push(NULL); +} + +/** + * @brief Destructor. + */ +RplSource::~RplSource() { +} + +/** + * @brief Returns the permanently valid source unit name. + * + * If unit names can be local objects (not string constants + * like __FILE__) then this method must be overridden by + * a method which builds a permanently valid string. + * + * @param unit unit to find + * @return a permanently valid unit name + */ +const char* RplSource::permanentUnitName(const char* unit) { + return unit; +} + +/** + * @brief Returns the stack with the positions of the open input resources. + * + * @return the stack + */ +QStack RplSource::sourcePositionStack() const { + return m_sourcePositionStack; +} + +/** + * @brief Returns the source unit stack. + * @return the stack of the source units + */ +QStack& RplSource::sourceUnitStack() +{ + return m_unitStack; +} + +/** + * @brief Adds a source reader. + * + * @param reader the new reader. Will be freed in the destructor + */ +void RplSource::addReader(RplReader* reader) { + m_readers.push_back(reader); + if (m_currentReader == NULL) + m_currentReader = reader; +} + +/** + * @brief Adds a source unit. + * + * @param unit the new unit. Will be freed in the destructor + */ +void RplSource::addSourceUnit(RplSourceUnit* unit) { + m_sourceUnits.push_back(unit); +} + +/** + * @brief Starts a new source unit. + * + * Saves the current source position onto the top of stack. + * Pushes the source unit onto the top of stack. + * + * @param unit + */ +bool RplSource::startUnit(const QString& unit, + const RplSourcePosition* sourcePosition) { + m_sourcePositionStack.push_back(sourcePosition); + RplReader* reader = NULL; + QList::iterator it; + for(it = m_readers.begin(); + reader == NULL && it != m_readers.end(); + it++) { + RplReader* current = *it; + if(current->openSourceUnit(unit)) { + reader = current; + } + } + return reader != NULL; +} + +/** + * @brief Pushes a source unit onto the stack. + * + * @param unit the source unit + */ +void RplSource::pushSourceUnit(RplSourceUnit* unit) { + m_unitStack.push(unit); +} + +/** + * @brief Removes the latest source unit from the stack. + * + * @param reader the current reader + * @return NULL: the current reader does not have an open source unit
+ * otherwise: the last entry from the source unit stack + */ +RplSourceUnit* RplSource::popSourceUnit(RplReader* reader) { + if(m_unitStack.size() > 0) + m_unitStack.pop(); + m_currentReader = m_unitStack.size() <= 0 + ? NULL : m_unitStack.top()->reader(); + RplSourceUnit* rc = NULL; + if(m_currentReader == reader) + rc = m_unitStack.top(); + else { + for(int ix = m_unitStack.size() - 2; ix >= 0; ix--){ + if(m_unitStack[ix]->reader() == reader){ + rc = m_unitStack[ix]; + break; + } + } + } + return rc; +} + +/** + * @brief Returns the reader of the current source unit. + * + * @return NULL: no reader active
+ * otherwise: the current reader + */ +RplReader* RplSource::currentReader() { + return m_currentReader; +} + +/** @class RplStringSourceUnit rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Stores the state of a string based source unit. + * + */ + +/** + * @brief Constructor. + * + * @param name name of the unit + * @param content content of the unit + * @param reader the parent + */ +RplStringSourceUnit::RplStringSourceUnit(const QString& name, + const QString& content, RplStringReader* reader) : + RplSourceUnit(name, reader), + m_currentPosition(0), + m_content(content) { +} + +/** + * @brief Destructor. + */ +RplStringSourceUnit::~RplStringSourceUnit() { +} +/** + * @brief Returns the current read position. + * + * @return the offset (count of QChars) of the end of the last read block + * inside m_content + */ +int RplStringSourceUnit::currentPosition() const { + return m_currentPosition; +} + +/** + * @brief Sets the current read position. + * + * @param currentPosition the offset (count of QChars) of the end of + * the last read block inside m_content + */ +void RplStringSourceUnit::setCurrentPosition(int currentPosition) { + m_currentPosition = currentPosition; +} +/** + * @brief Returns the content of the source unit. + * + * @return the content + */ +const QString& RplStringSourceUnit::content() const { + return m_content; +} + +/** @class RplStringReader rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Implements a source which provides reading from memory based buffers. + * + * Examples for usage: interactive input + */ + +/** + * @brief Constructor. + * + * @param source the parent + */ +RplStringReader::RplStringReader(RplSource& source) : + RplReader(source) { +} + +/** + * @brief Destructor. + */ +RplStringReader::~RplStringReader() { +} + +/** + * @brief Opens a new source unit. + * + * @param unit name of the source + * @return NULL: unknown source
+ * otherwise: an instance of a sub class of + * RplSourceUnit + */ +RplSourceUnit* RplStringReader::openSourceUnit(const QString& unit) { + RplSourceUnit* rc = NULL; + if(setCurrentSourceUnit(unit)) { + rc = m_currentSourceUnit; + ((RplStringSourceUnit*) rc)->setCurrentPosition(0); + } + return rc; +} +/** + * @brief Delivers the next line from the input medium or the first part of it. + * + * @param maxSize if the line length is longer than this value, only the + * first part of the line will be returned + * @param buffer OUT: the line will be put here + * @param hasMore true: the line was longer than maxSize, only + * the first part of the line is put into the buffer + * @return false: no more input available
+ * true: success + */ +bool RplStringReader::nextLine(int maxSize, QString& buffer, + bool& hasMore) { + m_currentSourceUnit->setLineNo(m_currentSourceUnit->lineNo() + 1); + bool rc = fillBuffer(maxSize, buffer, hasMore); + return rc; +} + +/** + * @brief Delivers the next part of a long line. + * + * @param maxSize if the remaining line length is longer than this value, + * only the a part of the line will be returned + * @param buffer OUT: the part of line will be put here + * @param hasMore true: the line was longer than maxSize, only + * the first part of the line is put into the buffer + * @return false: no more input available
+ * true: success + */ +bool RplStringReader::fillBuffer(int maxSize, QString& buffer, + bool& hasMore) { + RplStringSourceUnit* unit = (RplStringSourceUnit*) m_currentSourceUnit; + int start = unit->currentPosition(); + const QString& content = unit->content(); + int end = content.indexOf("\n", start); + hasMore = false; + if(end < 0) { + end = content.size() - 1; + } + int size = end - start + 1; + hasMore = false; + if(size > maxSize) { + size = maxSize; + hasMore = true; + } + if(size > 0) { + buffer += content.mid(start, size); + unit->setCurrentPosition(start + size); + } else { + removeSourceUnit(); + } + return size > 0; +} + +/** + * @brief Adds a source buffer to the reader + * + * @param name name of the medium + * @param content + */ +void RplStringReader::addSource(const QString& name, + const QString& content) { + // Deletion in the destructor of the base class RplReader + RplStringSourceUnit* unit = new RplStringSourceUnit(name, content, this); + m_units.insert(m_units.begin(), name, unit); + m_currentSourceUnit = unit; +} + +/** @class RplFileSourceUnit rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Stores the state of a file based source unit. + * + * This is the mostly used implementation of the RplSourceUnit/RplReader. + */ + +/** + * @brief Constructor. + * + * @param name name of the unit, the filename + * @param reader the parent + */ +RplFileSourceUnit::RplFileSourceUnit(const QDir& dir, + RplFileReader* reader) : + RplSourceUnit(dir.absolutePath(), reader), + m_currentPosition(0), + m_file(dir.absolutePath()), + m_textStream(&m_file), + m_line() { +} + +/** + * @brief Destructor. + */ +RplFileSourceUnit::~RplFileSourceUnit() { +} +/** @class RplFileReader rplsource.hpp "rplexpr/rplsource.hpp" + * + * @brief Implements a source which provides reading from memory based buffers. + * + * Examples for usage: interactive input + */ + +/** + * @brief Constructor. + */ +RplFileReader::RplFileReader(RplSource& source) : + RplReader(source) { +} + +/** + * @brief Destructor. + */ +RplFileReader::~RplFileReader() { +} + +/** + * @brief Opens a new source unit. + * + * @param unit name of the source + * @return NULL: unknown source
+ * otherwise: an instance of a sub class of + * RplSourceUnit + */ +RplSourceUnit* RplFileReader::openSourceUnit(const QString& unit) { + RplSourceUnit* rc = NULL; + if(m_units.contains(unit)) { + rc = *m_units.find(unit); + m_currentUnit = (RplFileSourceUnit*) rc; + } + return rc; +} +/** + * @brief Delivers the next line from the input medium or the first part of it. + * + * @param maxSize if the line length is longer than this value, only the + * first part of the line will be returned + * @param buffer OUT: the line will be put here + * @param hasMore true: the line was longer than maxSize, only + * the first part of the line is put into the buffer + * @return false: no more input available
+ * true: success + */ +bool RplFileReader::nextLine(int maxSize, QString& buffer, + bool& hasMore) { + RplFileSourceUnit* unit = (RplFileSourceUnit*) m_currentUnit; + bool rc = ! unit->m_textStream.atEnd(); + if(rc) { + unit->m_line = unit->m_textStream.readLine(); + rc = fillBuffer(maxSize, buffer, hasMore); + } + return rc; +} + +/** + * @brief Delivers the next part of a long line. + * + * @param maxSize if the remaining line length is longer than this value, + * only the a part of the line will be returned + * @param buffer OUT: the part of line will be put here + * @param hasMore true: the line was longer than maxSize, only + * the first part of the line is put into the buffer + * @return false: no more input available
+ * true: success + */ +bool RplFileReader::fillBuffer(int maxSize, QString& buffer, + bool& hasMore) { + RplFileSourceUnit* unit = (RplFileSourceUnit*) m_currentUnit; + int start = unit->m_currentPosition; + const QString& content = unit->m_line; + int size = content.size() - start; + if(size > maxSize) + size = maxSize; + buffer += content.mid(start, size); + unit->m_currentPosition = (start += size); + hasMore = start < content.length(); + return size > 0; +} + +/** + * @brief Adds a source file to the reader + * + * @param dirEntry the file + */ +void RplFileReader::addSource(const QDir& dirEntry) { + QString name = dirEntry.absolutePath(); + // Deleting in ~RplSourceUnit(): + RplFileSourceUnit* unit = new RplFileSourceUnit(dirEntry, this); + m_units.insert(m_units.begin(), name, unit); +} diff --git a/rplexpr/rplsource.hpp b/rplexpr/rplsource.hpp new file mode 100644 index 0000000..f44e532 --- /dev/null +++ b/rplexpr/rplsource.hpp @@ -0,0 +1,200 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + + +#ifndef RPLSOURCE_HPP +#define RPLSOURCE_HPP + +class RplSource; +class RplReader; + +class RplSourceUnit { +public: + RplSourceUnit(const QString& name, RplReader* reader); + virtual ~RplSourceUnit(); +public: + const QString& name() const; + int lineNo() const; + void setLineNo(int lineNo); + RplReader* reader() const; + +protected: + QString m_name; + int m_lineNo; + RplReader* m_reader; +}; + +class RplSourcePosition { +public: + RplSourcePosition(); + RplSourcePosition(RplSourceUnit* unit, int lineNo, int colNo); + ~ RplSourcePosition(); + RplSourcePosition(const RplSourcePosition& source); + RplSourcePosition& operator=(const RplSourcePosition& source); +public: + QString toString() const; + int lineNo() const; + void setLineNo(int lineNo); + + int column() const; + void setColumn(int column); + + RplSourceUnit* sourceUnit() const; + void setSourceUnit(RplSourceUnit* sourceUnit); + +private: + RplSourceUnit* m_sourceUnit; + int m_lineNo; + int m_column; + const RplSourcePosition* m_caller; +}; + +class RplReader { +public: + RplReader(RplSource& source); + ~RplReader(); +public: + /** + * @brief Prepares the reading from a given source unit. + * + * @param unit name of the unit + * @return NULL: unit not known
+ * otherwise: an instance with the state of the reader + * for the source. This is normally a sub class of + * RplSourceUnit + */ + virtual RplSourceUnit* openSourceUnit(const QString& unit) = 0; + /** + * @brief Reads the first part of the next line into a given buffer. + * + * @param maxSize the maximum length of the read input. + * If a line is longer the next part must be read + * by fillBuffer() + * @param buffer IN/OUT: the read input will be appended here + * @param hasMore OUT: true: the line is longer than maxSize + * @return true: the read was successful
+ * false: no more input is available + */ + virtual bool nextLine(int maxSize, QString& buffer, bool& hasMore) = 0; + /** + * @brief Reads the next part of the current line into a given buffer. + * + * @param maxSize the maximum length of the read input. + * @param buffer IN/OUT: the read input will be appended here + * @param hasMore OUT: true: the rest of line is longer than maxSize + * @return true: the read was successful
+ * false: no more input is available + */ + virtual bool fillBuffer(int maxSize, QString& buffer, bool& hasMore) = 0; +public: + RplSource& source(); + RplSourceUnit* currentSourceUnit() const; + bool setCurrentSourceUnit(const QString& currentSourceUnit); +protected: + void removeSourceUnit(); + +protected: + RplSourceUnit* m_currentSourceUnit; + /// name -> source + QMap m_units; + RplSource& m_source; +}; + + +class RplSource { +public: + RplSource(); + virtual ~RplSource(); +public: + virtual const char* permanentUnitName(const char* unit); + void finishSourceUnit(); + void addReader(RplReader* reader); + void addSourceUnit(RplSourceUnit* unit); + QStack sourcePositionStack() const; + QStack& sourceUnitStack(); + + bool startUnit(const QString& unit, const RplSourcePosition*); + void pushSourceUnit(RplSourceUnit* unit); + RplSourceUnit* popSourceUnit(RplReader* reader); + RplReader* currentReader(); +protected: + // stack of the info about the stacked (open) source units: + QStack m_sourcePositionStack; + QList m_sourcePositions; + QList m_readers; + QList m_sourceUnits; + // setCurrentSourceUnit() pushes one entry, removeSourceUnit() pops it + // (when end of input has been reached). + QStack m_unitStack; + RplReader* m_currentReader; +}; + +class RplStringReader; + +class RplStringSourceUnit : public RplSourceUnit { + friend class RplStringReader; +public: + RplStringSourceUnit(const QString& name, + const QString& content, RplStringReader* reader); + virtual ~RplStringSourceUnit(); +public: + int currentPosition() const; + void setCurrentPosition(int currentPosition); + const QString& content() const; + +private: + int m_currentPosition; + QString m_content; +}; + +class RplStringReader : public RplReader{ +public: + RplStringReader(RplSource& source); + virtual ~RplStringReader(); + // RplReader interface +public: + virtual RplSourceUnit* openSourceUnit(const QString& unit); + virtual bool nextLine(int maxSize, QString& buffer, bool& hasMore); + virtual bool fillBuffer(int maxSize, QString& buffer, bool& hasMore); +public: + void addSource(const QString& name, const QString& content); +}; + +class RplFileReader; + +class RplFileSourceUnit : public RplSourceUnit { + friend class RplFileReader; +public: + RplFileSourceUnit(const QDir& dir, RplFileReader* reader); + virtual ~RplFileSourceUnit(); +private: + int m_currentPosition; + QFile m_file; + QTextStream m_textStream; + QString m_line; +}; + +class RplFileReader : public RplReader{ +public: + RplFileReader(RplSource& source); + virtual ~RplFileReader(); + // RplReader interface +public: + virtual RplSourceUnit* openSourceUnit(const QString& unit); + virtual bool nextLine(int maxSize, QString& buffer, bool& hasMore); + virtual bool fillBuffer(int maxSize, QString& buffer, bool& hasMore); +public: + void addSource(const QDir& dirEntry); +private: + /// name -> source + QMap m_units; + RplFileSourceUnit* m_currentUnit; +}; + + +#endif // RPLSOURCE_HPP diff --git a/rplstatic/rplstatic.pro b/rplstatic/rplstatic.pro index 9d98af1..7269d87 100644 --- a/rplstatic/rplstatic.pro +++ b/rplstatic/rplstatic.pro @@ -30,6 +30,9 @@ SOURCES += \ ../rplnet/rpltcpclient.cpp \ ../rplnet/rpltcppeer.cpp \ ../rplnet/rpltcpserver.cpp \ + ../rplexpr/rpllexer.cpp \ + ../rplexpr/rplsource.cpp \ + ../rplcore/rplqstring.cpp HEADERS += ../rplmodules.hpp \ ../rplcore/rplconfig.hpp \ @@ -50,6 +53,10 @@ HEADERS += ../rplmodules.hpp \ ../rplnet/rpltcpclient.hpp \ ../rplnet/rpltcppeer.hpp \ ../rplnet/rpltcpserver.hpp \ + ../rplexpr/rpllexer.hpp \ + ../rplexpr/rplexpr.hpp \ + ../rplexpr/rplsource.hpp \ + ../rplcore/rplqstring.hpp unix:!symbian { maemo5 { diff --git a/unittests/main.cpp b/unittests/main.cpp index c370785..00d80d3 100644 --- a/unittests/main.cpp +++ b/unittests/main.cpp @@ -10,17 +10,39 @@ #include +void testCore(){ + extern void testRplQString(); + testRplQString(); + + extern void testRplString(); + testRplString(); + + extern void testRplException(); + testRplException(); +} + +void testExpr(){ + extern void testRplSource(); + testRplSource(); + +} + +void testStandard(){ + extern void testRplMatrix(); + testRplMatrix(); + + testExpr(); + testCore(); +} + int main(int argc, char *argv[]) { if (argc > 1) printf("not used: %s\n", argv[1]); - extern void testRplMatrix(); - testRplMatrix(); + extern void testRplLexer(); + testRplLexer(); - extern void testRplString(); - testRplString(); + testStandard(); - extern void testRplException(); - testRplException(); } diff --git a/unittests/rpllexer_test.cpp b/unittests/rpllexer_test.cpp new file mode 100644 index 0000000..95874d4 --- /dev/null +++ b/unittests/rpllexer_test.cpp @@ -0,0 +1,165 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + +#include "rplcore/rplcore.hpp" +#include "rplexpr/rplexpr.hpp" +#include "rplcore/rpltest.hpp" + +class TestRplLexer : public RplTest, public RplToken{ +public: + TestRplLexer() : + RplTest("RplLexer"), + RplToken(TOKEN_ID) + {} + +public: + void testRplToken(){ + // test constructor values: + checkE(TOKEN_ID, tokenType()); + checkE(0, m_value.m_id); + checkT(m_string.isEmpty()); + checkT(m_rawString.isEmpty()); + + m_value.m_id = 7422; + checkE(7422, RplToken::id()); + m_string = "Wow!"; + checkE("Wow!", RplToken::toString()); + m_rawString = "GooGoo"; + checkE("GooGoo", rawString()); + m_tokenType = TOKEN_NUMBER; + checkE(TOKEN_NUMBER, tokenType()); + + clear(); + checkE(TOKEN_UNDEF, tokenType()); + checkE(0, m_value.m_id); + checkT(m_string.isEmpty()); + checkT(m_rawString.isEmpty()); + + m_value.m_integer = 773322; + checkE(773322, asInteger()); + m_value.m_real = 0.25; + checkE(0.25, asReal()); + } + + RplToken* checkToken(RplToken* token, RplTokenType type, int id = 0, + const char* string = NULL){ + checkE(type, token->tokenType()); + if (id != 0) + checkE(id, token->id()); + if (string != NULL) + checkE(string, token->toString()); + return token; + } + enum { KEY_UNDEF, KEY_IF, KEY_THEN, KEY_ELSE, KEY_FI + }; +# define KEYWORDS "if then else fi" + enum { OP_UNDEF, OP_PLUS, OP_TIMES, OP_DIV, OP_ASSIGN, OP_GT, + OP_LT, OP_GE, OP_LE, OP_EQ + }; +# define OPERATORS "+ * / = > < >= <= ==" + enum { COMMENT_UNDEF, COMMENT_1 + }; +# define COMMENTS "# \n" + void testSpace(){ + RplSource source; + RplStringReader reader(source); +# define BLANKS1 "\t\t \n" +# define BLANKS2 " \n" + reader.addSource("
", BLANKS1 BLANKS2); + source.addReader(&reader); + RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS, + "A-Za-z_", + "A-Za-z0-9_", + RplLexer::NUMTYPE_DECIMAL, + RplLexer::SF_TICK, RplLexer::STORE_ALL); + checkToken(lex.nextToken(), TOKEN_SPACE, 0, BLANKS1); + checkToken(lex.nextToken(), TOKEN_SPACE, 0, BLANKS2); + } + void testNumeric(){ + RplSource source; + RplStringReader reader(source); + const char* blanks = "321 0x73 7.8e+5"; + reader.addSource("
", blanks); + source.addReader(&reader); + RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS, + "A-Za-z_", + "A-Za-z0-9_", + RplLexer::NUMTYPE_ALL, + RplLexer::SF_TICK, RplLexer::STORE_ALL); + RplToken* token = checkToken(lex.nextToken(), TOKEN_NUMBER); + checkE(321, token->asInteger()); + token = checkToken(lex.nextNonSpaceToken(), TOKEN_NUMBER); + checkE(0x73, token->asInteger()); + token = checkToken(lex.nextNonSpaceToken(), TOKEN_REAL); + checkE(7.8e+5, token->asReal()); + } + + void testOperators(){ + RplSource source; + RplStringReader reader(source); + const char* ops = "<< < <<< <= == = ( ) [ ]"; + reader.addSource("
", ops); + source.addReader(&reader); + enum { UNDEF, SHIFT, LT, SHIFT2, LE, EQ, ASSIGN, + LPARENT, RPARENT, LBRACKET, RBRACKET }; + RplLexer lex(&source, KEYWORDS, ops, COMMENTS, + "A-Za-z_", + "A-Za-z0-9_", + RplLexer::NUMTYPE_ALL, + RplLexer::SF_TICK, RplLexer::STORE_ALL); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, SHIFT); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LT); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, SHIFT2); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LE); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, EQ); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, ASSIGN); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LPARENT); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, RPARENT); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LBRACKET); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, RBRACKET); + checkToken(lex.nextNonSpaceToken(), TOKEN_END_OF_SOURCE); + reader.addSource("", "(([["); + lex.startUnit(""); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LPARENT); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LPARENT); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LBRACKET); + checkToken(lex.nextNonSpaceToken(), TOKEN_OPERATOR, LBRACKET); + checkToken(lex.nextNonSpaceToken(), TOKEN_END_OF_SOURCE); + } + + void testBasic(){ + RplSource source; + RplStringReader reader(source); + reader.addSource("
", "if i>1 then i=1+2*_x9 fi"); + RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS, + "A-Za-z_", + "A-Za-z0-9_" + ); + checkToken(lex.nextToken(), TOKEN_KEYWORD, KEY_IF); + checkToken(lex.nextToken(), TOKEN_SPACE, 0); + checkToken(lex.nextToken(), TOKEN_ID, 0, "i"); + checkToken(lex.nextToken(), TOKEN_OPERATOR, OP_GT); + checkToken(lex.nextToken(), TOKEN_NUMBER, 0, "1"); + checkToken(lex.nextToken(), TOKEN_SPACE, 0); + checkToken(lex.nextToken(), TOKEN_KEYWORD, KEY_THEN); + checkToken(lex.nextToken(), TOKEN_SPACE, 0); + + } + + virtual void doIt(void) { + testOperators(); + testNumeric(); + testSpace(); + testBasic(); + testRplToken(); + } +}; +void testRplLexer() { + TestRplLexer test; + test.run(); +} diff --git a/unittests/rplqstring_test.cpp b/unittests/rplqstring_test.cpp new file mode 100644 index 0000000..c2f7698 --- /dev/null +++ b/unittests/rplqstring_test.cpp @@ -0,0 +1,112 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ +#include "rplcore/rplcore.hpp" +#include "rplcore/rpltest.hpp" + +class TestRplQString : public RplTest { +public: + TestRplQString() : + RplTest("RplQString") + {} + +public: + void testLengthOfUInt64(){ + quint64 value = -3; + checkE(1, RplQString::lengthOfUInt64(QString("0"), 0, 10, &value)); + checkE(0LL, value); + checkE(3, RplQString::lengthOfUInt64("x432", 1, 10, &value)); + checkE(432LL, value); + checkE(3, RplQString::lengthOfUInt64("x432 x", 1, 10, &value)); + checkE(432LL, value); + checkE(3, RplQString::lengthOfUInt64("x432fabc x", 1, 10, &value)); + checkE(432LL, value); + checkE(16, RplQString::lengthOfUInt64("a1234567890123567", 1, 10, &value)); + checkE(1234567890123567LL, value); + checkE(10, RplQString::lengthOfUInt64("x1234abcdef", 1, 16, &value)); + checkE(0x1234abcdefLL, value); + checkE(3, RplQString::lengthOfUInt64("432", 0, 8, &value)); + checkE(0432LL, value); + checkE(6, RplQString::lengthOfUInt64(" 765432 ", 1, 8, &value)); + checkE(0765432LL, value); + + checkE(0, RplQString::lengthOfUInt64("1 ", 1, 8, &value)); + checkE(0, RplQString::lengthOfUInt64("", 1, 8, &value)); + } + void testLengthOfUInt(){ + uint value = 3; + checkE(1, RplQString::lengthOfUInt(QString("0"), 0, 10, &value)); + checkE(0, value); + checkE(3, RplQString::lengthOfUInt("x432", 1, 10, &value)); + checkE(432, value); + checkE(3, RplQString::lengthOfUInt("x432 x", 1, 10, &value)); + checkE(432, value); + checkE(3, RplQString::lengthOfUInt("x432fabc x", 1, 10, &value)); + checkE(432, value); + checkE(3, RplQString::lengthOfUInt("432", 0, 8, &value)); + checkE(0432, value); + checkE(6, RplQString::lengthOfUInt(" 765432 ", 1, 8, &value)); + checkE(0765432, value); + + checkE(0, RplQString::lengthOfUInt("1 ", 1, 8, &value)); + checkE(0, RplQString::lengthOfUInt("", 1, 8, &value)); + } + void testLengthOfReal(){ + qreal value; + checkE(4, RplQString::lengthOfReal(QString("0.25"), 0, &value)); + checkE(0.25, value); + checkE(3, RplQString::lengthOfReal(QString("X.25"), 1, &value)); + checkE(0.25, value); + checkE(1, RplQString::lengthOfReal(QString(" 0"), 1, &value)); + checkE(0.0, value); + checkE(17, RplQString::lengthOfReal(QString("X12345678901234567"), 1, &value)); + checkE(12345678901234567.0, value); + checkE(2, RplQString::lengthOfReal(QString(".5"), 0, &value)); + checkE(0.5, value); + checkE(5, RplQString::lengthOfReal(QString("2.5e2x"), 0, &value)); + checkE(250.0, value); + checkE(6, RplQString::lengthOfReal(QString("2.5e+2"), 0, &value)); + checkE(250.0, value); + checkE(7, RplQString::lengthOfReal(QString("2.5E-33"), 0, &value)); + checkE(2.5e-33, value); + + checkE(3, RplQString::lengthOfReal(QString("2.5E"), 0, &value)); + checkE(2.5, value); + checkE(3, RplQString::lengthOfReal(QString("2.5E+"), 0, &value)); + checkE(2.5, value); + checkE(3, RplQString::lengthOfReal(QString("2.5E-a"), 0, &value)); + checkE(2.5, value); + } + + void testValueOfHexDigit(){ + checkE(0, RplQString::valueOfHexDigit('0')); + checkE(9, RplQString::valueOfHexDigit('9')); + checkE(10, RplQString::valueOfHexDigit('a')); + checkE(15, RplQString::valueOfHexDigit('f')); + checkE(10, RplQString::valueOfHexDigit('A')); + checkE(15, RplQString::valueOfHexDigit('F')); + + checkE(-1, RplQString::valueOfHexDigit('0' - 1)); + checkE(-1, RplQString::valueOfHexDigit('9' + 1)); + checkE(-1, RplQString::valueOfHexDigit('A' - 1)); + checkE(-1, RplQString::valueOfHexDigit('F' + 1)); + checkE(-1, RplQString::valueOfHexDigit('a' - 1)); + checkE(-1, RplQString::valueOfHexDigit('f' + 1)); + } + + virtual void doIt(void) { + testLengthOfUInt64(); + testLengthOfUInt(); + testLengthOfReal(); + testValueOfHexDigit(); + } +}; +void testRplQString() { + TestRplQString test; + test.run(); +} + diff --git a/unittests/rplsource_test.cpp b/unittests/rplsource_test.cpp new file mode 100644 index 0000000..cc323e6 --- /dev/null +++ b/unittests/rplsource_test.cpp @@ -0,0 +1,96 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + +#include "rplcore/rplcore.hpp" +#include "rplexpr/rplexpr.hpp" +#include "rplcore/rpltest.hpp" + +class TestRplSource : public RplTest{ +public: + TestRplSource() : RplTest("TestRplSource") {} + +private: + QString m_content1_1; + QString m_content1_2; + QString m_content2; + RplSource m_source; + +protected: + void init(){ + m_content1_1 = "# test\nimport source2\n"; + m_content1_2 = "a=1;\nveeeeeeeeery looooooooooooooong\n"; + m_content2 = "x=2"; + } + + void testRplStringSourceUnit(){ + RplStringReader reader(m_source); + QString content("a=1;\nveeeeeeeeery looooooooooooooong\n"); + RplStringSourceUnit unit(QString("test"), content, &reader); + unit.setLineNo(144); + checkE(144, unit.lineNo()); + checkE("test", unit.name()); + } + void checkOne(int maxSize, RplReader& reader){ + QString total; + QString buffer; + int lineNo = 0; + bool hasMore; + checkF(reader.openSourceUnit("unknownSource")); + checkT(reader.openSourceUnit("source1")); + while(reader.nextLine(maxSize, buffer, hasMore)){ + lineNo++; + total += buffer; + buffer.clear(); + while(hasMore && reader.fillBuffer(maxSize, buffer, hasMore)){ + total += buffer; + buffer.clear(); + } + bool isImport = total.endsWith("source2\n"); + if (isImport){ + reader.openSourceUnit("source2"); + checkE("source2", reader.currentSourceUnit()->name()); + while(reader.nextLine(maxSize, buffer, hasMore)){ + lineNo++; + while(hasMore && reader.fillBuffer(maxSize, buffer, hasMore)){ + total += buffer; + buffer.clear(); + } + } + checkE("source1", reader.currentSourceUnit()->name()); + } + } + checkE(5, lineNo); + checkE(m_content1_1 + m_content2 + m_content1_2, total); + + } + + void testRplStringReader(){ + RplStringReader reader(m_source); + reader.addSource("source1", m_content1_1 + m_content1_2); + reader.addSource("source2", m_content2); + RplSourceUnit* unit = reader.openSourceUnit("source1"); + checkE("source1", unit->name()); + checkE(0, unit->lineNo()); + checkOne(6, reader); + checkOne(100, reader); + } + +public: + virtual void doIt(void) { + init(); + testRplStringSourceUnit(); + testRplStringReader(); + } +}; +void testRplSource() { + TestRplSource test; + test.run(); +} + + + diff --git a/unittests/unittests.pro b/unittests/unittests.pro index 742617b..97dbea9 100644 --- a/unittests/unittests.pro +++ b/unittests/unittests.pro @@ -18,11 +18,17 @@ TEMPLATE = app SOURCES += main.cpp \ rplmatrix_test.cpp \ - ../rplmath/rplmatrix.cpp \ ../rplcore/rpllogger.cpp \ ../rplcore/rpltest.cpp \ ../rplcore/rplstring.cpp \ ../rplcore/rplexception.cpp \ + ../rplmath/rplmatrix.cpp \ + ../rplexpr/rplsource.cpp \ + ../rplexpr/rpllexer.cpp \ rplexception_test.cpp \ - rplstring_test.cpp + rplstring_test.cpp \ + rplsource_test.cpp \ + rpllexer_test.cpp \ + rplqstring_test.cpp \ + ../rplcore/rplqstring.cpp -- 2.39.5