From 53c79b1247256b0a8a08b3c9cd1409aba49323a2 Mon Sep 17 00:00:00 2001 From: hama Date: Fri, 27 Jun 2014 00:07:33 +0200 Subject: [PATCH] day's work --- rplexpr/rpllexer.cpp | 57 ++++++++++++++++++++++++++++++---- rplexpr/rpllexer.hpp | 2 ++ rplexpr/rplmfparser.cpp | 43 +++++++++++++------------ rplexpr/rplmfparser.hpp | 51 +++++++++++++++++++++--------- unittests/rpllexer_test.cpp | 27 ++++++++++++++-- unittests/rplmfparser_test.cpp | 38 +++++++++++++++++++++++ unittests/unittests.pro | 4 ++- 7 files changed, 178 insertions(+), 44 deletions(-) create mode 100644 unittests/rplmfparser_test.cpp diff --git a/rplexpr/rpllexer.cpp b/rplexpr/rpllexer.cpp index c2fc2c1..d7ad9a2 100644 --- a/rplexpr/rpllexer.cpp +++ b/rplexpr/rpllexer.cpp @@ -260,7 +260,9 @@ static void charClassToCharInfo(const char* charClass, int flag, * @param source the input source handler * @param keywords a string with all keywords delimited by ' '. * Example: "if then else fi while do done" - * @param operators a string with all operators delimited by ' ' + * @param operators a string with the operators separated by blank or '\n'. + * '\n' separates the operators with the same priority. + * Lower position means lower priority * @param comments a string with pairs of comment begin and end delimited * by ' '. The comment end can be '\n' for line end. * Example: "/ * * / // \n" (ignore the blank in "* /") @@ -322,14 +324,36 @@ RplLexer::RplLexer(RplSource* source, RplLexer::~RplLexer() { } + +/** + * @brief Returns the count of blanks in a given range of a string. + * + * @param start pointer to the first char to check + * @param end pointer to the last char to check + * @return the count of blanks + */ +int countBlanks(const char* start, const char* end){ + int rc = 0; + while(start != end){ + if (*start++ == ' '){ + rc++; + } + } + return rc; +} + /** * @brief Stores the operators in the internal members * - * @param operators a string with the operators separated by blank + * @param operators a string with the operators separated by blank or '\n'. + * '\n' separates the operators with the same priority. + * Lower position means lower priority */ void RplLexer::prepareOperators(const char* operators){ - itemsToVector(operators, m_operators, CC_FIRST_OP, CC_2nd_OP, CC_3rd_OP, - CC_REST_OP, m_charInfo); + QByteArray op2(operators); + op2.replace("\n", " "); + itemsToVector(op2.constData(), m_operators, CC_FIRST_OP, CC_2nd_OP, + CC_3rd_OP, CC_REST_OP, m_charInfo); // m_operators is now sorted: // test whether the successor of 1 char operators is starting with this char: // if not this operator will be marked with CC_OP_1_ONLY: @@ -341,6 +365,24 @@ void RplLexer::prepareOperators(const char* operators){ m_charInfo[id] |= CC_OP_1_ONLY; } } + const char* start = operators; + const char* end; + int prio = 0; + int endId = 0; + int startId = 1; + bool again = true; + while (again){ + if ( (end = strchr(start, '\n')) == NULL){ + end = strchr(start, '\0'); + again = false; + } + prio++; + endId = startId + countBlanks(start, end) + 1 - 1; + while(startId <= endId){ + m_prioOfOp[startId++] = prio; + } + start = end + 1; + } } void RplLexer::initializeComments(const char* comments) @@ -890,8 +932,11 @@ RplSource* RplLexer::source() * @param op the operator * @return the priority of the op */ -int RplLexer::prioOfOp(int op){ - int prio = m_prioOfOp[op]; +int RplLexer::prioOfOp(int op) const +{ + int rc = op > 0 && op < sizeof m_prioOfOp / sizeof m_prioOfOp[0] + ? m_prioOfOp[op] : 0; + return rc; } diff --git a/rplexpr/rpllexer.hpp b/rplexpr/rpllexer.hpp index 06b8bf5..49a11e6 100644 --- a/rplexpr/rpllexer.hpp +++ b/rplexpr/rpllexer.hpp @@ -170,6 +170,7 @@ public: void setMaxTokenLength(size_t maxTokenLength); void startUnit(const QString& unit); RplSource* source(); + int prioOfOp(int op) const; private: void prepareOperators(const char* operators); void initializeComments(const char* comments); @@ -213,6 +214,7 @@ protected: bool m_hasMoreInput; int m_stringFeatures; int m_storageFlags; + /// priority of the operators: index: id of the operator. content: prio char m_prioOfOp[128]; }; diff --git a/rplexpr/rplmfparser.cpp b/rplexpr/rplmfparser.cpp index 769ea8b..69b56ed 100644 --- a/rplexpr/rplmfparser.cpp +++ b/rplexpr/rplmfparser.cpp @@ -19,13 +19,13 @@ * for matrix operations, simulation and graphics. */ -RplMFParser::RplMFParser(RplSource& source) : +RplMFParser::RplMFParser(RplSource& source, RplASTree& abstractSyntaxTree) : m_lexer(&source, MF_KEYWORDS, MF_OPERATORS, "/* */ // \n", "a-zA-Z_", "a-zA-Z0-9_", - NUMTYPE_ALL, SF_C_ALL - ) + RplLexer::NUMTYPE_ALL, RplLexer::SF_LIKE_C), + m_tree(abstractSyntaxTree) { } @@ -74,7 +74,7 @@ void RplMFParser::parseFor() * @param clazz NULL or the type of the variable * @param attribute 0 or attribute of the variable: K_CONST or K_LAZY */ -void RplMFParser::parseDefinition(RplASClass* clazz, Keyword attribute = K_UNDEF) +void RplMFParser::parseDefinition(RplASClass* clazz, Keyword attribute) { } @@ -88,7 +88,7 @@ void RplMFParser::parseDefinition(RplASClass* clazz, Keyword attribute = K_UNDEF */ RplASItem* RplMFParser::parseOperand(int level) { - RplToken* token = m_lexer->nextNonSpaceToken(); + RplToken* token = m_lexer.nextNonSpaceToken(); RplASItem* rc = NULL; switch(token->tokenType()){ case TOKEN_OPERATOR: @@ -119,11 +119,11 @@ RplASItem* RplMFParser::parseOperand(int level) case TOKEN_ID: { QString name = token->toString(); - token = m_lexer->nextNonSpaceToken(); + token = m_lexer.nextNonSpaceToken(); if (token->tokenType() != TOKEN_OPERATOR){ RplASNamedValue* var = new RplASNamedValue(name); item = var; - m_lexer->undoLastToken(); + m_lexer.undoLastToken(); } else { if (token->id() == O_LPARENTH){ RplASItem* args = parseArguments(); @@ -155,24 +155,27 @@ RplASItem* RplMFParser::parseOperand(int level) */ RplASItem* RplMFParser::parseTerm(int depth){ RplToken* token; - RplSourcePosition* start = m_lexer->currentSourcePosition(); + RplSourcePosition* start = m_lexer.currentSourcePosition(); RplASItem* top = NULL; RplASItem* item; int lastPrio = -1; bool again = true; do { item = parseOperand(level); - token = m_lexer->nextNonSpaceToken(); - switch(token->tokenType()){ + token = m_lexer.nextNonSpaceToken(); + RplTokenType tokenType = token->tokenType(); + switch(tokenType){ case TOKEN_OPERATOR: { - RplASBinaryOp* op = new RplASBinaryOp(); - op->setOp(token->id()); - int prio = m_lexer->prioOfOp(token->id()); - op->setChild(item); - op->setPosition(m_lexer->currentPosition()); - op->setChild2(parseOperand(level)); - if + if (IS_BINARY_OP(tokenType)){ + RplASBinaryOp* op = new RplASBinaryOp(); + op->setOp(token->id()); + int prio = m_lexer.prioOfOp(token->id()); + op->setChild(item); + op->setPosition(m_lexer.currentPosition()); + op->setChild2(parseOperand(level)); + if + } break; } case TOKEN_STRING: @@ -212,14 +215,14 @@ RplToken* RplMFParser::parseExpr() */ void RplMFParser::parseBody() { - RplToken token = m_lexer->nextNonSpaceToken(); + RplToken token = m_lexer.nextNonSpaceToken(); switch(token.tokenType()) { case TOKEN_STRING: case TOKEN_NUMBER: case TOKEN_REAL: case TOKEN_OPERATOR: - m_lexer->undoNextToken(); + m_lexer.undoNextToken(); token = parseExpr(); break; case TOKEN_KEYWORD: @@ -269,7 +272,7 @@ void RplMFParser::parseBody() if (clazz != NULL){ parseDefinition(clazz); } else { - m_lexer->undoNextToken(); + m_lexer.undoNextToken(); parseExpr(); } break; diff --git a/rplexpr/rplmfparser.hpp b/rplexpr/rplmfparser.hpp index b4093d3..fbe938d 100644 --- a/rplexpr/rplmfparser.hpp +++ b/rplexpr/rplmfparser.hpp @@ -24,20 +24,43 @@ public: " class end function generator import" \ " const lazy int float bool None True False" enum Operator { - O_UNDEF, O_SEMI_SEMICOLON, O_SEMICOLON, O_COMMA, O_COLON, O_QUESTION, + O_UNDEF, O_SEMI_SEMICOLON, O_SEMICOLON, O_COMMA, O_COLON, + O_ASSIGN, O_PLUS_ASSIGN, O_MINUS_ASSIGN, O_DIV_ASSIGN, O_TIMES_ASSIGN, + O_MOD_ASSIGN, O_POWER_ASSIGN, O_OR_ASSIGN, O_AND_ASSIGN, + O_LSHIFT_ASSIGN, O_RSHIFT_ASSIGN, O_RSHIFT2_ASSIGN, O_OR, O_AND, - O_XOR, O_BIT_OR, O_BIT_AND, O_LSHIFT, O_RSHIFT, O_RSHIFT2, - O_LT, O_GT, O_LE, O_GE, O_EQ, O_NE, - O_PLUS, O_MINUS, O_DIV, O_MOD, O_TIMES, O_POWER, O_INC, O_DEC, - O_DOT, O_LPARENTH, O_RPARENT, O_LBRACKET, O_RBRACKET + O_EQ, O_NE, + O_LT, O_GT, O_LE, O_GE, + O_QUESTION, + O_PLUS, O_MINUS, + O_DIV, O_MOD, O_TIMES, + O_POWER, + O_XOR, O_BIT_OR, O_BIT_AND, + O_LSHIFT, O_RSHIFT, O_RSHIFT2, + O_DOT, + O_NOT, O_BIT_NOT, + O_INC, O_DEC, + O_LPARENTH, O_RPARENT, O_LBRACKET, O_RBRACKET }; - -#define MF_OPERATORS ";; ; , : ?" \ - " || && " \ - " ^ | & << >> >>>" \ - " < > <= >= == !=" \ - " + - / % * ** ++ --" \ - " . ( ) [ ]" +#define IS_BINARY_OP(op) ((op) >= O_ASSIGN && op <= O_DOT) +#define IS_UNARY_OP(op) (op==O_PLUS || op==O_MINUS || op>=O_NOT && op<=O_DEC) +/// \n separates the priority classes +#define MF_OPERATORS ";; ; , :\n" \ + "= += -= /= *= %= **= |= &= <<= >>= >>>=\n" \ + "||\n" \ + "&&\n" \ + "== !=\n" \ + "< > <= >=\n" \ + "?\n" \ + "+ -\n" \ + "/ % *\n" \ + "**\n" \ + "^ | &\n" \ + "<< >> >>>\n" \ + ".\n" \ + "! ~\n" \ + "++ --\n" \ + ". ( ) [ ]" public: RplMFParser(RplSource& source, RplASTree& ast); public: @@ -46,13 +69,13 @@ public: void parseWhile(); void parseRepeat(); void parseFor(); - void parseDefinition(RplASClass* clazz); + void parseDefinition(RplASClass* clazz, Keyword attribute); RplToken* parseExpr(); void parseBody(); void parseClass(); void parseModule(const QString& name); void parse(); - RplASItem* parseOperand(); + RplASItem* parseOperand(int level); RplASItem* parseTerm(int depth); private: RplLexer m_lexer; diff --git a/unittests/rpllexer_test.cpp b/unittests/rpllexer_test.cpp index a88e9fa..93998c5 100644 --- a/unittests/rpllexer_test.cpp +++ b/unittests/rpllexer_test.cpp @@ -58,10 +58,11 @@ public: enum { KEY_UNDEF, KEY_IF, KEY_THEN, KEY_ELSE, KEY_FI }; # define KEYWORDS "if then else fi" - enum { OP_UNDEF, OP_PLUS, OP_TIMES, OP_DIV, OP_ASSIGN, OP_GT, - OP_LT, OP_GE, OP_LE, OP_EQ + enum { OP_UNDEF, OP_PLUS, OP_TIMES, OP_DIV, OP_GT, + OP_LT, OP_GE, OP_LE, OP_EQ, OP_ASSIGN, OP_PLUS_ASSIGN, OP_DIV_ASSIGN, + OP_TIMES_ASSIGN }; -# define OPERATORS "+ * / = > < >= <= ==" +# define OPERATORS "+\n* /\n> < >= <= ==\n= += /= *=" enum { COMMENT_UNDEF, COMMENT_1, COMMENT_MULTILINE, COMMENT_2 }; # define COMMENTS "/* */ // \n" @@ -236,8 +237,28 @@ public: checkToken(lex.nextToken(), TOKEN_SPACE, 0); } + void testPrio(){ + RplSource source; + RplStringReader reader(source); + source.addReader(&reader); + reader.addSource("x", ""); + enum { O_UNDEF, O_ASSIGN, O_PLUS, O_MINUS, O_TIMES, O_DIV + }; + RplLexer lex(&source, KEYWORDS, + "=\n+ -\n* /", + COMMENTS, + "A-Za-z_", + "A-Za-z0-9_", + RplLexer::NUMTYPE_ALL, + RplLexer::SF_LIKE_C, RplLexer::STORE_ALL); + checkT(lex.prioOfOp(O_ASSIGN) < lex.prioOfOp(O_PLUS)); + checkE(lex.prioOfOp(O_PLUS), lex.prioOfOp(O_MINUS)); + checkT(lex.prioOfOp(O_MINUS) < lex.prioOfOp(O_TIMES)); + checkE(lex.prioOfOp(O_TIMES), lex.prioOfOp(O_DIV)); + } virtual void doIt(void) { + testPrio(); testBasic(); testIds(); testKeywords(); diff --git a/unittests/rplmfparser_test.cpp b/unittests/rplmfparser_test.cpp new file mode 100644 index 0000000..5a635d0 --- /dev/null +++ b/unittests/rplmfparser_test.cpp @@ -0,0 +1,38 @@ +/* + * Licence: + * You can use and modify this file without any restriction. + * There is no warranty. + * You also can use the licence from http://www.wtfpl.net/. + * The original sources can be found on https://github.com/republib. +*/ + +#include "rplcore/rplcore.hpp" +#include "rplexpr/rplexpr.hpp" +#include "rplcore/rpltest.hpp" + +class TestRplMFParser : public RplTest, public RplMFParser{ +private: + RplSource m_source; + RplASTree m_tree; +public: + TestRplMFParser() : + RplTest("RplMFParser"), + RplMFParser(m_source, m_tree), + m_source(), + m_tree() + {} + +public: + void baseTest(){ + + } + + virtual void doIt(void) { + } +}; +void testRplMFParser() { + TestRplMFParser test; + test.run(); +} + + diff --git a/unittests/unittests.pro b/unittests/unittests.pro index 6edd0ec..29e0bce 100644 --- a/unittests/unittests.pro +++ b/unittests/unittests.pro @@ -26,6 +26,7 @@ SOURCES += main.cpp \ ../rplexpr/rplsource.cpp \ ../rplexpr/rpllexer.cpp \ ../rplexpr/rplastree.cpp \ + ../rplexpr/rplmfparser.cpp \ rplexception_test.cpp \ rplstring_test.cpp \ rplsource_test.cpp \ @@ -33,5 +34,6 @@ SOURCES += main.cpp \ rplqstring_test.cpp \ ../rplcore/rplqstring.cpp \ ../rplexpr/rplasclasses.cpp \ - rplastree_test.cpp + rplastree_test.cpp \ + rplmfparser_test.cpp -- 2.39.5