From 5ac49798dc1ec15840b7625a0eedda612bc9cdd0 Mon Sep 17 00:00:00 2001 From: hama Date: Sun, 3 Jan 2016 09:27:55 +0100 Subject: [PATCH] dayly work --- appl/recform/CFormatter.cpp | 339 +++++++++++++++++++++++++------- appl/recform/CFormatter.hpp | 25 ++- appl/recform/CppParser.cpp | 2 +- appl/recform/cuReCFormatter.cpp | 107 ++++++++-- appl/recform/recformmain.cpp | 2 +- expr/ReLexer.cpp | 42 ++-- expr/ReLexer.hpp | 11 +- 7 files changed, 418 insertions(+), 110 deletions(-) diff --git a/appl/recform/CFormatter.cpp b/appl/recform/CFormatter.cpp index fd14e84..96f2158 100644 --- a/appl/recform/CFormatter.cpp +++ b/appl/recform/CFormatter.cpp @@ -30,14 +30,25 @@ CFormatter::CFormatter(ReProgramArgs& args, ReLogger* logger) : m_continued(false), m_writer(NULL), m_logger(logger), + m_maxLineLength(80), + m_maxRestLineLength(80), m_tabSize(4), m_useTab(true), m_lexer(NULL), m_parser(NULL), - m_countIsTypeList(1024), - m_isInTypeList(new bool[m_countIsTypeList]) + m_countIsTypeList(256), + m_isInTypeList(new bool[m_countIsTypeList]), + m_countTokenLengths(256), + m_tokenLengths(new int[m_countTokenLengths]) { } +/** + * Destructor. + */ +CFormatter::~CFormatter(){ + delete[] m_isInTypeList; + delete[] m_tokenLengths; +} /** * @file @@ -65,10 +76,49 @@ void CFormatter::addToken(ReToken* token) void CFormatter::findTypeLists() { const FormatToken* item2; + int count = m_logicalLine.size(); + // Search for variable definition: + // examples: char* const x; String* x[8*10]; List z = 1+2; + int endOfTypelist = count - 1; + CppOperator op; + for (int ix = 0; ix < count; ix++){ + const FormatToken& item = m_logicalLine.constData()[ix]; + if (item.isOperator(OP_ASSIGN)){ + endOfTypelist = ix - 1; + break; + } + if (item.isTokenType(TOKEN_OPERATOR)){ + op = (CppOperator) item.id(); + if (op == OP_LBRACKET){ + while(++ix < count){ + const FormatToken& item2 = m_logicalLine.constData()[ix]; + if (item2.isOperator(OP_RBRACKET)) + break; + } + } else if (! (op == OP_STAR || op == OP_BIT_AND || op == OP_LT + || op == OP_GT)){ + endOfTypelist = -1; + break; + } + } + } + for(int ix = endOfTypelist; ix >= 0; ix--){ + FormatToken const& item = m_logicalLine.constData()[ix]; + if (item.isOperator(OP_RBRACKET)){ + while(--ix >= 0){ + FormatToken const& item2 = m_logicalLine.constData()[ix]; + if (item2.isOperator(OP_LBRACKET)) + break; + } + } else { + m_isInTypeList[ix] = true; + } + } + if (endOfTypelist < 0) + endOfTypelist = 0; // search for cast expressions with '*' and/or '&' and/or "[]": // examples: (char&) (int[]) (int* const) - int count = m_logicalLine.size(); - for (int ix = count - 1; ix > 0; ix--){ + for (int ix = count - 1; ix > endOfTypelist; ix--){ const FormatToken& item = m_logicalLine.constData()[ix]; // at least 3 elements: '(' "int" '*' if (item.isOperator(OP_RPARENTH) && ix >= 3){ @@ -83,7 +133,7 @@ void CFormatter::findTypeLists() } // search for new expressions: // examples: new String; new Type*[3]; new Type(1, 2); - for (int ix = 0; ix < count; ix++){ + for (int ix = endOfTypelist; ix < count; ix++){ const FormatToken& item = m_logicalLine.constData()[ix]; if (item.isKeyword(K_NEW)){ while(++ix < count && ((item2 = &m_logicalLine.constData()[ix]) @@ -94,7 +144,7 @@ void CFormatter::findTypeLists() } // search for template expressions: '<' types '>': // examples: new QList - for (int ix = 0; ix < count; ix++){ + for (int ix = endOfTypelist; ix < count; ix++){ const FormatToken& item = m_logicalLine.constData()[ix]; if (item.isOperator(OP_LT)){ bool found = false; @@ -114,43 +164,99 @@ void CFormatter::findTypeLists() } } } - // Search for variable definition: - // examples: char* const x; String* x[8*10]; List z = 1+2; - int maxIx = count - 1; - CppOperator op; - for (int ix = 0; ix < count; ix++){ - const FormatToken& item = m_logicalLine.constData()[ix]; - if (item.isOperator(OP_ASSIGN)){ - maxIx = ix - 1; - break; - } - if (item.isTokenType(TOKEN_OPERATOR)){ - op = (CppOperator) item.id(); - if (op == OP_LBRACKET){ - while(++ix < count){ - const FormatToken& item2 = m_logicalLine.constData()[maxIx]; - if (item2.isOperator(OP_RBRACKET)) - break; +} + +/** + * Write the first part of the current locical line to the output media. + * + * @param maxIndex the index of the first token with a newline (in comment) + * @param rawLength the sum of all token lengths (until maxIndex) + */ +void CFormatter::flushLine(int maxIndex, int rawLength) +{ + FormatToken* lastItem; + int maxRestLineLength = m_maxRestLineLength; + while (maxIndex > 0){ + int maxIx = maxIndex; + if (rawLength > m_maxRestLineLength){ + maxIx = 0; + while (maxIx < maxIndex && m_tokenLengths[maxIx] < maxRestLineLength) + maxIx++; + int lastLevel = -1; + for (int ix = maxIx; ix > 0; ix--){ + if ((( (lastItem = &m_logicalLine[ix])->isTokenType(TOKEN_OPERATOR)) + && lastLevel < 0) + || (lastItem->level() < lastLevel + && m_tokenLengths[maxIx] - m_tokenLengths[ix - 1] + <= maxRestLineLength)){ + maxIx = ix - 1; + lastLevel = lastItem->level(); } - } else if (! (op == OP_STAR || op == OP_BIT_AND || op == OP_LT - || op == OP_GT)){ - maxIx = -1; - break; } } + lastItem = &m_logicalLine.data()[0]; + flushToken(lastItem); + for (int ix = 1; ix < maxIx; ix++){ + FormatToken* item = &m_logicalLine.data()[ix]; + if (m_outputBuffer.length() > 0 + && needsBlank(lastItem, item, m_isInTypeList[ix])) + m_outputBuffer.append(' '); + flushToken(item); + lastItem = item; + } + if (m_outputBuffer.length() > 0){ + flushBuffer(); + } + m_logicalLine.remove(0, maxIx + 1); + memmove(m_isInTypeList, m_isInTypeList + maxIx, + (maxIndex - maxIx + 1) * sizeof m_isInTypeList[0]); + memmove(m_tokenLengths, m_tokenLengths + maxIx, + (maxIndex - maxIx + 1) * sizeof m_tokenLengths[0]); + maxIndex -= maxIx; + rawLength -= m_tokenLengths[maxIx]; + maxRestLineLength = (m_blockLevel + 1) * m_tabSize; } - while(--maxIx >= 0){ - FormatToken const& item = m_logicalLine.constData()[maxIx]; - if (item.isOperator(OP_RBRACKET)){ - while(--maxIx >= 0){ - FormatToken const& item2 = m_logicalLine.constData()[maxIx]; - if (item2.isOperator(OP_LBRACKET)) - break; - } - } else { - m_isInTypeList[maxIx] = true; +} + +/** + * Calculates the sum of the token length for a given set of tokens. + * + * @param maxIx IN/OUT: the index of the last token to inspect
+ * OUT: index of the first token with a comment starting + * with a newline + * @return the (formatted) length of the tokens (until maxIx) + */ +int CFormatter::calcLength(int& maxIx){ + int length = 0; + if (maxIx > m_countTokenLengths){ + delete[] m_tokenLengths; + m_countTokenLengths = max(m_countTokenLengths + 256, maxIx + 1); + m_tokenLengths = new int[m_countTokenLengths]; + } + FormatToken* lastItem = &m_logicalLine.data()[0]; + length += lastItem->string().length(); + if (lastItem->comment().length() > 0){ + if (lastItem->comment().at(0) == '\n') + maxIx = 0; + else + length += 1 + lastItem->comment().length(); + } + m_tokenLengths[0] = length; + for (int ix = 1; ix < maxIx; ix++){ + FormatToken* item = &m_logicalLine.data()[ix]; + if (needsBlank(lastItem, item, m_isInTypeList[ix])) + length += 1; + length += item->string().length(); + if (item->comment().length() > 0){ + if (lastItem->comment().at(0) == '\n') + maxIx = ix; + else + length += 1 + lastItem->comment().length(); } + m_tokenLengths[ix] = m_tokenLengths[ix - 1] + length; + lastItem = item; } + return length; } /** @@ -162,25 +268,101 @@ void CFormatter::flush(bool isPart) { resetIsTypeList(); findTypeLists(); - if (m_logicalLine.size() > 0){ - QByteArray buffer; - buffer.reserve(8096); - indent(m_blockLevel + m_continued ? 1 : 0, buffer); - FormatToken* lastItem = &m_logicalLine.data()[0]; - m_lexer->textOfToken(lastItem, buffer); - for (int ix = 1; ix < m_logicalLine.size(); ix++){ - FormatToken* item = &m_logicalLine.data()[ix]; - if (needsBlank(lastItem, item, m_isInTypeList[ix])) - buffer.append(' '); - m_lexer->textOfToken(item, buffer); - lastItem = item; - } - ReStringUtils::chomp(buffer, ' '); - m_writer->writeLine(buffer); - m_logicalLine.clear(); + FormatToken* token; + if (m_logicalLine.size() > 0 && (token = &m_logicalLine.data()[0]) + ->isTokenType(TOKEN_COMMENT_START)){ + flushComment(token->string()); + flushBuffer(); + m_logicalLine.remove(0, 1); + } + while (m_logicalLine.size() > 0){ + int maxIx = m_logicalLine.size() - 1; + int rawLength = calcLength(maxIx); + flushLine(maxIx, rawLength); } m_continued = isPart; } +/** + * Writes the buffer (m_outputBuffer) to the output media. + */ +void CFormatter::flushBuffer() +{ + if (m_outputBuffer.length() == 0) + m_writer->writeLine(); + else { + int count = m_blockLevel + m_continued ? 1 : 0; + if (m_useTab) + m_indentBuffer.fill('\t', count); + else + m_indentBuffer.fill(' ', count * m_tabSize); + m_writer->write(m_indentBuffer.constData()); + m_writer->writeLine(m_outputBuffer.constData()); + m_continued = true; + } + m_maxRestLineLength = (m_blockLevel + m_continued) * m_tabSize; +} + +/** + * Writes a comment to the m_outputBuffer. + * + * @param comment the comment text. If starting with '\n' the comment must be + * put in the next line + */ +void CFormatter::flushComment(QByteArray comment) +{ + if (comment.at(0) == '\n'){ + flushBuffer(); + comment.remove(0, 1); + } + bool newlineAtEnd = comment.at(comment.length() - 1) == '\n'; + ReStringUtils::chomp(comment); + if (comment.indexOf('\n') < 0) + m_outputBuffer.append(' ').append(comment); + else { + QByteArrayList lines = comment.split('\n'); + bool docComment = comment.startsWith("/**"); + QByteArray line; + for (int ix = 0; ix < lines.size(); ix++){ + line = lines.at(ix).trimmed(); + if (docComment){ + if (ix > 0) + m_outputBuffer.append(' '); + if (! line.startsWith('*')) + m_outputBuffer.append("* "); + else if (! line.startsWith("* ")){ + line.insert(1, ' '); + } + } + m_outputBuffer.append(line); + } + } + if (newlineAtEnd && m_outputBuffer.length() > 0) + flushBuffer(); +} +/** + * Puts a token and its following comment to the m_outputBuffer. + * + * Line splitting is done if the max. line length is exceeded. + * + * @param token the token + */ +void CFormatter::flushToken(FormatToken* token){ + int length = token->string().size(); + int commentSize = token->comment().size(); + if (commentSize > 0 && token->comment().at(0) != '\n'){ + length += 1 + commentSize; + } + if (m_outputBuffer.length() == 0 || length > m_maxRestLineLength){ + flushBuffer(); + } + m_outputBuffer.append(token->string()); + if (commentSize > 0){ + if (token->comment().at(0) == '\n'){ + flushBuffer(); + } + flushComment(token->comment()); + } +} /** * Formats a C++ file. @@ -203,23 +385,6 @@ void CFormatter::formatAll() } } -/** - * Writes the indention characters (tabs or blanks) into a buffer. - * - * @param level the level to indent - * @param buffer OUT: the buffer to append - */ -void CFormatter::indent(int level, QByteArray& buffer){ - int count = m_blockLevel; - if (m_continued) - count++; - int cols = count * m_tabSize; - if (! m_useTab) - count *= m_tabSize; - while(--count > 0) - buffer.append(m_useTab ? '\t' : ' '); -} - /** * Handles a label. * @@ -297,7 +462,7 @@ bool CFormatter::needsBlank(ReToken* first, ReToken* second, bool isDeclaration) case TOKEN_ID: if ( (op = (CppOperator) first->id()) == OP_RPARENTH) rc = true; - else if (isDeclaration && op == OP_GT) + else if (isDeclaration && op == OP_LT) rc = false; else rc = needsTrailingBlank((CppOperator) first->id()); @@ -307,7 +472,7 @@ bool CFormatter::needsBlank(ReToken* first, ReToken* second, bool isDeclaration) rc = needsTrailingBlank((CppOperator) first->id()) || needsPrecedingBlank((CppOperator) second->id()); else { - rc = false; + rc = second->isOperator(OP_ASSIGN); } break; default: @@ -335,7 +500,7 @@ void CFormatter::resetIsTypeList() { if (m_countIsTypeList < m_logicalLine.size()){ delete[] m_isInTypeList; - m_countIsTypeList = max(m_logicalLine.size(), 2 * m_countIsTypeList); + m_countIsTypeList = max(m_logicalLine.size() + 1, m_countIsTypeList + 256); m_isInTypeList = new bool[m_countIsTypeList]; } memset(m_isInTypeList, false, sizeof *m_isInTypeList * m_countIsTypeList); @@ -549,3 +714,31 @@ ReWriter* CFormatter::writer() const return m_writer; } +/** + * Returns the trailing comment. + * + * @return the comment + */ +const QByteArray& FormatToken::comment() const +{ + return m_comment; +} + +/** + * Sets the trailing comment. + * + * @param comment the comment to set + */ +void FormatToken::setComment(const QByteArray& comment) +{ + m_comment = comment; +} + +/** + * Returns the parenthesis level of the token + * @return the level of the token + */ +int FormatToken::level() const +{ + return m_level; +} diff --git a/appl/recform/CFormatter.hpp b/appl/recform/CFormatter.hpp index e1f73d5..a227ce9 100644 --- a/appl/recform/CFormatter.hpp +++ b/appl/recform/CFormatter.hpp @@ -36,10 +36,17 @@ public: */ FormatToken(ReToken* token, int level) : ReToken(*token), - m_level(level){ + m_level(level), + m_comment(){ } + const QByteArray& comment() const; + int level() const; + void setComment(const QByteArray& comment); + private: - int m_level; + short m_level; + /// the comment comes below the token. Can start and/or end with '\n' + QByteArray m_comment; }; class CppParser; class CppLexer; @@ -48,6 +55,7 @@ class CFormatter { public: CFormatter(ReProgramArgs& args, ReLogger* logger); + ~CFormatter(); public: void addToken(ReToken* token); void flush(bool isPart); @@ -66,8 +74,12 @@ public: ReWriter* writer() const; protected: + int calcLength(int& maxIx); void findTypeLists(); - void indent(int level, QByteArray& buffer); + void flushBuffer(); + void flushComment(QByteArray comment); + void flushLine(int maxIndex, int rawLength); + void flushToken(FormatToken* token); bool needsPrecedingBlank(CppOperator op); bool needsTrailingBlank(CppOperator op); bool needsBlank(ReToken* first, ReToken* second, bool isTypeList); @@ -82,6 +94,8 @@ private: bool m_continued; ReWriter* m_writer; ReLogger* m_logger; + int m_maxLineLength; + int m_maxRestLineLength; int m_tabSize; bool m_useTab; CppLexer* m_lexer; @@ -89,6 +103,11 @@ private: int m_countIsTypeList; // index: position of a token in m_logicalLine. Value: token is in type list. bool* m_isInTypeList; + QByteArray m_indentBuffer; + QByteArray m_outputBuffer; + int m_countTokenLengths; + /// index: ix value: sum of lengths of the tokens until (including) ix + int* m_tokenLengths; }; #endif // CFORMATTER_HPP diff --git a/appl/recform/CppParser.cpp b/appl/recform/CppParser.cpp index 5eca9ab..e7a3b62 100644 --- a/appl/recform/CppParser.cpp +++ b/appl/recform/CppParser.cpp @@ -306,7 +306,7 @@ CppLexer::CppLexer(ReSource* source) : | ReLexer::NUMTYPE_FLOAT, ReLexer::SF_TICK | ReLexer::SF_QUOTE | ReLexer::SF_C_ESCAPING | ReLexer::SF_C_SPECIAL | ReLexer::SF_C_HEX_CHARS, - ReLexer::STORE_ALL), + ReLexer::STORE_AS_FORMMATER), m_preview(false), m_queue(), m_indexQueue(0) diff --git a/appl/recform/cuReCFormatter.cpp b/appl/recform/cuReCFormatter.cpp index 9d3af70..9c355e9 100644 --- a/appl/recform/cuReCFormatter.cpp +++ b/appl/recform/cuReCFormatter.cpp @@ -39,30 +39,112 @@ public: m_formatter.addToken(token); } while (! token->isTokenType(TOKEN_END_OF_SOURCE)); } + void checkOp(const char* op1, CppOperator op2){ + m_reader.replaceSource("std", op1); + ReToken* token = m_lexer.nextNonSpaceToken(); + if (TOKEN_OPERATOR != token->tokenType()) + checkEqu(TOKEN_OPERATOR, token->tokenType()); + if (op2 != token->id()) + checkEqu(op2, token->id()); + checkEqu(op1, token->string()); + } + + void testOp(){ + /* ",\n? : = += -= /= %= <<= >>= &= ^= |=\n" + "||\n&&\n|\n^\n&\n<< >>\n< <= > >=\n== !=\n+ -\n* / %\n.* ->*\n" + "++ -- ! ~\n( ) [ ] . ->\n::\n{ } ;"; */ + checkOp(",", OP_COMMA); + checkOp("?", OP_QUESTIONMARK); + checkOp(":", OP_COLON); + checkOp("=", OP_ASSIGN); + checkOp("||", OP_OR); + checkOp("&&", OP_AND); + checkOp("&", OP_BIT_AND); + checkOp(">", OP_GT); + checkOp("<", OP_LT); + checkOp("==", OP_EQ); + checkOp("+", OP_PLUS); + checkOp("*", OP_STAR); + checkOp("++", OP_PLUS_PLUS); + checkOp("!", OP_NOT); + checkOp("(", OP_LPARENTH); + checkOp(")", OP_RPARENTH); + checkOp("[", OP_LBRACKET); + checkOp("]", OP_RBRACKET); + checkOp("{", OP_LBRACE); + checkOp("}", OP_RBRACE); + checkOp(";", OP_SEMICOLON); + } + void checkKeyword(const char* key1, CppKeyword key2){ + m_reader.replaceSource("std", key1); + ReToken* token = m_lexer.nextNonSpaceToken(); + if (TOKEN_KEYWORD != token->tokenType()) + checkEqu(TOKEN_KEYWORD, token->tokenType()); + if (key2 != token->id()) + checkEqu(key2, token->id()); + checkEqu(key1, token->string()); + } + void testKeywords() { + /* "alignas alignof asm auto bool break case catch " + "char char16_t char32_t class const constexpr const_cast continue " + "decltype default delete do double dynamic_cast else enum explicit " + "export extern false float for friend goto if inline int long mutable " + "namespace new noexcept nullptr operator private protected public " + "register reinterpret_cast return short signed sizeof static " + "static_assert static_cast struct switch template this thread_local " + "throw true try typedef typeid typename union unsigned using virtual " + "void volatile wchar_t while"; */ + checkKeyword("alignas", K_ALIGNAS); + checkKeyword("bool", K_BOOL); + checkKeyword("char", K_CHAR); + checkKeyword("class", K_CLASS); + checkKeyword("continue", K_CONTINUE); + checkKeyword("do", K_DO); + checkKeyword("double", K_DOUBLE); + checkKeyword("else", K_ELSE); + checkKeyword("enum", K_ENUM); + checkKeyword("false", K_FALSE); + checkKeyword("for", K_FOR); + checkKeyword("if", K_IF); + checkKeyword("int", K_INT); + checkKeyword("long", K_LONG); + checkKeyword("new", K_NEW); + checkKeyword("operator", K_OPERATOR); + checkKeyword("private", K_PRIVATE); + checkKeyword("protected", K_PROTECTED); + checkKeyword("public", K_PUBLIC); + checkKeyword("return", K_RETURN); + checkKeyword("static", K_STATIC); + checkKeyword("struct", K_STRUCT); + checkKeyword("switch", K_SWITCH); + checkKeyword("this", K_THIS); + checkKeyword("throw", K_THROW); + checkKeyword("true", K_TRUE); + checkKeyword("try", K_TRY); + checkKeyword("typedef", K_TYPEDEF); + checkKeyword("union", K_UNION); + checkKeyword("virtual", K_VIRTUAL); + checkKeyword("void", K_VOID); + checkKeyword("while", K_WHILE); + } void testBasic() { - const char* line = "(char*) abc(1 + 2);\n"; + const char* line; + line = "String ptr[3 + (4 / 8)] = f(z[ix++ - 1] + 1 - 2 * 5);\n"; setTokens(line); m_writer.buffer().clear(); - m_formatter.setLastDeclToken(-1); m_formatter.flush(true); checkEqu(line, m_writer.buffer()); line = "const char* ptr = (const char*) abc(1 + 2) * 3 / 4;\n"; setTokens(line); m_writer.buffer().clear(); - m_formatter.flush(true); - checkEqu(line, m_writer.buffer()); - - line = "String ptr[3+(4/8)] = x(z[ix++ - 1] + 1 - 2 * 5);\n"; - setTokens(line); - m_writer.buffer().clear(); m_formatter.setLastDeclToken(4); m_formatter.flush(true); checkEqu(line, m_writer.buffer()); } void testLexer(){ - m_reader.replaceSource("std", "const int x = 3;"); + m_reader.replaceSource("std", "const String x = \"a\n\x09\";"); CppLexer lexer(&m_source); lexer.setPreview(true); ReToken* token; @@ -83,14 +165,15 @@ public: token = lexer.nextNonSpaceToken(); checkT(token->isOperator(OP_ASSIGN)); token = lexer.nextNonSpaceToken(); - int value; - checkT(token->isInteger(&value)); - checkEqu(3, value); + checkEqu(TOKEN_STRING, token->tokenType()); + checkEqu("\"a\n\x09\"", token->string()); } virtual void runTests() { testBasic(); testLexer(); + testKeywords(); + testOp(); } ReProgramArgs m_args; ReSource m_source; diff --git a/appl/recform/recformmain.cpp b/appl/recform/recformmain.cpp index 6b58655..9625f80 100644 --- a/appl/recform/recformmain.cpp +++ b/appl/recform/recformmain.cpp @@ -24,7 +24,7 @@ int main(int argc, char *argv[]) "Formats c++ sources", "$0 -v"); args.addBool("verbose", "logs additional information", 'v', "verbose", false); args.init(argc, argv); - if (true || args.argCount() == 1 && strcmp(args.arg(0), "test") == 0){ + if (true /* || args.argCount() == 1 && strcmp(args.arg(0), "test") == 0*/){ extern void testCppFormatter(); testCppFormatter(); return 0; diff --git a/expr/ReLexer.cpp b/expr/ReLexer.cpp index fa12991..66cc75f 100644 --- a/expr/ReLexer.cpp +++ b/expr/ReLexer.cpp @@ -158,7 +158,7 @@ qreal ReToken::asReal() const { /** * @brief Returns the floating point value of the token * - * Only relevant if a TOKEN_NUMBER. + * Only relevant if a TOKEN_STRING. * * @return the value of the token as floating point value */ @@ -455,7 +455,7 @@ static void charClassToCharInfo(const char* charClass, int flag, * the operators which are right associative * @param comments a string with pairs of comment begin and end delimited * by " ". The comment end can be "\n" for line end. - * Example: "/ * * / // \n" (ignore the blank in "* /") + * Example: "/_* *_/ // \n" (ignore the underscore '_') * @param firstCharsId string with the characters which are allowed as first * characters of an id * @param restCharsId string with the characters which are allowed as non @@ -756,8 +756,8 @@ ReToken* ReLexer::findTokenWithId(ReTokenType tokenType, int flag2, rc = m_currentToken; rc->m_tokenType = tokenType; rc->m_value.m_id = id; - if (tokenType == TOKEN_COMMENT_START - && (m_storageFlags & STORE_COMMENT) != 0) + if (m_storeAll || (tokenType == TOKEN_COMMENT_START + && (m_storageFlags & STORE_COMMENT) != 0)) rc->m_string.append(m_input.mid(0, length)); m_input.remove(0, length); m_currentCol += length; @@ -842,6 +842,22 @@ ReToken*ReLexer::scanString() { m_currentToken->m_value.m_id = delim; bool again = false; do { + again = false; + if (m_storageFlags & STORE_ORG_STRING_ONLY){ + while (length < inputLength) { + if ( (cc = m_input[length]) != (char) delim){ + length++; + } else { + if ((m_stringFeatures & SF_DOUBLE_DELIM) != 0 + && length < inputLength + && m_input[length] == (char) delim) + length++; + else + break; + } + } + break; + } while (length < inputLength && (cc = m_input[length]) != delim) { length++; if (cc != '\\' @@ -903,12 +919,11 @@ ReToken*ReLexer::scanString() { } if (cc == delim) { length++; - } - if ((m_stringFeatures & SF_DOUBLE_DELIM) && length < inputLength - && m_input[length] == (char) delim) { - m_currentToken->m_printableString.append(delim); - length++; - again = true; + if ((m_stringFeatures & SF_DOUBLE_DELIM) && length < inputLength + && m_input[length] == (char) delim) { + length++; + again = true; + } } } while (again); if (m_storageFlags & STORE_ORG_STRING) @@ -1229,12 +1244,7 @@ void ReLexer::startUnit(ReSourceUnitName unit) { */ QByteArray& ReLexer::textOfToken(ReToken* token, QByteArray& buffer) { - if (token->tokenType() == TOKEN_OPERATOR) - buffer.append(nameOfOp(token->id())); - else if (token->tokenType() == TOKEN_KEYWORD) - buffer.append(nameOfKeyword(token->id())); - else - buffer.append(token->string()); + buffer.append(token->string()); return buffer; } /** diff --git a/expr/ReLexer.hpp b/expr/ReLexer.hpp index fb4c0e4..0c2ff67 100644 --- a/expr/ReLexer.hpp +++ b/expr/ReLexer.hpp @@ -150,16 +150,19 @@ public: }; enum StorageFlags { S_UNDEF, - /// the original string will be stored in m_string too - /// (not only m_rawString) + /// the original string will be stored unchanged in m_printableString too + /// (not only in m_string) STORE_ORG_STRING = 1 << 1, /// comments will be stored in m_string STORE_COMMENT = 1 << 2, /// blanks will be stored in m_string STORE_BLANK = 1 << 3, - /// redefinitions for better reading: + /// the origin string will be stored unchanged in m_string (only) + STORE_ORG_STRING_ONLY = 1 << 4, + /// Comfortable combinations: STORE_NOTHING = 0, - STORE_ALL = STORE_ORG_STRING | STORE_COMMENT | STORE_BLANK + STORE_ALL = STORE_ORG_STRING | STORE_COMMENT | STORE_BLANK, + STORE_AS_FORMMATER = STORE_ORG_STRING_ONLY | STORE_COMMENT | STORE_BLANK }; public: -- 2.39.5