From ae875a4007c6be525b750244c9ec425393e78697 Mon Sep 17 00:00:00 2001
From: hama <hama@siduction.net>
Date: Thu, 19 Jun 2014 23:51:58 +0200
Subject: [PATCH] RplLexer fully tested, definition AST

---
 rplexpr/rplastree.cpp       | 308 ++++++++++++++++++++++++++++++++++++
 rplexpr/rplastree.hpp       | 223 ++++++++++++++++++++++++++
 rplexpr/rplexpr.hpp         |   2 +
 rplexpr/rpllexer.cpp        | 100 ++++++------
 rplexpr/rpllexer.hpp        |  12 +-
 rplexpr/rplsource.cpp       |  12 +-
 rplexpr/rplsource.hpp       |   4 +-
 rplstatic/rplstatic.pro     |   6 +-
 unittests/rpllexer_test.cpp | 108 +++++++++++--
 unittests/unittests.pro     |   1 +
 10 files changed, 708 insertions(+), 68 deletions(-)
 create mode 100644 rplexpr/rplastree.cpp
 create mode 100644 rplexpr/rplastree.hpp

diff --git a/rplexpr/rplastree.cpp b/rplexpr/rplastree.cpp
new file mode 100644
index 0000000..411614a
--- /dev/null
+++ b/rplexpr/rplastree.cpp
@@ -0,0 +1,308 @@
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+#include "rplcore/rplcore.hpp"
+#include "rplexpr/rplexpr.hpp"
+
+/** @class RplASItem rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements the abstract base class of all entries of an AST.
+ *
+ */
+
+/**
+ * @brief Constructor.
+ *
+ * @param type  the type of the instance
+ */
+RplASItem::RplASItem(RplASItemType type) :
+    m_type(type),
+    m_position(NULL)
+{
+
+}
+/**
+ * @brief Destructor.
+ */
+
+RplASItem::~RplASItem()
+{
+}
+
+RplSourcePosition* RplASItem::position() const
+{
+    return m_position;
+}
+
+void RplASItem::setPosition(RplSourcePosition* position)
+{
+    m_position = position;
+}
+
+/** @class RplASExpr rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements the abstract base class of value containing items.
+ *
+ */
+RplASExpr::RplASExpr()
+{
+}
+
+/** @class RplASValue rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements the abstract base class of all named values, e.g. variables.
+ *
+ */
+
+/**
+ * @brief Constructor.
+ *
+ * @param type      the type of the instance
+ * @param name      the name of the instance
+ * @param isConst   true: the value is constant: It is not allowed to change it
+ */
+RplASValue::RplASValue(RplASItemType type, const QString& name, boolean isConst) :
+    RplASItem(type),
+    m_name(name),
+    m_isConst(isConst)
+{
+}
+/**
+ * @brief Destructor.
+ */
+RplASValue::~RplASValue()
+{
+}
+
+
+/** @class RplASScalar rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a variable/constant with exactly one value.
+ */
+/**
+ * @brief Constructor
+ * @param name
+ * @param isConst
+ */
+RplASScalar::RplASScalar(const QString& name, boolean isConst) :
+    RplASValue(AST_SCALAR, name, isConst),
+    m_value()
+{
+}
+
+/**
+ * @brief Destructor.
+ */
+RplASScalar::~RplASScalar()
+{
+}
+
+/**
+ * @brief RplASScalar::calc
+ * @return  the value of the variable/constant
+ */
+QVariant RplASScalar::calc()
+{
+    return m_value;
+}
+
+/** @class RplASNode1 rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a inner node of the abstract syntax tree with one child.
+ *
+ * This class is an abstract class.
+ */
+
+/**
+ * @brief RplASNode1::RplASNode1
+ * @param type
+ */
+RplASNode1::RplASNode1(RplASItemType type) :
+    RplASItem(type),
+    m_child(NULL)
+{
+}
+
+/**
+ * @brief Destructor.
+ */
+RplASNode1::~RplASNode1()
+{
+    delete m_child;
+    m_child = NULL;
+}
+
+/** @class RplASNode2 rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a inner node of the abstract syntax tree with two childs.
+ *
+ * This class is an abstract class.
+ */
+
+/**
+ * @brief RplASNode2::RplASNode2
+ * @param type
+ */
+RplASNode2::RplASNode2(RplASItemType type) :
+    RplASNode1(type),
+    m_child2(NULL)
+{
+}
+
+/**
+ * @brief Destructor.
+ */
+RplASNode2::~RplASNode2()
+{
+    delete m_child2;
+    m_child2 = NULL;
+}
+
+/** @class RplASNode3 rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a inner node of the abstract syntax tree with 3 childs.
+ *
+ * This class is an abstract class.
+ */
+
+/**
+ * @brief RplASNode3::RplASNode3
+ * @param type
+ */
+RplASNode3::RplASNode3(RplASItemType type) :
+    RplASNode2(type),
+    m_child3(NULL)
+{
+}
+
+/**
+ * @brief Destructor.
+ */
+RplASNode3::~RplASNode3()
+{
+    delete m_child3;
+    m_child3 = NULL;
+}
+
+/** @class RplASBinOp rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a binary operator.
+ */
+/**
+ * @brief Constructor.
+ *
+ * @param op    the operator
+ */
+RplASBinOp::RplASBinOp(RplASBinaryOperator op) :
+    RplASNode2(type),
+    m_op(op)
+{
+}
+/**
+ * @brief Destructor.
+ */
+RplASUnaryOp::~RplASBinOp()
+{
+}
+
+/** @class RplASUnaryOp rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a unary operator.
+ */
+
+/**
+ * @brief Constructor.
+ *
+ * @param op    operator
+ */
+RplASUnaryOp::RplASUnaryOp(RplASUnaryOperator op) :
+    RplASItem(AST_UNOP),
+    m_op(op)
+{
+}
+/**
+ * @brief Destructor.
+ */
+RplASUnaryOp::~RplASUnaryOp()
+{
+
+}
+
+/** @class RplASStatement rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a base class for all statements.
+ */
+
+/**
+ * @brief RplASStatement::RplASStatement
+ * @param op
+ */
+RplASStatement::RplASStatement() :
+    m_successor(NULL)
+{
+}
+
+/**
+ * @brief Destructor
+ */
+RplASStatement::~RplASStatement()
+{
+    delete m_successor;
+    m_successor = NULL;
+}
+
+/** @class RplASFor rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a for statement.
+ *
+ * The for statement has an initialization, a condition, a forwarding
+ * statement and a body.
+ * The initialization will be called first.
+ * Then the condition will be tested. If true the body will be executed
+ * and then the forwarding statement.
+ */
+
+/**
+ * @brief Constructor.
+ */
+RplASFor::RplASFor() :
+    RplASNode3(AST_FOR),
+    RplASStatement()
+{
+}
+/**
+ * @brief Destructor.
+ */
+RplASFor::~RplASFor()
+{
+}
+
+void RplASFor::execute()
+{
+    ((RplASStatement*) m_child)->execute();
+    RplASTerm* condition = (RplASTerm* condition)RplASTerm
+}
+
+/** @class RplASWhile rplastree.hpp "rplexpr/rplastree.hpp"
+ *
+ * @brief Implements a while statement.
+ *
+ * The while statement has an a condition and a body.
+ * The body will be executed while the condition returns true.
+ */
+
+RplASWhile::RplASWhile() :
+    RplASNode2(AST_WHILE),
+    RplASStatement()
+{
+}
+/**
+ * @brief Destructor.
+ */
+RplASWhile::~RplASWhile()
+{
diff --git a/rplexpr/rplastree.hpp b/rplexpr/rplastree.hpp
new file mode 100644
index 0000000..f1bfb72
--- /dev/null
+++ b/rplexpr/rplastree.hpp
@@ -0,0 +1,223 @@
+/*
+ * Licence:
+ * You can use and modify this file without any restriction.
+ * There is no warranty.
+ * You also can use the licence from http://www.wtfpl.net/.
+ * The original sources can be found on https://github.com/republib.
+*/
+
+
+#ifndef RPLASTREE_HPP
+#define RPLASTREE_HPP
+
+enum RplASItemType {
+    AST_UNDEF,
+    AST_SCALAR,
+    AST_MATRIX,
+    AST_LIST,
+    AST_MAP,
+    AST_BINOP,
+    AST_UNOP,
+    AST_FUNCTION,
+    AST_CALL,
+    AST_WHILE,
+    AST_IF,
+    AST_FOR,
+    AST_SWITCH,
+    AST_STATEMENT
+
+};
+
+enum RplASBinaryOperator {
+    ASOP_UNDEF,
+    ASOP_PLUS,
+    ASOP_MINUS,
+    ASOP_TIMES,
+    ASOP_DIV,
+    ASOP_MOD,
+    ASOP_OR,
+    ASOP_AND,
+    ASOP_GT,
+    ASOP_LT,
+    ASOP_GE,
+    ASOP_LE,
+    ASOP_EQ,
+    ASOP_NE,
+    ASOP_ASSIGN,
+    ASOP_ASSIGN_PLUS,
+    ASOP_ASSIGN_MINUS,
+    ASOP_ASSIGN_TIMES,
+    ASOP_ASSIGN_DIV,
+    ASOP_ASSIGN_MOD
+};
+enum RplASUnaryOperator {
+    ASUOP_UNDEF,
+    ASUOP_PLUS,
+    ASOUP_MINUS,
+    ASUOP_POST_DECREMENT,
+    ASUOP_PRE_DECREMENT,
+    ASUOP_POST_INCREMENT,
+    ASUOP_PRE_INCREMENT
+};
+
+class RplASTree;
+class RplASItem
+{
+public:
+    friend class RplASTree;
+    RplASItem(RplASItemType type);
+    virtual ~RplASItem();
+
+protected:
+    RplSourcePosition* m_position;
+};
+
+class RplASExpr
+{
+public:
+    RplASExpr();
+public:
+    virtual QVariant calc() = null;
+};
+
+class RplASValue : public RplASItem, public RplASExpr
+{
+public:
+    RplASValue(RplASItemType type, const QString& name,
+               boolean isConst);
+    virtual ~RplASValue();
+private:
+    QString m_name;
+    boolean m_isConst;
+};
+
+class RplASScalar : public RplASValue
+{
+public:
+    RplASScalar(const QString& name, boolean isConst);
+    virtual ~RplASScalar();
+public:
+    virtual QVariant calc();
+private:
+    QVariant m_value;
+};
+
+class RplASNode1 : public RplASItem
+{
+public:
+    RplASNode1(RplASItemType type);
+    virtual ~RplASNode1();
+protected:
+    RplASItem* m_child;
+};
+
+class RplASNode2 : public RplASNode1
+{
+public:
+    RplASNode2(RplASItemType type);
+    virtual ~RplASNode2();
+protected:
+    RplASItem* m_child2;
+};
+
+class RplASNode3 : public RplASNode2
+{
+public:
+    RplASNode3(RplASItemType type);
+    virtual ~RplASNode3();
+protected:
+    RplASItem* m_child3;
+};
+
+class RplASBinOp : public RplASNode2
+{
+public:
+    RplASBinOp(RplASBinaryOperator op);
+    virtual ~RplASBinOp();
+private:
+    RplASBinaryOperator m_op;
+};
+
+class RplASUnaryOp : public RplASNode1
+{
+public:
+    RplASUnaryOp(RplASUnaryOperator op);
+    virtual ~RplASUnaryOp();
+private:
+    RplASUnaryOperator m_op;
+};
+
+class RplASStatement
+{
+public:
+    RplASStatement();
+    virtual ~RplASStatement();
+public:
+    virtual void execute() = 0;
+private:
+    RplASItem* m_successor;
+};
+
+class RplASFor : public RplASNode3, public RplASStatement
+{
+public:
+    RplASFor();
+    virtual ~RplASFor();
+public:
+    virtual void execute();
+private:
+    RplASStatement* m_body;
+};
+
+class RplASWhile : public RplASNode2, public RplASStatement
+{
+public:
+    RplASWhile();
+    virtual ~RplASWhile();
+};
+
+class RplArgument : public RplAsNode2, public RplASStatement
+{
+public:
+    RplArgument();
+    virtual ~RplArgument();
+};
+
+class RplAsCall : public RplASItem, public RplASStatement
+{
+public:
+    RplArgument();
+    virtual ~RplArgument();
+private:
+    RplASFunction* m_function;
+    RplArgument* m_arg1;
+};
+
+class RplParameter : RplASItem
+{
+public:
+    RplParameter();
+    virtual ~RplParameter();
+private:
+    QString m_name;
+    RplASValue* m_default;
+};
+
+class RplASFunction : public RplASNodeMany
+{
+public:
+    RplASFunction(const QString& name);
+    virtual ~RplASFunction();
+protected:
+    RplASItemType* body;
+    RplArgument* arg;
+    QString m_name;
+};
+
+class RplASTree
+{
+public:
+    RplASTree();
+};
+
+#endif // RPLASTREE_HPP
diff --git a/rplexpr/rplexpr.hpp b/rplexpr/rplexpr.hpp
index a9d4d8d..62437c1 100644
--- a/rplexpr/rplexpr.hpp
+++ b/rplexpr/rplexpr.hpp
@@ -16,8 +16,10 @@
 #include <QTextStream>
 #include <QDir>
 #include <QtAlgorithms>
+#include <QVariant>
 
 #include "rplexpr/rplsource.hpp"
 #include "rplexpr/rpllexer.hpp"
+#include "rplexpr/rplastree.hpp"
 
 #endif // RPLEXPR_HPP
diff --git a/rplexpr/rpllexer.cpp b/rplexpr/rpllexer.cpp
index c4482f8..1bf42b5 100644
--- a/rplexpr/rpllexer.cpp
+++ b/rplexpr/rpllexer.cpp
@@ -47,7 +47,7 @@ RplLexException::RplLexException(RplSourcePosition& position,
 RplToken::RplToken(RplTokenType type) :
     m_tokenType(type),
     m_string(),
-    m_rawString()
+    m_printableString()
     // m_value
 {
     memset(&m_value, 0, sizeof m_value);
@@ -67,7 +67,7 @@ RplToken::~RplToken()
 RplToken::RplToken(const RplToken& source) :
     m_tokenType(source.m_tokenType),
     m_string(source.m_string),
-    m_rawString(source.m_rawString),
+    m_printableString(source.m_printableString),
     m_value(source.m_value)
 {
 }
@@ -91,24 +91,7 @@ RplToken& RplToken::operator =(const RplToken& source)
  */
 const QString& RplToken::toString()
 {
-    QString& rc = m_string;
-    switch(m_tokenType)
-    {
-    case TOKEN_STRING:
-        rc = m_rawString;
-        break;
-    case TOKEN_KEYWORD:
-    case TOKEN_NUMBER:
-    case TOKEN_OPERATOR:
-    case TOKEN_ID:
-    case TOKEN_COMMENT_REST_OF_LINE:
-    case TOKEN_COMMENT_START:
-    case TOKEN_COMMENT_END:
-    case TOKEN_SPACE:
-    default:
-        break;
-    }
-    return rc;
+    return m_string;
 }
 
 /**
@@ -156,7 +139,7 @@ qreal RplToken::asReal() const
  */
 const QString& RplToken::rawString() const
 {
-    return m_rawString;
+    return m_printableString;
 }
 /**
  * @brief Returns the id of the token.
@@ -186,7 +169,7 @@ RplTokenType RplToken::tokenType() const
 void RplToken::clear()
 {
     m_string.clear();
-    m_rawString.clear();
+    m_printableString.clear();
     m_tokenType = TOKEN_UNDEF;
     m_value.m_integer = 0;
 }
@@ -398,6 +381,7 @@ int RplLexer::findInVector(int tokenLength, const QVector<QString>& vector)
     int id = 0;
     int lbound = 0;
     int ubound = vector.size() - 1;
+    // binary search over the sorted vector:
     while(lbound <= ubound){
         int half = (ubound + lbound) / 2;
         int compareRc = 0;
@@ -428,11 +412,6 @@ int RplLexer::findInVector(int tokenLength, const QVector<QString>& vector)
     }
     return id;
 }
-
-void RplLexer::startUnit(const char* unit)
-{
-    //m_source->
-}
 const char* RplLexer::nextText(int& length, bool &isLast)
 {
     return NULL;
@@ -468,6 +447,8 @@ bool RplLexer::fillInput()
 /**
  * @brief Finds a token with an id: TOKEN_OP, TOKEN_KEYWORD, ...
  *
+ * @postcond    the token is removed from the input
+ *
  * @param tokenType the token type
  * @param flag2     the flag of the 2nd char
  * @param names     the vector with the names, sorted
@@ -509,13 +490,19 @@ RplToken* RplLexer::findTokenWithId(RplTokenType tokenType, int flag2,
             && (m_charInfo[cc] & CC_REST_ID))) {
         int id;
         // the length could be too long: the CC_2nd_.. flag could be ambigous
-        while(length >= 1 && (id = findInVector(length, names)) <= 0){
+        while( (id = findInVector(length, names)) <= 0){
+            if (length == 1 || tokenType == TOKEN_KEYWORD){
+                break;
+            }
             length--;
         }
         if (id > 0){
             rc = m_currentToken;
             rc->m_tokenType = tokenType;
             rc->m_value.m_id = id;
+            if (tokenType == TOKEN_COMMENT_START
+                    && (m_storageFlags & STORE_COMMENT) != 0)
+                rc->m_string.append(m_input.mid(0, length));
             m_input.remove(0, length);
             m_currentCol += length;
         }
@@ -593,7 +580,8 @@ RplToken*RplLexer::scanString()
     int inputLength = m_input.size();
     int cc;
     int length = 1;
-    m_currentToken->m_string.append(m_input[0]);
+    m_currentToken->m_tokenType = TOKEN_STRING;
+    m_currentToken->m_value.m_id = delim;
     bool again = false;
     do {
         while(length < inputLength && (cc = m_input[length].unicode()) != delim){
@@ -601,7 +589,7 @@ RplToken*RplLexer::scanString()
             if (cc != '\\'
                 || (m_stringFeatures
                     & (SF_C_ESCAPING | SF_C_HEX_CHARS | SF_C_SPECIAL)) == 0){
-                m_currentToken->m_rawString.append(QChar(cc));
+                m_currentToken->m_string.append(QChar(cc));
             } else {
                 if (length >= inputLength)
                     throw RplLexException(*m_currentPosition,
@@ -625,7 +613,7 @@ RplToken*RplLexer::scanString()
                            hexVal = hexVal * 16 + nibble;
                        }
                     }
-                    m_currentToken->m_rawString.append(QChar(hexVal));
+                    m_currentToken->m_string.append(QChar(hexVal));
                 } else if ( (m_stringFeatures & SF_C_SPECIAL)){
                     switch(cc){
                     case 'r':
@@ -643,12 +631,15 @@ RplToken*RplLexer::scanString()
                     case 'v':
                         cc = '\v';
                         break;
+                    case 'f':
+                        cc = '\f';
+                        break;
                     default:
                         break;
                     }
-                    m_currentToken->m_rawString.append(QChar(cc));
+                    m_currentToken->m_string.append(QChar(cc));
                 } else {
-                   m_currentToken->m_rawString.append(QChar(cc));
+                   m_currentToken->m_string.append(QChar(cc));
                 }
             }
         }
@@ -657,14 +648,14 @@ RplToken*RplLexer::scanString()
         }
         if ( (m_stringFeatures & SF_DOUBLE_DELIM) && length < inputLength
              && m_input[length].unicode() == delim){
-            m_currentToken->m_rawString.append(delim);
+            m_currentToken->m_printableString.append(delim);
             length++;
             again = true;
         }
     }
     while(again);
     if (m_storageFlags & STORE_ORG_STRING)
-        m_currentToken->m_string.append(m_input.mid(0, length));
+        m_currentToken->m_printableString.append(m_input.mid(0, length));
     m_input.remove(0, length);
     m_currentCol += length;
     return m_currentToken;
@@ -678,9 +669,8 @@ RplToken*RplLexer::scanString()
 void RplLexer::scanComment()
 {
     int inputLength = m_input.size();
-    int cc;
     int length = 1;
-    QString commentEnd = m_commentEnds[m_currentToken->id()];
+    QString& commentEnd = m_commentEnds[m_currentToken->id()];
     int ix;
     if (commentEnd[0].unicode() == '\n'){
         // single line comment:
@@ -692,13 +682,15 @@ void RplLexer::scanComment()
         while( (ix = m_input.indexOf(commentEnd)) < 0){
             if (m_storageFlags & STORE_COMMENT)
                 m_currentToken->m_string.append(m_input);
+            m_input.clear();
             if (! fillInput())
                 throw RplLexException(*m_currentPosition,
                     "comment end not found");
         }
-        if (m_storageFlags & STORE_COMMENT)
-            m_currentToken->m_rawString.append(m_input.mid(0, ix));
         length = ix + commentEnd.size();
+        if (m_storageFlags & STORE_COMMENT)
+            m_currentToken->m_string
+                    .append(m_input.mid(0, length));
     }
     m_input.remove(0, length);
     m_currentCol += length;
@@ -727,6 +719,7 @@ RplToken* RplLexer::nextToken()
             m_currentToken->m_tokenType = TOKEN_END_OF_SOURCE;
         } else {
             QChar cc = m_input.at(0);
+            int cc2 = cc.unicode();
             if (cc.isSpace()){
                 m_currentToken->m_tokenType = TOKEN_SPACE;
                 ix = 1;
@@ -740,14 +733,14 @@ RplToken* RplLexer::nextToken()
                 rc = m_currentToken;
             } else if (cc.isDigit()){
                 rc = scanNumber();
-            } else if (cc == '"' || cc == '\''){
+            } else if ( (cc2 == '"' && (m_stringFeatures & SF_QUOTE) != 0)
+                       || (cc2 == '\'' && (m_stringFeatures & SF_TICK) != 0)){
                 rc = scanString();
             } else {
-                int cc2 = cc.unicode();
                 if (cc2 >= CHAR_INFO_SIZE)
                     throw RplLexException(*m_currentPosition,
-                        "no lexical symbol can start with this char: %ls",
-                        QChar(cc2));
+                        "no lexical symbol can start with this char: %lc",
+                        cc);
                 else
                 {
                     if (rc == NULL && (m_charInfo[cc2] & CC_FIRST_COMMENT_START)){
@@ -773,6 +766,18 @@ RplToken* RplLexer::nextToken()
                         rc = findTokenWithId(TOKEN_KEYWORD,
                                              CC_2nd_KEYWORD, m_keywords);
                     }
+                    if (rc == NULL && (m_charInfo[cc2] & CC_FIRST_ID)){
+                        int length = 1;
+                        while(length < m_input.size()
+                            && (cc2 = m_input[length].unicode()) < CHAR_INFO_SIZE
+                                && (m_charInfo[cc2] & CC_REST_ID) != 0)
+                              length++;
+                        rc = m_currentToken;
+                        rc->m_tokenType = TOKEN_ID;
+                              rc->m_string.append(m_input.mid(0, length));
+                        m_input.remove(0, length);
+                        m_currentCol += length;
+                    }
 
                 }
             }
@@ -826,17 +831,18 @@ RplToken* RplLexer::nextNonSpaceToken()
 }
 
 /**
- * @brief Starts a new source unit.
+ * @brief Prepares a given source unit for reading.
  *
  * Saves the current source position onto the top of stack.
  * Pushes the source unit onto the top of stack.
  *
- * @param unit
+ * Precondition: the unit must be known by exactly one reader
+ *
+ * @param unit  the new source unit
  */
 void RplLexer::startUnit(const QString& unit)
 {
-    // m_source->startUnit(unit, new RplSourcePosition(
-    //    m_source->currentReader()));
+    m_source->startUnit(unit, *m_currentPosition);
 }
 /**
  * @brief Returns the source of the instance.
diff --git a/rplexpr/rpllexer.hpp b/rplexpr/rpllexer.hpp
index db6e8a1..8d87710 100644
--- a/rplexpr/rpllexer.hpp
+++ b/rplexpr/rpllexer.hpp
@@ -54,8 +54,8 @@ public:
 protected:
     RplTokenType m_tokenType;
     QString m_string;
-    // only for TOKEN_STRING: copy from source but without escaped chars like "\\n"
-    QString m_rawString;
+    // only for TOKEN_STRING: copy from source but with escaped chars like "\\n"
+    QString m_printableString;
     union {
         // only for TOKEN_KEYWORD and TOKEN_OPERATOR
         int m_id;
@@ -130,7 +130,10 @@ public:
         /// characters can be written in hexadecimal notation: \x20 is ' '
         SF_C_HEX_CHARS  = 1 << 5,
         /// A delimiter inside a string must be doubled (like in Pascal)
-        SF_DOUBLE_DELIM = 1 << 6
+        SF_DOUBLE_DELIM = 1 << 6,
+        // Redefinitions for better reading:
+        SF_LIKE_C       = SF_TICK | SF_QUOTE | SF_C_ESCAPING | SF_C_SPECIAL
+                            | SF_C_HEX_CHARS
     };
     enum StorageFlags {
         S_UNDEF,
@@ -158,7 +161,6 @@ public:
         int storageFlags = STORE_NOTHING);
     virtual ~RplLexer();
 public:
-    void startUnit(const char* unit);
     RplToken* nextToken();
     RplToken* peekNonSpaceToken();
     RplToken* nextNonSpaceToken();
@@ -183,7 +185,7 @@ protected:
     QVector<QString> m_keywords;
     // sorted, string ends with the id of the operator
     QVector<QString> m_operators;
-    // sorted, string ends with the id of the comment start
+    // sorted, each entry ends with the id of the comment start
     QVector<QString> m_commentStarts;
     // index: id content: comment_end
     QVector<QString> m_commentEnds;
diff --git a/rplexpr/rplsource.cpp b/rplexpr/rplsource.cpp
index e885db8..05b0943 100644
--- a/rplexpr/rplsource.cpp
+++ b/rplexpr/rplsource.cpp
@@ -364,11 +364,15 @@ void RplSource::addSourceUnit(RplSourceUnit* unit) {
  * Saves the current source position onto the top of stack.
  * Pushes the source unit onto the top of stack.
  *
- * @param unit
+ * @param unit      the source unit
+ * @param caller    the position of the include
+ *
  */
 bool RplSource::startUnit(const QString& unit,
-                          const RplSourcePosition* sourcePosition) {
-    m_sourcePositionStack.push_back(sourcePosition);
+                          const RplSourcePosition& caller) {
+    RplSourcePosition* position = new RplSourcePosition(caller);
+    m_sourcePositions.append(position);
+    m_sourcePositionStack.push_back(position);
     RplReader* reader = NULL;
     QList<RplReader*>::iterator it;
     for(it = m_readers.begin();
@@ -377,6 +381,8 @@ bool RplSource::startUnit(const QString& unit,
         RplReader* current = *it;
         if(current->openSourceUnit(unit)) {
             reader = current;
+            m_currentReader = current;
+            break;
         }
     }
     return reader != NULL;
diff --git a/rplexpr/rplsource.hpp b/rplexpr/rplsource.hpp
index f44e532..1672711 100644
--- a/rplexpr/rplsource.hpp
+++ b/rplexpr/rplsource.hpp
@@ -118,14 +118,14 @@ public:
     QStack<const RplSourcePosition*> sourcePositionStack() const;
     QStack<RplSourceUnit*>& sourceUnitStack();
 
-    bool startUnit(const QString& unit, const RplSourcePosition*);   
+    bool startUnit(const QString& unit, const RplSourcePosition& caller);
     void pushSourceUnit(RplSourceUnit* unit);
     RplSourceUnit* popSourceUnit(RplReader* reader);
     RplReader* currentReader();
 protected:
     // stack of the info about the stacked (open) source units:
     QStack<const RplSourcePosition*> m_sourcePositionStack;
-    QList<const RplSourcePosition> m_sourcePositions;
+    QList<const RplSourcePosition*> m_sourcePositions;
     QList<RplReader*> m_readers;
     QList<RplSourceUnit*> m_sourceUnits;
     // setCurrentSourceUnit() pushes one entry, removeSourceUnit() pops it
diff --git a/rplstatic/rplstatic.pro b/rplstatic/rplstatic.pro
index 7269d87..9d694f6 100644
--- a/rplstatic/rplstatic.pro
+++ b/rplstatic/rplstatic.pro
@@ -32,7 +32,8 @@ SOURCES += \
     ../rplnet/rpltcpserver.cpp \
     ../rplexpr/rpllexer.cpp \
     ../rplexpr/rplsource.cpp \
-    ../rplcore/rplqstring.cpp
+    ../rplcore/rplqstring.cpp \
+    ../rplexpr/rplastree.cpp
 
 HEADERS += ../rplmodules.hpp \
     ../rplcore/rplconfig.hpp \
@@ -56,7 +57,8 @@ HEADERS += ../rplmodules.hpp \
     ../rplexpr/rpllexer.hpp \
     ../rplexpr/rplexpr.hpp \
     ../rplexpr/rplsource.hpp \
-    ../rplcore/rplqstring.hpp
+    ../rplcore/rplqstring.hpp \
+    ../rplexpr/rplastree.hpp
 
 unix:!symbian {
     maemo5 {
diff --git a/unittests/rpllexer_test.cpp b/unittests/rpllexer_test.cpp
index 95874d4..a88e9fa 100644
--- a/unittests/rpllexer_test.cpp
+++ b/unittests/rpllexer_test.cpp
@@ -23,13 +23,13 @@ public:
         checkE(TOKEN_ID, tokenType());
         checkE(0, m_value.m_id);
         checkT(m_string.isEmpty());
-        checkT(m_rawString.isEmpty());
+        checkT(m_printableString.isEmpty());
 
         m_value.m_id = 7422;
         checkE(7422, RplToken::id());
         m_string = "Wow!";
         checkE("Wow!", RplToken::toString());
-        m_rawString = "GooGoo";
+        m_printableString = "GooGoo";
         checkE("GooGoo", rawString());
         m_tokenType = TOKEN_NUMBER;
         checkE(TOKEN_NUMBER, tokenType());
@@ -38,7 +38,7 @@ public:
         checkE(TOKEN_UNDEF, tokenType());
         checkE(0, m_value.m_id);
         checkT(m_string.isEmpty());
-        checkT(m_rawString.isEmpty());
+        checkT(m_printableString.isEmpty());
 
         m_value.m_integer = 773322;
         checkE(773322, asInteger());
@@ -62,9 +62,9 @@ public:
            OP_LT, OP_GE, OP_LE, OP_EQ
          };
 #   define OPERATORS "+ * / = > < >= <= =="
-    enum { COMMENT_UNDEF, COMMENT_1
+    enum { COMMENT_UNDEF, COMMENT_1, COMMENT_MULTILINE, COMMENT_2
     };
-#   define COMMENTS "# \n"
+#   define COMMENTS "/* */ // \n"
     void testSpace(){
         RplSource source;
         RplStringReader reader(source);
@@ -132,19 +132,105 @@ public:
         checkToken(lex.nextNonSpaceToken(), TOKEN_END_OF_SOURCE);
     }
 
+    void testComments(){
+        RplSource source;
+        RplStringReader reader(source);
+
+        reader.addSource("<main>", "/**/9//\n8/***/7// wow\n/*\n*\n*\n**/");
+        source.addReader(&reader);
+
+        enum { COMMENT_UNDEF, COMMENT_MULTILINE, COMMENT_1
+        };
+        RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS,
+                 "A-Za-z_",
+                 "A-Za-z0-9_",
+                 RplLexer::NUMTYPE_ALL,
+                 RplLexer::SF_LIKE_C, RplLexer::STORE_ALL);
+        checkToken(lex.nextToken(), TOKEN_COMMENT_START, COMMENT_MULTILINE,
+                   "/**/");
+        checkToken(lex.nextToken(), TOKEN_NUMBER);
+        checkToken(lex.nextToken(), TOKEN_COMMENT_START, COMMENT_1,
+                   "//\n");
+        checkToken(lex.nextToken(), TOKEN_NUMBER);
+        checkToken(lex.nextToken(), TOKEN_COMMENT_START, COMMENT_MULTILINE,
+                   "/***/");
+        checkToken(lex.nextToken(), TOKEN_NUMBER);
+        checkToken(lex.nextToken(), TOKEN_COMMENT_START, COMMENT_1,
+                   "// wow\n");
+        checkToken(lex.nextToken(), TOKEN_COMMENT_START, COMMENT_MULTILINE,
+                   "/*\n*\n*\n**/");
+    }
+    void testStrings(){
+        RplSource source;
+        RplStringReader reader(source);
+
+        reader.addSource("<main>", "\"abc\\t\\r\\n\\a\\v\"'1\\x9Z\\x21A\\X9'");
+        source.addReader(&reader);
+
+        RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS,
+                 "A-Za-z_",
+                 "A-Za-z0-9_",
+                 RplLexer::NUMTYPE_ALL,
+                 RplLexer::SF_LIKE_C, RplLexer::STORE_ALL);
+        checkToken(lex.nextToken(), TOKEN_STRING, '"', "abc\t\r\n\a\v");
+        checkToken(lex.nextToken(), TOKEN_STRING, '\'', "1\tZ!A\t");
+    }
+    void testKeywords(){
+        RplSource source;
+        RplStringReader reader(source);
+
+        reader.addSource("<main>", "if\n\tthen else\nfi");
+        source.addReader(&reader);
+
+        RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS,
+                 "A-Za-z_",
+                 "A-Za-z0-9_",
+                 RplLexer::NUMTYPE_ALL,
+                 RplLexer::SF_LIKE_C, RplLexer::STORE_ALL);
+        checkToken(lex.nextToken(), TOKEN_KEYWORD, KEY_IF);
+        checkToken(lex.nextNonSpaceToken(), TOKEN_KEYWORD, KEY_THEN);
+        checkToken(lex.nextNonSpaceToken(), TOKEN_KEYWORD, KEY_ELSE);
+        checkToken(lex.nextNonSpaceToken(), TOKEN_KEYWORD, KEY_FI);
+        checkToken(lex.nextNonSpaceToken(), TOKEN_END_OF_SOURCE);
+    }
+
+    void testIds(){
+        RplSource source;
+        RplStringReader reader(source);
+
+        reader.addSource("<main>", "i\n\tifs\n"
+            "_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789");
+        source.addReader(&reader);
+
+        RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS,
+                 "A-Za-z_",
+                 "A-Za-z0-9_",
+                 RplLexer::NUMTYPE_ALL,
+                 RplLexer::SF_LIKE_C, RplLexer::STORE_ALL);
+        checkToken(lex.nextToken(), TOKEN_ID, 0, "i");
+        checkToken(lex.nextNonSpaceToken(), TOKEN_ID, 0,
+           "ifs");
+        checkToken(lex.nextNonSpaceToken(), TOKEN_ID, 0,
+           "_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789");
+    }
+
     void testBasic(){
         RplSource source;
         RplStringReader reader(source);
+        source.addReader(&reader);
         reader.addSource("<main>", "if i>1 then i=1+2*_x9 fi");
         RplLexer lex(&source, KEYWORDS, OPERATORS, COMMENTS,
                  "A-Za-z_",
-                 "A-Za-z0-9_"
-                 );
+                 "A-Za-z0-9_",
+                 RplLexer::NUMTYPE_ALL,
+                 RplLexer::SF_LIKE_C, RplLexer::STORE_ALL);
+        RplToken* token;
         checkToken(lex.nextToken(), TOKEN_KEYWORD, KEY_IF);
         checkToken(lex.nextToken(), TOKEN_SPACE, 0);
         checkToken(lex.nextToken(), TOKEN_ID, 0, "i");
         checkToken(lex.nextToken(), TOKEN_OPERATOR, OP_GT);
-        checkToken(lex.nextToken(), TOKEN_NUMBER, 0, "1");
+        token = checkToken(lex.nextToken(), TOKEN_NUMBER);
+        checkE(1, token->asInteger());
         checkToken(lex.nextToken(), TOKEN_SPACE, 0);
         checkToken(lex.nextToken(), TOKEN_KEYWORD, KEY_THEN);
         checkToken(lex.nextToken(), TOKEN_SPACE, 0);
@@ -152,10 +238,14 @@ public:
     }
 
     virtual void doIt(void) {
+        testBasic();
+        testIds();
+        testKeywords();
+        testComments();
+        testStrings();
         testOperators();
         testNumeric();
         testSpace();
-        testBasic();
         testRplToken();
     }
 };
diff --git a/unittests/unittests.pro b/unittests/unittests.pro
index 97dbea9..ca425c5 100644
--- a/unittests/unittests.pro
+++ b/unittests/unittests.pro
@@ -25,6 +25,7 @@ SOURCES += main.cpp \
     ../rplmath/rplmatrix.cpp \
     ../rplexpr/rplsource.cpp \
     ../rplexpr/rpllexer.cpp \
+    ../rplexpr/rplastree.cpp \
     rplexception_test.cpp \
     rplstring_test.cpp \
     rplsource_test.cpp \
-- 
2.39.5