Fixed some bugs in Parser::firstSet and added a bit of caching. It still doesn't work quite right, though, there's some problem with nullable left recursion. However, it's better than it was, and I need to go to bed. More work later.

include/GraphStructuredStack.h

@@ -27,6 +27,7 @@ class GraphStructuredStack {
 		void addEdge(NodeTree<int>* start, NodeTree<int>* end, NodeTree<Symbol>* edge);
 		void clear();
 
+        std::vector<int> getFrontier(int frontier);
 		std::string toString();
 	private:
 		std::vector<std::vector<NodeTree<int>*>*> gss;

include/ParseAction.h

@@ -11,6 +11,7 @@
 #include <vector>
 #include <string>
 
+
 class ParseAction {
 	public:
 		enum ActionType { INVALID, REDUCE, SHIFT, ACCEPT, REJECT };
@@ -18,10 +19,11 @@ class ParseAction {
 		ParseAction(ActionType action, ParseRule* reduceRule);
 		ParseAction(ActionType action, int shiftState);
 		~ParseAction();
-		bool const equalsExceptLookahead(const ParseAction &other);
+		bool const equalsExceptLookahead(const ParseAction &other) const;
-		bool const operator==(const ParseAction &other);
+		bool const operator==(const ParseAction &other) const;
-		bool const operator!=(const ParseAction &other);
+		bool const operator!=(const ParseAction &other) const;
-		std::string toString();
+        bool const operator<(const ParseAction &other) const;
+        std::string toString(bool printRuleLookahead = true);
 		static std::string actionToString(ActionType action);
 
 		ActionType action;

include/ParseRule.h

@@ -16,10 +16,10 @@ class ParseRule {
 		ParseRule();
 		ParseRule(Symbol leftHandle, int pointerIndex, std::vector<Symbol> &rightSide, std::vector<Symbol>* lookahead);
 		~ParseRule();
-		const bool equalsExceptLookahead(const ParseRule &other);
+		const bool equalsExceptLookahead(const ParseRule &other) const;
-		bool const operator==(const ParseRule &other);
+		bool const operator==(const ParseRule &other) const;
-		bool const operator!=(const ParseRule &other);
+		bool const operator!=(const ParseRule &other) const;
-
+        bool const operator<(const ParseRule &other) const; //Used for ordering so we can put ParseRule's in sets, and also so that ParseActions will have an ordering
 		ParseRule* clone();
 
 		void setLeftHandle(Symbol leftHandle);
@@ -40,7 +40,7 @@ class ParseRule {
 		void addLookahead(std::vector<Symbol>* lookahead);
 		std::vector<Symbol>* getLookahead();
 
-		std::string toString();
+		std::string toString(bool printLookahead = true);
 		std::string toDOT();
 
 	private:

include/Parser.h

@@ -12,6 +12,7 @@
 #include "Table.h"
 
 #include <queue>
+#include <set>
 #include <map>
 #include <vector>
 #include <algorithm>
@@ -36,8 +37,13 @@ class Parser {
 		void importTable(char* tableData);
 
 	protected:
-		std::vector<Symbol>* firstSet(Symbol token);
+		std::vector<Symbol> firstSet(Symbol token, std::vector<Symbol> avoidList = std::vector<Symbol>(), bool addNewTokens = true);
-		std::vector<Symbol>* firstSet(Symbol token, std::vector<Symbol> avoidList);
+        bool isNullable(Symbol token);
+        bool isNullableHelper(Symbol token, std::set<Symbol> done);
+
+        std::map<Symbol, std::vector<Symbol>> tokenFirstSet;
+        std::map<Symbol, bool> tokenNullable;
+
 		std::vector<Symbol>* incrementiveFollowSet(ParseRule* rule);
 		virtual void closure(State* state);
 		virtual void addStates(std::vector< State* >* stateSets, State* state, std::queue<State*>* toDo);

include/RNGLRParser.h

@@ -5,6 +5,8 @@
 #include <queue>
 #include <map>
 #include <vector>
+#include <set>
+#include <utility>
 #include <algorithm>
 #include "Parser.h"
 #include "Symbol.h"
@@ -16,6 +18,7 @@ class RNGLRParser: public Parser {
 		RNGLRParser();
 		~RNGLRParser();
 		NodeTree<Symbol>* parseInput(std::string inputString);
+        void printReconstructedFrontier(int frontier);
 
 	private:
 		void reducer(int i);

include/Table.h

@@ -1,5 +1,8 @@
 #include <fstream>
 
+#include <string>
+#include <utility>
+
 #include "util.h"
 #include "ParseRule.h"
 #include "ParseAction.h"
@@ -20,6 +23,7 @@ class Table {
 		void remove(int stateNum, Symbol tranSymbol);
 		std::vector<ParseAction*>* get(int state, Symbol token);
 		ParseAction* getShift(int state, Symbol token);
+	    std::vector<std::pair<std::string, ParseAction>> stateAsParseActionVector(int state);
         std::string toString();
 	private:
 		std::vector< std::vector< std::vector<ParseAction*>* >* >  table;

src/GraphStructuredStack.cpp

@@ -117,6 +117,13 @@ void GraphStructuredStack::addEdge(NodeTree<int>* start, NodeTree<int>* end, Nod
 	edges[std::make_pair(start, end)] = edge;
 }
 
+std::vector<int> GraphStructuredStack::getFrontier(int frontier) {
+    std::vector<int> toReturn;
+    for (int i = 0; i < gss[frontier]->size(); i++)
+        toReturn.push_back((*(gss[frontier]))[i]->getData());
+    return toReturn;
+}
+
 std::string GraphStructuredStack::toString() {
 	std::string tostring = "";
 	for (std::vector<std::vector<NodeTree<int>*>*>::size_type i = 0; i < gss.size(); i++) {

src/Importer.cpp

@@ -66,7 +66,8 @@ NodeTree<ASTData>* Importer::importFirstPass(std::string fileName) {
 	NodeTree<ASTData>* ast = getUnit(fileName);
 	if (ast == NULL) {
 		NodeTree<Symbol>* parseTree = parseAndTrim(fileName);
-
+        if (!parseTree)
+            return NULL;
 		//Call with ourself to allow the transformation to call us to import files that it needs
 		ast = ASTTransformer->firstPass(fileName, parseTree); //This firstPass will register itself
 	}
@@ -160,6 +161,8 @@ NodeTree<Symbol>* Importer::parseAndTrim(std::string fileName) {
 		outFile << parseTree->DOTGraphString() << std::endl;
 	} else {
 		std::cout << "ParseTree returned from parser for " << fileName << " is NULL!" << std::endl;
+	    outFile.close(); outFileTransformed.close();
+        return NULL;
     }
 	outFile.close();
 

src/ParseAction.cpp

@@ -22,18 +22,33 @@ ParseAction::~ParseAction() {
 
 }
 
-const bool ParseAction::equalsExceptLookahead(const ParseAction &other) {
+const bool ParseAction::equalsExceptLookahead(const ParseAction &other) const {
 	return( action == other.action && ( reduceRule == other.reduceRule || reduceRule->equalsExceptLookahead(*(other.reduceRule)) ) && shiftState == other.shiftState);
 }
 
-const bool ParseAction::operator==(const ParseAction &other) {
+const bool ParseAction::operator==(const ParseAction &other) const {
 	return( action == other.action && ( reduceRule == other.reduceRule || *reduceRule == *(other.reduceRule) ) && shiftState == other.shiftState);
 }
 
-const bool ParseAction::operator!=(const ParseAction &other) {
+const bool ParseAction::operator!=(const ParseAction &other) const {
 	return !(this->operator==(other));
 }
 
+//Exists so we can put ParseActions into sets
+const bool ParseAction::operator<(const ParseAction &other) const {
+    if (action != other.action)
+        return action < other.action;
+    if (reduceRule != other.reduceRule) {
+        if (! (reduceRule && other.reduceRule)) {
+           return reduceRule < other.reduceRule;
+        } else {
+            return *reduceRule < *(other.reduceRule);
+        }
+    }
+
+    return shiftState < other.shiftState;
+}
+
 std::string ParseAction::actionToString(ActionType action) {
 	switch (action) {
 		case REDUCE:
@@ -53,11 +68,11 @@ std::string ParseAction::actionToString(ActionType action) {
 	}
 }
 
-std::string ParseAction::toString() {
+std::string ParseAction::toString(bool printRuleLookahead) {
 	std::string outputString = "";
 	outputString += actionToString(action);
 	if (reduceRule != NULL)
-		outputString += " " + reduceRule->toString(); 
+		outputString += " " + reduceRule->toString(printRuleLookahead);
 	if (shiftState != -1)
 		outputString += " " + intToString(shiftState);
 	return(outputString);

src/ParseRule.cpp

@@ -16,18 +16,34 @@ ParseRule::~ParseRule() {
 
 }
 
-const bool ParseRule::equalsExceptLookahead(const ParseRule &other) {
+const bool ParseRule::equalsExceptLookahead(const ParseRule &other) const {
 	return(leftHandle == other.leftHandle && rightSide == other.rightSide && pointerIndex == other.pointerIndex);
 }
 
-const bool ParseRule::operator==(const ParseRule &other) {
+const bool ParseRule::operator==(const ParseRule &other) const {
 	return(equalsExceptLookahead(other) && (lookahead == NULL ? other.lookahead == NULL : (*lookahead) == *(other.lookahead)));
 }
 
-const bool ParseRule::operator!=(const ParseRule &other) {
+const bool ParseRule::operator!=(const ParseRule &other) const  {
 	return !(this->operator==(other));
 }
 
+const bool ParseRule::operator<(const ParseRule &other) const {
+    //Used for ordering so we can put ParseRule's in sets, and also so that ParseActions will have an ordering
+    if (leftHandle != other.leftHandle)
+        return leftHandle < other.leftHandle;
+    if (rightSide != other.rightSide)
+        return rightSide < other.rightSide;
+    if (lookahead != other.lookahead) {
+        if (! (lookahead && other.lookahead)) {
+            return lookahead < other.lookahead;
+        } else {
+            return *lookahead < *(other.lookahead);
+        }
+    }
+    return false;
+}
+
 ParseRule* ParseRule::clone() {
 	std::vector<Symbol>* newLookahead = NULL;
 	if (lookahead) {
@@ -111,7 +127,7 @@ std::vector<Symbol>* ParseRule::getLookahead() {
 	return lookahead;
 }
 
-std::string ParseRule::toString() {
+std::string ParseRule::toString(bool printLookahead) {
 	std::string concat = leftHandle.toString() + " -> ";
 	for (int i = 0; i < rightSide.size(); i++) {
 		if (i == pointerIndex)
@@ -120,7 +136,7 @@ std::string ParseRule::toString() {
 	}
 	if (pointerIndex >= rightSide.size())
 		concat += "(*)";
-	if (lookahead != NULL) {
+	if (printLookahead && lookahead != NULL) {
 		concat += "**";
 		for (std::vector<Symbol>::size_type i = 0; i < lookahead->size(); i++)
 			concat += (*lookahead)[i].toString();

src/Parser.cpp

@@ -117,59 +117,76 @@ int Parser::stateNum(State* state) {
 	return -1;
 }
 
-std::vector<Symbol>* Parser::firstSet(Symbol token) {
+std::vector<Symbol> Parser::firstSet(Symbol token, std::vector<Symbol> avoidList, bool addNewTokens) {
-	std::vector<Symbol> avoidList;
+	if (tokenFirstSet.find(token) != tokenFirstSet.end())
-	return firstSet(token, avoidList);
+        return tokenFirstSet[token];
-}
-
-std::vector<Symbol>* Parser::firstSet(Symbol token, std::vector<Symbol> avoidList) {
     //If we've already done this token, don't do it again
 	for (std::vector<Symbol>::size_type i = 0; i < avoidList.size(); i++)
-		if (avoidList[i] == token) {
+		if (avoidList[i] == token)
-			return new std::vector<Symbol>();
+			return std::vector<Symbol>();
-		}
 	avoidList.push_back(token);
-	std::vector<Symbol>* first = new std::vector<Symbol>();
+
+    std::vector<Symbol> first;
 	//First, if the symbol is a terminal, than it's first set is just itself.
 	if (token.isTerminal()) {
-		first->push_back(token);
+		first.push_back(token);
 		return(first);
 	}
 	//Otherwise....
 	//Ok, to make a first set, go through the grammer, if the token it's left side, add it's production's first token's first set.
 	//If that one includes mull, do the next one too (if it exists).
 	Symbol rightToken;
-	std::vector<Symbol>* recursiveFirstSet = NULL;
+	std::vector<Symbol> recursiveFirstSet;
 	for (std::vector<ParseRule*>::size_type i = 0; i < loadedGrammer.size(); i++) {
 		if (token == loadedGrammer[i]->getLeftSide()) {
 			//Loop through the rule adding first sets for each token if the previous token contained NULL
-			bool recFirstSetHasNull = false;
 			int j = 0;
 			do {
 				rightToken = loadedGrammer[i]->getRightSide()[j]; //Get token of the right side of this rule
 				if (rightToken.isTerminal()) {
-					recursiveFirstSet = new std::vector<Symbol>();
+					recursiveFirstSet.push_back(rightToken);
-					recursiveFirstSet->push_back(rightToken);
 				} else {
 					//Add the entire set
-					recursiveFirstSet = firstSet(rightToken, avoidList);
+					recursiveFirstSet = firstSet(rightToken, avoidList, false);//Don't add children to cache, as early termination may cause them to be incomplete
 				}
-				first->insert(first->end(), recursiveFirstSet->begin(), recursiveFirstSet->end());
+				first.insert(first.end(), recursiveFirstSet.begin(), recursiveFirstSet.end());
-				//Check to see if the current recursiveFirstSet contains NULL, if so, then go through again with the next token. (if there is one)
-				recFirstSetHasNull = false;
-				for (std::vector<Symbol>::size_type k = 0; k < recursiveFirstSet->size(); k++) {
-					if ((*recursiveFirstSet)[k] == nullSymbol) {
-						recFirstSetHasNull = true;
-					}
-				}
-				delete recursiveFirstSet;
 				j++;
-			} while (recFirstSetHasNull && loadedGrammer[i]->getRightSide().size() > j);
+			} while (isNullable(rightToken) && loadedGrammer[i]->getRightSide().size() > j);
 		}
 	}
+    if (addNewTokens)
+        tokenFirstSet[token] = first;
 	return(first);
 }
 
+bool Parser::isNullable(Symbol token) {
+    if (tokenNullable.find(token) != tokenNullable.end())
+        return tokenNullable[token];
+    bool nullable = isNullableHelper(token, std::set<Symbol>());
+    tokenNullable[token] = nullable;
+    return nullable;
+}
+//We use this helper function to recurse because it is possible to wind up with loops, and if so we want
+//early termination. However, this means that nullable determinations in the middle of the loop are inaccurate
+//(since we terminated early), so we don't want to save them. Thus, for simplicity, only the main method will
+//add to the cache. This is somewhat unfortunate for preformance, but the necessary additions to keep track of
+//invalidated state are more complicated than it's worth.
+bool Parser::isNullableHelper(Symbol token, std::set<Symbol> done) {
+    if (token.isTerminal())
+        return token == nullSymbol;
+    if (done.find(token) != done.end())
+        return false;
+    done.insert(token);
+    if (tokenNullable.find(token) != tokenNullable.end())
+        return tokenNullable[token];
+	//Note that we only have to check the first token on the right side, as we would only get to the other tokens if it is nullable, which is what we're checking
+	for (std::vector<ParseRule*>::size_type i = 0; i < loadedGrammer.size(); i++)
+		if (token == loadedGrammer[i]->getLeftSide())
+            if (isNullableHelper(loadedGrammer[i]->getRightSide()[0], done))
+                return true;
+    return false;
+}
+
 //Return the correct lookahead. This followSet is built based on the current rule's lookahead if at end, or the next Symbol's first set.
 std::vector<Symbol>* Parser::incrementiveFollowSet(ParseRule* rule) {
 	//Advance the pointer past the current Symbol (the one we want the followset for) to the next symbol (which might be in our follow set, or might be the end)
@@ -178,25 +195,24 @@ std::vector<Symbol>* Parser::incrementiveFollowSet(ParseRule* rule) {
 
 	//Get the first set of the next Symbol. If it contains nullSymbol, keep doing for the next one
 	std::vector<Symbol>* followSet = new std::vector<Symbol>();
-	std::vector<Symbol>* symbolFirstSet;
+	std::vector<Symbol> symbolFirstSet;
 	bool symbolFirstSetHasNull = true;
 	while (symbolFirstSetHasNull && !rule->isAtEnd()) {
 		symbolFirstSetHasNull = false;
 		symbolFirstSet = firstSet(rule->getAtNextIndex());
-		for (std::vector<Symbol>::size_type i = 0; i < symbolFirstSet->size(); i++) {
+		for (std::vector<Symbol>::size_type i = 0; i < symbolFirstSet.size(); i++) {
-			if ((*symbolFirstSet)[i] == nullSymbol) {
+			if (symbolFirstSet[i] == nullSymbol) {
 				symbolFirstSetHasNull = true;
-				symbolFirstSet->erase(symbolFirstSet->begin()+i);
+				symbolFirstSet.erase(symbolFirstSet.begin()+i);
 				break;
 			}
 		}
-		followSet->insert(followSet->end(), symbolFirstSet->begin(), symbolFirstSet->end());
+		followSet->insert(followSet->end(), symbolFirstSet.begin(), symbolFirstSet.end());
-		delete symbolFirstSet;
 		rule->advancePointer();
 	}
 	if (rule->isAtEnd()) {
-		symbolFirstSet = rule->getLookahead();
+		symbolFirstSet = *(rule->getLookahead());
-		followSet->insert(followSet->end(), symbolFirstSet->begin(), symbolFirstSet->end());
+		followSet->insert(followSet->end(), symbolFirstSet.begin(), symbolFirstSet.end());
 	}
 	std::vector<Symbol>* followSetReturn = new std::vector<Symbol>();
 	for (std::vector<Symbol>::size_type i = 0; i < followSet->size(); i++) {

src/RNGLRParser.cpp

@@ -8,6 +8,20 @@ RNGLRParser::~RNGLRParser() {
 	//
 }
 
+void RNGLRParser::printReconstructedFrontier(int frontier) {
+    std::vector<int> lastFrontier = gss.getFrontier(frontier);
+    for (int j = 0; j < lastFrontier.size(); j++) {
+        std::cout << "State: " << lastFrontier[j] << std::endl;
+        std::vector<std::pair<std::string, ParseAction>> stateParseActions = table.stateAsParseActionVector(lastFrontier[j]);
+        std::set<std::pair<std::string, ParseAction>> noRepeats;
+        for (auto k : stateParseActions)
+            noRepeats.insert(k);
+        for (auto k : noRepeats)
+            std::cout << k.first << " " << k.second.toString(false) << std::endl;
+        std::cout << std::endl;
+    }
+}
+
 NodeTree<Symbol>* RNGLRParser::parseInput(std::string inputString) {
 	input.clear();
 	gss.clear();
@@ -74,7 +88,6 @@ NodeTree<Symbol>* RNGLRParser::parseInput(std::string inputString) {
 		else if (firstActions[i]->action == ParseAction::REDUCE && fullyReducesToNull(firstActions[i]->reduceRule)) {
 			Reduction newReduction = {v0, firstActions[i]->reduceRule->getLeftSide(), 0, getNullableParts(firstActions[i]->reduceRule), NULL};
 			toReduce.push(newReduction);
-			//toReduce.push(std::make_pair(std::make_pair(v0, firstActions[i]->reduceRule->getLeftSide()), 0));
 		}
 	}
 
@@ -89,13 +102,20 @@ NodeTree<Symbol>* RNGLRParser::parseInput(std::string inputString) {
 			std::cout << "Parsing failed on " << input[i].toString() << std::endl;
 			std::cout << "Problem is on line: " << findLine(i) << std::endl;
 			std::cout << "Nearby is:" << std::endl;
-			const int range = 10;
+			int range = 10;
 			for (int j = (i-range >= 0 ? i-range : 0); j < (i+range < input.size() ? i+range : input.size()); j++)
 				if (j == i)
 					std::cout << "||*||*||" << input[j].toString() << "||*||*|| ";
 				else
 					std::cout << input[j].toString() << " ";
 			std::cout << std::endl;
+            range = 3;
+            std::cout << "\n\n\nThe states in the GSS at last frontiers:" << std::endl;
+			for (int j = (i-range >= 0 ? i-range : 0); j < i; j++) {
+                std::cout << "Frontier:" << j << " (would get): " << input[j].toString() << std::endl;
+                printReconstructedFrontier(j);
+            }
+            std::cout << "\n\n\n\n" << std::endl;
             break;
 		}
 

src/Table.cpp

@@ -353,6 +353,17 @@ ParseAction* Table::getShift(int state, Symbol token) {
 	return shift;
 }
 
+std::vector<std::pair<std::string, ParseAction>> Table::stateAsParseActionVector(int state) {
+    std::vector<std::pair<std::string, ParseAction>> reconstructedState;
+    std::vector<std::vector<ParseAction*>*>* stateVec = table[state];
+    for (int i = 0; i < stateVec->size(); i++)
+        if (std::vector<ParseAction*>* forStateAndSymbol = (*stateVec)[i])
+            for (int j = 0; j < forStateAndSymbol->size(); j++)
+                reconstructedState.push_back(std::make_pair(symbolIndexVec[i].toString(),*((*forStateAndSymbol)[j])));
+
+    return reconstructedState;
+}
+
 std::string Table::toString() {
 	std::string concat = "";
 	for (std::vector<Symbol>::size_type i = 0; i < symbolIndexVec.size(); i++)

stdlib/io.krak

@@ -2,6 +2,10 @@ __if_comp__ __C__ __simple_passthrough__ """
 	#include <stdio.h>
 """
 
+void println() {
+    print("\n");
+}
+
 void print(char* toPrint) {
 	__if_comp__ __C__ {
 		__simple_passthrough__ """
@@ -13,7 +17,7 @@ void print(char* toPrint) {
 
 void println(char* toPrint) {
     print(toPrint);
-    print("\n");
+    println();
 }
 
 void print(int toPrint) {
@@ -27,7 +31,7 @@ void print(int toPrint) {
 
 void println(int toPrint) {
     print(toPrint);
-    print("\n");
+    println();
 }
 
 void print(float toPrint) {
@@ -41,6 +45,6 @@ void print(float toPrint) {
 
 void println(float toPrint) {
     print(toPrint);
-    print("\n");
+    println();
 }
 

stdlib/mem.krak

@@ -50,6 +50,12 @@ template <T> T* new() {
 	return new<T>(1);
 }
 
+template <T> void delete(T* toDelete, int itemDestructCount) {
+    for (int i = 0; i < itemDestructCount; i++)
+        toDelete[i].destruct();
+    delete(toDelete);
+}
+
 template <T> void delete(T* toDelete) {
 	free<T>(toDelete);
 }

stdlib/util.krak

@@ -0,0 +1,12 @@
+
+template<T> T greater(T a, T b) {
+    if (a > b)
+        return a;
+    return b;
+}
+
+template<T> T lesser(T a, T b) {
+    if (a > b)
+        return b;
+    return a;
+}

stdlib/vector.krak

@@ -0,0 +1,52 @@
+import mem;
+import util;
+
+typedef template<T> vector {
+    T *data;
+    int size;
+    int available;
+    vector<T> *construct() {
+        size = 0;
+        available = 8;
+        data = new<T>(8);
+        return this;
+    }
+
+    void destruct() {
+        //Destruction of contained data should depend on if the stored things are objects or primitives.
+        delete<T>(data, size);
+    }
+
+    bool resize(int newSize) {
+        T* newData = new<T>(newSize);
+        if (!newData)
+            return false;
+        for (int i = 0; i < lesser(size, newSize); i++;)
+            newData[i] = data[i];
+        delete<T>(data, 0);
+        return true;
+    }
+    
+    T at(int index) {
+        return get(index);
+    }
+    
+    T get(int index) {
+        if (index < 0 || index >= size)
+            return null;
+        return data[index];
+    }
+
+    void set(int index, T dataIn) {
+        if (index < 0 || index  => size)
+            return;
+        data[index] = dataIn;
+    }
+    void addEnd(T dataIn) {
+        if (size < available)
+            size++;
+        else
+            resize(size*2);
+        data[size-1];
+    }
+};

tests/vectorTest.expected_results

@@ -0,0 +1,2 @@
+1337
+

tests/vectorTest.krak

@@ -0,0 +1,15 @@
+import io;
+import vector;
+
+int main() {
+    vector<int> intVec.construct();
+    intVec.addBack(1);
+    intVec.addBack(3);
+    intVec.addBack(3);
+    intVec.addBack(7);
+    for (int i = 0; i < intVec.size(); i++;)
+        print(intVec.at(i));
+    
+    println();
+    return 0;
+}