Improved the lexer to be functionally equlivant to the C++ version and ported the tests, commented out the dot generation from Import as it was slowing things down significantly.

stdlib/lexer.krak

@@ -30,16 +30,23 @@ obj lexer {
     fun add_regex(newOne: regex::regex) {
         regs.add(newOne)
     }
+    fun add_regex(newOne: char*) {
+        regs.add(regex::regex(newOne))
+    }
     fun set_input(in: string::string) {
         input = in
     }
     fun next(): symbol::symbol {
+        if (position >= input.length())
+            return symbol::symbol("$EOF$", true)
         var max = regs.map(fun(reg: regex::regex): util::pair<int, string::string> {
                 return util::make_pair(reg.long_match(input.slice(position, -1)), reg.regexString); })
                 .max(fun(first: util::pair<int, string::string>, second: util::pair<int, string::string>): bool
                         { return first.first < second.first; })
+        if (max.first < 0)
+            return symbol::symbol("$INVALID$", true)
         position += max.first
-        return symbol::symbol(input.slice(position-max.first, position), max.second, true)
+        return symbol::symbol(max.second, true, input.slice(position-max.first, position))
     }
 }
 

stdlib/symbol.krak

@@ -1,12 +1,22 @@
 import string
 
-fun symbol(dataIn: char*, nameIn: char*, terminalIn: bool): symbol {
-    var toRet.construct(string::string(dataIn), string::string(nameIn), terminalIn): symbol
+fun symbol(nameIn: char*, terminalIn: bool): symbol {
+    var toRet.construct(string::string(nameIn), terminalIn, string::string("no_value")): symbol
     return toRet
 }
 
-fun symbol(dataIn: string::string, nameIn: string::string, terminalIn: bool): symbol {
-    var toRet.construct(dataIn, nameIn, terminalIn): symbol
+fun symbol(nameIn: string::string, terminalIn: bool): symbol {
+    var toRet.construct(nameIn, terminalIn, string::string("no_value")): symbol
+    return toRet
+}
+
+fun symbol(nameIn: char*, terminalIn: bool, dataIn: char*): symbol {
+    var toRet.construct(string::string(nameIn), terminalIn, string::string(dataIn)): symbol
+    return toRet
+}
+
+fun symbol(nameIn: string::string, terminalIn: bool, dataIn: string::string): symbol {
+    var toRet.construct(nameIn, terminalIn, dataIn): symbol
     return toRet
 }
 
@@ -20,10 +30,10 @@ obj symbol {
         name.construct()
         return this
     }
-    fun construct(dataIn: string::string, nameIn: string::string, terminalIn: bool): symbol* {
-        data.construct(dataIn)
+    fun construct(nameIn: string::string, terminalIn: bool, dataIn: string::string): symbol* {
         name.construct(nameIn)
         terminal = terminalIn
+        data.construct(dataIn)
         return this
     }
     fun destruct() {