Improved the lexer to be functionally equlivant to the C++ version and ported the tests, commented out the dot generation from Import as it was slowing things down significantly.

tests/test_lexer.expected_results

@@ -1,2 +1,22 @@
 a+: aaaa true
 test: test true
+old contributed tests
+b: b true
+b: b true
+$EOF$: no_value true
+
+a*: aaa true
+b: b true
+a*: aa true
+b: b true
+b: b true
+$EOF$: no_value true
+
+a|b: b true
+$INVALID$: no_value true
+
+xyzzy: xyzzy true
+$EOF$: no_value true
+
+(i|n|t|e)+: intent true
+$EOF$: no_value true

tests/test_lexer.krak

@@ -3,14 +3,55 @@ import regex:*
 import string:*
 import symbol:*
 import io:*
+import util:*
 
 fun main(): int {
     var lex.construct(): lexer
     lex.set_input(string("aaaatesta"))
     lex.add_regex(regex("a+"))
-    lex.add_regex(regex("test"))
+    lex.add_regex("test")
     println(lex.next().to_string())
     println(lex.next().to_string())
+
+    println("old contributed tests")
+
+    {
+        var lex.construct(): lexer
+        lex.add_regex("b")
+        lex.set_input(string("bb"))
+        range(3).for_each(fun(i: int) { println(lex.next().to_string()); } )
+    }
+    println()
+    {
+        var lex.construct(): lexer
+        lex.add_regex("a*")
+        lex.add_regex("b")
+        lex.set_input(string("aaabaabb"))
+        range(6).for_each(fun(i: int) { println(lex.next().to_string()); } )
+    }
+    println()
+    {
+        var lex.construct(): lexer
+        lex.add_regex("a|b")
+        lex.set_input(string("blah"))
+        range(2).for_each(fun(i: int) { println(lex.next().to_string()); } )
+    }
+    println()
+    {
+        var lex.construct(): lexer
+        lex.add_regex("xyzzy")
+        lex.set_input(string("xyzzy"))
+        range(2).for_each(fun(i: int) { println(lex.next().to_string()); } )
+    }
+    println()
+    {
+        var lex.construct(): lexer
+        lex.add_regex("int")
+        lex.add_regex("(i|n|t|e)+")
+        lex.set_input(string("intent"))
+        range(2).for_each(fun(i: int) { println(lex.next().to_string()); } )
+    }
+
     return 0
 }