Merge pull request #1 from jorendorff/master

Merge jorendoff's very kind cleanup (and tests!) of StringReader code

include/StringReader.h

@@ -14,8 +14,10 @@ class StringReader
         void setString(std::string inputString);
         std::string word(bool truncateEnd = true);
         std::string line(bool truncateEnd = true);
-        std::string getTokens(std::vector<std::string> get_chars, bool truncateEnd = true);
+        std::string getTokens(const char *get_chars, bool truncateEnd = true);
         std::string truncateEnd(std::string to_truncate);
+
+        static void test();
     protected:
     private:
         std::string rd_string;

include/util.h

@@ -10,8 +10,6 @@
 #include <sstream>
 
 std::string intToString(int theInt);
-std::string truncateEnd(std::string to_truncate);
-std::string removeBeginning(std::string to_remove);
 std::string replaceExEscape(std::string first, std::string search, std::string replace);
 
-#endif
+#endif

main.cpp

@@ -17,7 +17,11 @@
 
 
 int main(int argc, char* argv[]) {
-	
+	if (argc == 2 && std::string(argv[1]) == "--test") {
+		StringReader::test();
+		return 0;
+	}
+
 	std::ifstream programInFile, grammerInFile;
 	std::ofstream outFile, outFileTransformed, outFileAST;
 
@@ -147,4 +151,4 @@ int main(int argc, char* argv[]) {
 
 	return(0);
 }
- 
+ 

src/StringReader.cpp

@@ -1,4 +1,5 @@
 #include "StringReader.h"
+#include <cassert>
 
 StringReader::StringReader()
 {
@@ -24,84 +25,50 @@ void StringReader::setString(std::string inputString)
 
 std::string StringReader::word(bool truncateEnd)
 {
-    std::vector<std::string> stop_chars;
+    std::string result = getTokens(" \n\t", truncateEnd);
-    stop_chars.push_back(" ");
-    stop_chars.push_back("\n");
-    stop_chars.push_back("\t");
-    
-
-    std::string result = getTokens(stop_chars, truncateEnd);
     while (result == " " || result == "\n" || result == "\t")
     {
-        result = getTokens(stop_chars, truncateEnd);
+        result = getTokens(" \n\t", truncateEnd);
     }
     return(result);
 }
 
 std::string StringReader::line(bool truncateEnd)
 {
-    std::vector<std::string> stop_chars;
+    return getTokens("\n", truncateEnd);
-    stop_chars.push_back("\n");
-    return getTokens(stop_chars, truncateEnd);
 }
 
-std::string StringReader::getTokens(std::vector<std::string> stop_chars, bool truncateEnd)
+std::string StringReader::getTokens(const char *stop_chars, bool truncateEnd)
 {
-    int found_pos, new_found_pos;
+    size_t found_pos = rd_string.find_first_of(stop_chars, str_pos);
-    std::string stop_char;
-
-    found_pos = rd_string.find(stop_chars[0], str_pos);
-    stop_char = stop_chars[0];
-
-    for (unsigned int i = 1; i < stop_chars.size(); i++)
-    {
-        new_found_pos = rd_string.find(stop_chars[i], str_pos);
-        
-        //Ok, if the position we found is closer than what we have and is not the end of file, OR the position we are at is the end of file
-        //assign the new found position to the currrent found position
-        if ( ((new_found_pos <= found_pos) && (new_found_pos != std::string::npos)) || found_pos == std::string::npos )
-        {
-            found_pos = new_found_pos;
-            stop_char = stop_chars[i];
-        }
-    }
 
     if (rd_string[str_pos] == '\"') {
-        //See if we have an even or odd number of backslashes (that is, this quote is not or is escaped)
+        //Find the next quote
+        found_pos = rd_string.find("\"", str_pos+1);
+        //Check to see if the quote is escaped
         int numBackslashes = 0;
         int countBack = 1;
-        while (str_pos-countBack >= 0 && rd_string[str_pos-countBack] == '\\') {
+        while (found_pos >= countBack && rd_string[found_pos-countBack] == '\\') {
             numBackslashes++;
             countBack++;
         }
-        //If the quote is not escaped
+        //While the quote is escaped
-        if (numBackslashes % 2 == 0) {
+        while (numBackslashes % 2 == 1) {
-            //Find the next quote
+            //find the next quote
-            found_pos = rd_string.find("\"", str_pos+1);
+            found_pos = rd_string.find("\"", found_pos+1);
-            //Check to see if the quote is escaped
+            //Check to see if it's escaped
             numBackslashes = 0;
             countBack = 1;
-            while (found_pos-countBack >= 0 && rd_string[found_pos-countBack] == '\\') {
+            while (found_pos >= countBack && rd_string[found_pos-countBack] == '\\') {
                 numBackslashes++;
                 countBack++;
             }
-            //While the quote is escaped
-            while (numBackslashes % 2 == 1) {
-                //find the next quote
-                found_pos = rd_string.find("\"", found_pos+1);
-                //Check to see if it's escaped
-                numBackslashes = 0;
-                countBack = 1;
-                while (found_pos-countBack >= 0 && rd_string[found_pos-countBack] == '\\') {
-                    numBackslashes++;
-                    countBack++;
-                }
-            }
         }
     }
 
     if (found_pos == str_pos)                                   //We are at the endline
     {
+        std::string stop_char(1, rd_string[str_pos]);
         str_pos++;
         return stop_char;
     } else if (found_pos == std::string::npos)                         //We are at the end of the file
@@ -118,18 +85,8 @@ std::string StringReader::getTokens(std::vector<std::string> stop_chars, bool tr
         if (rd_string[str_pos] == '\"')
             found_pos++;
 
-        std::string string_section;
+        std::string string_section = rd_string.substr(str_pos, found_pos - str_pos + 1);
-
+        str_pos = found_pos + 1;
-        for (; str_pos <= found_pos; str_pos++)
-        {
-            string_section += rd_string[str_pos];
-        }
-
-        // if (str_pos <= found_pos) {
-        //     string_section = rd_string.substr(str_pos, found_pos+1);
-        //     str_pos = found_pos+1;
-        // }
-        // std::cout << string_section << " - " << str_pos << " - " << found_pos << std::endl;
 
         if (truncateEnd)                                       //Ok, we didn't add the last char, but str_pos now points at that char. So we move it one ahead.
             str_pos++;
@@ -137,10 +94,71 @@ std::string StringReader::getTokens(std::vector<std::string> stop_chars, bool tr
     }
 }
 
-std::string StringReader::truncateEnd(std::string to_truncate)
+void StringReader::test()
 {
-    std::string to_return = "";
+    {
-    for (unsigned int i = 0; i < to_truncate.length()-1; i++)
+        StringReader reader("\"x\"");
-        to_return = to_return + to_truncate[i];
+        assert(reader.word() == "\"x\"");
-    return to_return;
+        assert(reader.word() == "");
+    }
+
+    {
+        StringReader reader("\"y\" ;\n");
+        assert(reader.word() == "\"y\"");
+        assert(reader.word() == ";");
+        assert(reader.word() == "");
+    }
+
+    {
+        StringReader reader("Goal = greeting ;\n"
+                            "greeting = \"hello\" | greeting \"world\" ;\n");
+        assert(reader.word() == "Goal");
+        assert(reader.word() == "=");
+        assert(reader.word() == "greeting");
+        assert(reader.word() == ";");
+        assert(reader.word() == "greeting");
+        assert(reader.word() == "=");
+        assert(reader.word() == "\"hello\"");
+        assert(reader.word() == "|");
+        assert(reader.word() == "greeting");
+        assert(reader.word() == "\"world\"");
+        assert(reader.word() == ";");
+        assert(reader.word() == "");
+    }
+
+    {
+        StringReader reader("one   # pretend this is a comment\n"
+                            "    two\n");
+        assert(reader.word() == "one");
+        assert(reader.word() == "#");
+        assert(reader.line() == "pretend this is a comment");
+        assert(reader.word() == "two");
+        assert(reader.word() == "");
+    }
+
+    {
+        // Quoted strings can span lines.
+        StringReader reader("x = \"\n \" ;\n");
+        assert(reader.word() == "x");
+        assert(reader.word() == "=");
+        assert(reader.word() == "\"\n \"");
+        assert(reader.word() == ";");
+        assert(reader.word() == "");
+    }
+
+    {
+        // Strings may contain backslash-escaped quote characters.
+        StringReader reader(    "\"abc\\\"def\\\\\\\\\\\" \"\n");
+        assert(reader.word() == "\"abc\\\"def\\\\\\\\\\\" \"");
+        assert(reader.word() == "");
+    }
+
+    {
+        // A backslash-escaped backslash can be the last character in a string.
+        StringReader reader(    "\"\\\\\" \n");
+        assert(reader.word() == "\"\\\\\"");
+        assert(reader.word() == "");
+    }
+
+    std::cout << "StringReader tests pass\n";
 }

src/util.cpp

@@ -5,21 +5,6 @@ std::string intToString(int theInt) {
 	converter << theInt;
 	return converter.str();
 }
-std::string truncateEnd(std::string to_truncate)
-{
-    std::string to_return = "";
-    for (unsigned int i = 0; i < to_truncate.length()-1; i++)
-        to_return = to_return + to_truncate[i];
-    return to_return;
-}
-
-std::string removeBeginning(std::string to_remove)
-{
-    std::string to_return = "";
-    for (unsigned int i = 1; i < to_remove.length(); i++)
-        to_return = to_return + to_remove[i];
-    return to_return;
-}
 
 std::string replaceExEscape(std::string first, std::string search, std::string replace) {
     size_t pos = 0;