miloignis/kraken
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1 from jorendorff/master

Browse Source 
Merge jorendoff's very kind cleanup (and tests!) of StringReader code
This commit is contained in:
Nathan Braswell
2013-10-26 08:51:37 -07:00
parent e818236e90 83af1b1c5a
commit 752b04b71b
5 changed files with 95 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
include/StringReader.h
View File

@@ -14,8 +14,10 @@ class StringReader
void setString(std::string inputString);
std::string word(bool truncateEnd = true);
std::string line(bool truncateEnd = true);
std::string getTokens(std::vector<std::string> get_chars, bool truncateEnd = true);
std::string getTokens(const char *get_chars, bool truncateEnd = true);
std::string truncateEnd(std::string to_truncate);
static void test();
protected:
private:
std::string rd_string;

4
include/util.h
View File

@@ -10,8 +10,6 @@
#include <sstream>
std::string intToString(int theInt);
std::string truncateEnd(std::string to_truncate);
std::string removeBeginning(std::string to_remove);
std::string replaceExEscape(std::string first, std::string search, std::string replace);
#endif
#endif

8
main.cpp
View File

@@ -17,7 +17,11 @@
int main(int argc, char* argv[]) {
if (argc == 2 && std::string(argv[1]) == "--test") {
StringReader::test();
return 0;
}
std::ifstream programInFile, grammerInFile;
std::ofstream outFile, outFileTransformed, outFileAST;
@@ -147,4 +151,4 @@ int main(int argc, char* argv[]) {
return(0);
}

152
src/StringReader.cpp
View File

@@ -1,4 +1,5 @@
#include "StringReader.h"
#include <cassert>
StringReader::StringReader()
{
@@ -24,84 +25,50 @@ void StringReader::setString(std::string inputString)
std::string StringReader::word(bool truncateEnd)
{
std::vector<std::string> stop_chars;
stop_chars.push_back(" ");
stop_chars.push_back("\n");
stop_chars.push_back("\t");
std::string result = getTokens(stop_chars, truncateEnd);
std::string result = getTokens(" \n\t", truncateEnd);
while (result == " " || result == "\n" || result == "\t")
{
result = getTokens(stop_chars, truncateEnd);
result = getTokens(" \n\t", truncateEnd);
}
return(result);
}
std::string StringReader::line(bool truncateEnd)
{
std::vector<std::string> stop_chars;
stop_chars.push_back("\n");
return getTokens(stop_chars, truncateEnd);
return getTokens("\n", truncateEnd);
}
std::string StringReader::getTokens(std::vector<std::string> stop_chars, bool truncateEnd)
std::string StringReader::getTokens(const char *stop_chars, bool truncateEnd)
{
int found_pos, new_found_pos;
std::string stop_char;
found_pos = rd_string.find(stop_chars[0], str_pos);
stop_char = stop_chars[0];
for (unsigned int i = 1; i < stop_chars.size(); i++)
{
new_found_pos = rd_string.find(stop_chars[i], str_pos);
//Ok, if the position we found is closer than what we have and is not the end of file, OR the position we are at is the end of file
//assign the new found position to the currrent found position
if ( ((new_found_pos <= found_pos) && (new_found_pos != std::string::npos)) || found_pos == std::string::npos )
{
found_pos = new_found_pos;
stop_char = stop_chars[i];
}
}
size_t found_pos = rd_string.find_first_of(stop_chars, str_pos);
if (rd_string[str_pos] == '\"') {
//See if we have an even or odd number of backslashes (that is, this quote is not or is escaped)
//Find the next quote
found_pos = rd_string.find("\"", str_pos+1);
//Check to see if the quote is escaped
int numBackslashes = 0;
int countBack = 1;
while (str_pos-countBack >= 0 && rd_string[str_pos-countBack] == '\\') {
while (found_pos >= countBack && rd_string[found_pos-countBack] == '\\') {
numBackslashes++;
countBack++;
}
//If the quote is not escaped
if (numBackslashes % 2 == 0) {
//Find the next quote
found_pos = rd_string.find("\"", str_pos+1);
//Check to see if the quote is escaped
//While the quote is escaped
while (numBackslashes % 2 == 1) {
//find the next quote
found_pos = rd_string.find("\"", found_pos+1);
//Check to see if it's escaped
numBackslashes = 0;
countBack = 1;
while (found_pos-countBack >= 0 && rd_string[found_pos-countBack] == '\\') {
while (found_pos >= countBack && rd_string[found_pos-countBack] == '\\') {
numBackslashes++;
countBack++;
}
//While the quote is escaped
while (numBackslashes % 2 == 1) {
//find the next quote
found_pos = rd_string.find("\"", found_pos+1);
//Check to see if it's escaped
numBackslashes = 0;
countBack = 1;
while (found_pos-countBack >= 0 && rd_string[found_pos-countBack] == '\\') {
numBackslashes++;
countBack++;
}
}
}
}
if (found_pos == str_pos) //We are at the endline
{
std::string stop_char(1, rd_string[str_pos]);
str_pos++;
return stop_char;
} else if (found_pos == std::string::npos) //We are at the end of the file
@@ -118,18 +85,8 @@ std::string StringReader::getTokens(std::vector<std::string> stop_chars, bool tr
if (rd_string[str_pos] == '\"')
found_pos++;
std::string string_section;
for (; str_pos <= found_pos; str_pos++)
{
string_section += rd_string[str_pos];
}
// if (str_pos <= found_pos) {
// string_section = rd_string.substr(str_pos, found_pos+1);
// str_pos = found_pos+1;
// }
// std::cout << string_section << " - " << str_pos << " - " << found_pos << std::endl;
std::string string_section = rd_string.substr(str_pos, found_pos - str_pos + 1);
str_pos = found_pos + 1;
if (truncateEnd) //Ok, we didn't add the last char, but str_pos now points at that char. So we move it one ahead.
str_pos++;
@@ -137,10 +94,71 @@ std::string StringReader::getTokens(std::vector<std::string> stop_chars, bool tr
}
}
std::string StringReader::truncateEnd(std::string to_truncate)
void StringReader::test()
{
std::string to_return = "";
for (unsigned int i = 0; i < to_truncate.length()-1; i++)
to_return = to_return + to_truncate[i];
return to_return;
{
StringReader reader("\"x\"");
assert(reader.word() == "\"x\"");
assert(reader.word() == "");
}
{
StringReader reader("\"y\" ;\n");
assert(reader.word() == "\"y\"");
assert(reader.word() == ";");
assert(reader.word() == "");
}
{
StringReader reader("Goal = greeting ;\n"
"greeting = \"hello\" | greeting \"world\" ;\n");
assert(reader.word() == "Goal");
assert(reader.word() == "=");
assert(reader.word() == "greeting");
assert(reader.word() == ";");
assert(reader.word() == "greeting");
assert(reader.word() == "=");
assert(reader.word() == "\"hello\"");
assert(reader.word() == "|");
assert(reader.word() == "greeting");
assert(reader.word() == "\"world\"");
assert(reader.word() == ";");
assert(reader.word() == "");
}
{
StringReader reader("one # pretend this is a comment\n"
" two\n");
assert(reader.word() == "one");
assert(reader.word() == "#");
assert(reader.line() == "pretend this is a comment");
assert(reader.word() == "two");
assert(reader.word() == "");
}
{
// Quoted strings can span lines.
StringReader reader("x = \"\n \" ;\n");
assert(reader.word() == "x");
assert(reader.word() == "=");
assert(reader.word() == "\"\n \"");
assert(reader.word() == ";");
assert(reader.word() == "");
}
{
// Strings may contain backslash-escaped quote characters.
StringReader reader( "\"abc\\\"def\\\\\\\\\\\" \"\n");
assert(reader.word() == "\"abc\\\"def\\\\\\\\\\\" \"");
assert(reader.word() == "");
}
{
// A backslash-escaped backslash can be the last character in a string.
StringReader reader( "\"\\\\\" \n");
assert(reader.word() == "\"\\\\\"");
assert(reader.word() == "");
}
std::cout << "StringReader tests pass\n";
}

15
src/util.cpp
View File

@@ -5,21 +5,6 @@ std::string intToString(int theInt) {
converter << theInt;
return converter.str();
}
std::string truncateEnd(std::string to_truncate)
{
std::string to_return = "";
for (unsigned int i = 0; i < to_truncate.length()-1; i++)
to_return = to_return + to_truncate[i];
return to_return;
}
std::string removeBeginning(std::string to_remove)
{
std::string to_return = "";
for (unsigned int i = 1; i < to_remove.length(); i++)
to_return = to_return + to_remove[i];
return to_return;
}
std::string replaceExEscape(std::string first, std::string search, std::string replace) {
size_t pos = 0;