detect invalid utf8 source

Author: ghewgill

lexer.cpp

@@ -1,8 +1,11 @@
 #include "lexer.h"
 
+#include <algorithm>
 #include <iso646.h>
 #include <sstream>
 
+#include <utf8.h>
+
 #include "util.h"
 
 std::string Token::tostring() const
@@ -70,6 +73,17 @@ std::string Token::tostring() const
 
 std::vector<Token> tokenize(const std::string &source)
 {
+    auto inv = utf8::find_invalid(source.begin(), source.end());
+    if (inv != source.end()) {
+        int line = 1 + std::count(source.begin(), inv, '\n');
+        auto bol = source.rfind('\n', inv-source.begin());
+        if (bol == std::string::npos) {
+            bol = -1;
+        }
+        int column = inv - (source.begin() + bol);
+        fprintf(stderr, "%d:%d invalid utf8 data in source\n", line, column);
+        exit(1);
+    }
     int line = 1;
     int column = 1;
     std::vector<Token> tokens;

run_test.py

@@ -7,7 +7,11 @@
 def run(fn):
     print ("Running {}...".format(fn))
 
-    src = codecs.open(fn, encoding="UTF-8").read().replace("\r\n", "\n")
+    try:
+        src = codecs.open(fn, encoding="UTF-8").read().replace("\r\n", "\n")
+    except UnicodeDecodeError:
+        # One test has invalid UTF-8 data, so read it as the default encoding.
+        src = open(fn).read().replace("\r\n", "\n")
 
     all_comments = re.findall("^%(.*)$", src, re.MULTILINE)
     todo = any("TODO" in x for x in all_comments)