From 661a104af0f1ef00e3559119245b363ebeac80b3 Mon Sep 17 00:00:00 2001 From: malcolm Date: Fri, 23 Sep 2016 21:44:27 -0400 Subject: [PATCH 1/7] WIP: Initial commit to implement osvr_json_to_c.cpp in Python. --- devtools/osvr_json_to_c.py | 72 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 devtools/osvr_json_to_c.py diff --git a/devtools/osvr_json_to_c.py b/devtools/osvr_json_to_c.py new file mode 100644 index 000000000..0886fe918 --- /dev/null +++ b/devtools/osvr_json_to_c.py @@ -0,0 +1,72 @@ +""" + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http:www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + """ + +import argparse +import sys + + +def migrate_file_data(json_filename, variable_name, output_filename): + """ + Provide a JSON filename, a variable name, and an (optional) output filename. + It reads in the JSON file (discarding comments, turns it back into a string in "compressed" format - that is, + without unneeded whitespaces, etc. It then writes out a file that basically is a C/C++ source file defining a + string array by converting every character to hex. + :return: + """ + try: + # 'with' keyword will handle closing the files + with open(json_filename, 'rb') as json_input: + with open(output_filename, 'w') as cpp_output: + end_of_file = '' + cpp_output.write('static const char {}[] = {{'.format(variable_name)) + line = json_input.readline() + while line != end_of_file: + # TODO: Remove comments not found in key/value of JSON + # TODO: Strip whitespace characters not found in key/value of JSON + # TODO: Handle the End of File newline separately. Keep it instead? + json_file_contents = list(line) + cpp_file_contents = [hex(ord(char)) for char in json_file_contents] + write_to_file = ", ".join(cpp_file_contents) + line = json_input.readline() + if line != end_of_file: + # Add the extra trailing comma to connect data from multiple lines + write_to_file = "{}, ".format(write_to_file) + cpp_output.write(write_to_file) + + cpp_output.write('};\n') + except IOError: + print "Could not read file:", json_file + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('input', help="input file") + parser.add_argument('-o', '--output', help="output file (defaults to standard out)") + parser.add_argument('-s', '--symbol', default='json', help="symbol/variable name to create in generated file") + args = parser.parse_args(argv) + input_filename = args.input + output_filename = args.output + symbol = args.symbol + + # Generate a default output filename based on the input filename if output filename is not provided + if output_filename is None: + filename_parts = input_filename.split('.')[:-1] + filename_parts.append('cpp') + output_filename = '.'.join(filename_parts) + + migrate_file_data(json_filename=input_filename, variable_name=symbol, output_filename=output_filename) + +if __name__ == "__main__": + main(sys.argv[1:]) + From 912531da52ac52407d9a6dae1365cc84bbede93f Mon Sep 17 00:00:00 2001 From: malcolm Date: Sat, 24 Sep 2016 23:38:03 -0400 Subject: [PATCH 2/7] Added comment and whitespace removal. --- devtools/osvr_json_to_c.py | 57 ++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/devtools/osvr_json_to_c.py b/devtools/osvr_json_to_c.py index 0886fe918..1a93927a4 100644 --- a/devtools/osvr_json_to_c.py +++ b/devtools/osvr_json_to_c.py @@ -14,6 +14,36 @@ import argparse import sys +import re + + +# http://stackoverflow.com/questions/2319019/using-regex-to-remove-comments-from-source-files +# http://stackoverflow.com/a/18381470 +def remove_comments(string): + pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)" + # first group captures quoted strings (double or single) + # second group captures comments (//single-line or /* multi-line */) + regex = re.compile(pattern, re.MULTILINE | re.DOTALL) + + def _replacer(match): + # if the 2nd group (capturing comments) is not None, + # it means we have captured a non-quoted (real) comment string. + if match.group(2) is not None: + return "" + else: # otherwise, we will return the 1st group + return match.group(1) # captured quoted-string + return regex.sub(_replacer, string) + + +# http://stackoverflow.com/questions/3609596/python-regular-expression-must-strip-whitespace-except-between-quotes +# http://stackoverflow.com/a/3609802 +def stripwhite(text): + # Remove whitespace characters not found in between double-quotes + lst = text.split('"') + for i, item in enumerate(lst): + if not i % 2: + lst[i] = re.sub("\s+", "", item) + return '"'.join(lst) def migrate_file_data(json_filename, variable_name, output_filename): @@ -32,21 +62,26 @@ def migrate_file_data(json_filename, variable_name, output_filename): cpp_output.write('static const char {}[] = {{'.format(variable_name)) line = json_input.readline() while line != end_of_file: - # TODO: Remove comments not found in key/value of JSON - # TODO: Strip whitespace characters not found in key/value of JSON - # TODO: Handle the End of File newline separately. Keep it instead? - json_file_contents = list(line) - cpp_file_contents = [hex(ord(char)) for char in json_file_contents] - write_to_file = ", ".join(cpp_file_contents) - line = json_input.readline() - if line != end_of_file: + # TODO: Are comments allowed in JSON outside of data? + # http://www.json.org/ + # https://plus.google.com/+DouglasCrockfordEsq/posts/RK8qyGVaGSr + # https://groups.yahoo.com/neo/groups/json/conversations/topics/156 + uncommented_line = remove_comments(line) + stripped_line = stripwhite(uncommented_line) + if stripped_line != '': + # Don't write (now) empty line contents to the file + json_file_contents = list(stripped_line) + cpp_file_contents = [hex(ord(char)) for char in json_file_contents] + write_to_file = ", ".join(cpp_file_contents) # Add the extra trailing comma to connect data from multiple lines write_to_file = "{}, ".format(write_to_file) - cpp_output.write(write_to_file) - + cpp_output.write(write_to_file) + line = json_input.readline() + # TODO: Handle the End of File newline separately. Keep it instead? + cpp_output.write(hex(ord('\n'))) cpp_output.write('};\n') except IOError: - print "Could not read file:", json_file + print "Could not read file:", json_filename def main(argv): From c079e6f926241ffa6cf65aa194fd8f9fb8c490cb Mon Sep 17 00:00:00 2001 From: malcolm Date: Wed, 5 Oct 2016 06:48:51 -0400 Subject: [PATCH 3/7] Fixed formatting issue with single-digit hex values. --- devtools/osvr_json_to_c.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devtools/osvr_json_to_c.py b/devtools/osvr_json_to_c.py index 1a93927a4..36a92d513 100644 --- a/devtools/osvr_json_to_c.py +++ b/devtools/osvr_json_to_c.py @@ -71,14 +71,14 @@ def migrate_file_data(json_filename, variable_name, output_filename): if stripped_line != '': # Don't write (now) empty line contents to the file json_file_contents = list(stripped_line) - cpp_file_contents = [hex(ord(char)) for char in json_file_contents] + cpp_file_contents = ["0x{}".format(char.encode("hex")) for char in json_file_contents] write_to_file = ", ".join(cpp_file_contents) # Add the extra trailing comma to connect data from multiple lines write_to_file = "{}, ".format(write_to_file) cpp_output.write(write_to_file) line = json_input.readline() - # TODO: Handle the End of File newline separately. Keep it instead? - cpp_output.write(hex(ord('\n'))) + # The End of File newline would have been removed in the while loop above. Re-add it. + cpp_output.write("0x{}".format('\n'.encode("hex"))) cpp_output.write('};\n') except IOError: print "Could not read file:", json_filename From ffdbe13f8496d9ceb5784c8e88d2419a6f9dfe28 Mon Sep 17 00:00:00 2001 From: malcolm Date: Sat, 3 Dec 2016 16:43:33 -0500 Subject: [PATCH 4/7] Adjusted migrate_file_data() method signature to allow default keyword args. Added initial unit tests for osvr_json_to_c.py. --- devtools/osvr_json_to_c.py | 26 ++++++---- devtools/test_osvr_json_to_c.py | 85 +++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 devtools/test_osvr_json_to_c.py diff --git a/devtools/osvr_json_to_c.py b/devtools/osvr_json_to_c.py index 36a92d513..d91240da4 100644 --- a/devtools/osvr_json_to_c.py +++ b/devtools/osvr_json_to_c.py @@ -20,6 +20,11 @@ # http://stackoverflow.com/questions/2319019/using-regex-to-remove-comments-from-source-files # http://stackoverflow.com/a/18381470 def remove_comments(string): + """ + + :param string: + :return: string + """ pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)" # first group captures quoted strings (double or single) # second group captures comments (//single-line or /* multi-line */) @@ -38,6 +43,11 @@ def _replacer(match): # http://stackoverflow.com/questions/3609596/python-regular-expression-must-strip-whitespace-except-between-quotes # http://stackoverflow.com/a/3609802 def stripwhite(text): + """ + + :param text: + :return: string + """ # Remove whitespace characters not found in between double-quotes lst = text.split('"') for i, item in enumerate(lst): @@ -46,15 +56,21 @@ def stripwhite(text): return '"'.join(lst) -def migrate_file_data(json_filename, variable_name, output_filename): +def migrate_file_data(json_filename, variable_name='json', output_filename=None): """ - Provide a JSON filename, a variable name, and an (optional) output filename. + Provide a JSON filename, an (optional) variable name, and an (optional) output filename. It reads in the JSON file (discarding comments, turns it back into a string in "compressed" format - that is, without unneeded whitespaces, etc. It then writes out a file that basically is a C/C++ source file defining a string array by converting every character to hex. :return: """ try: + # Generate a default output filename based on the input filename if output filename is not provided + if output_filename is None: + filename_parts = json_filename.split('.')[:-1] + filename_parts.append('cpp') + output_filename = '.'.join(filename_parts) + # 'with' keyword will handle closing the files with open(json_filename, 'rb') as json_input: with open(output_filename, 'w') as cpp_output: @@ -94,12 +110,6 @@ def main(argv): output_filename = args.output symbol = args.symbol - # Generate a default output filename based on the input filename if output filename is not provided - if output_filename is None: - filename_parts = input_filename.split('.')[:-1] - filename_parts.append('cpp') - output_filename = '.'.join(filename_parts) - migrate_file_data(json_filename=input_filename, variable_name=symbol, output_filename=output_filename) if __name__ == "__main__": diff --git a/devtools/test_osvr_json_to_c.py b/devtools/test_osvr_json_to_c.py new file mode 100644 index 000000000..9300f7921 --- /dev/null +++ b/devtools/test_osvr_json_to_c.py @@ -0,0 +1,85 @@ +from os import path +import shutil +import tempfile +import unittest +from osvr_json_to_c import migrate_file_data + + +class OSVRJsonToCTestCase(unittest.TestCase): + """Tests for `osvr_json_to_c.py`.""" + + def setUp(self): + # Create a temporary directory + self.test_dir = tempfile.mkdtemp() + self.test_json_filename = 'test.json' + self.test_json_filepath = path.join(self.test_dir, self.test_json_filename) + with open(self.test_json_filepath, 'w') as json_file: + json_file_contents = """{ + "hello": "world" + }""" + json_file.write(json_file_contents) + + def tearDown(self): + # Remove the directory after the test + shutil.rmtree(self.test_dir) + + def test_set_variable_name_in_output(self): + """ + Does the variable name set as expected? + :return: + """ + symbol = 'variable_name' + temp_output_filepath = path.join(self.test_dir, 'test.cpp') + migrate_file_data(json_filename=self.test_json_filepath, variable_name=symbol) + # 'with' keyword will handle closing the files + with open(temp_output_filepath, 'rb') as cpp_file: + # Reopen the file and check if what we read back is the same + line = cpp_file.readline() + self.assertTrue('static const char {}[]'.format(symbol) in line) + + def test_default_variable_name_in_output(self): + """ + Does the variable name default as expected? + :return: + """ + symbol = 'json' + output_filepath = path.join(self.test_dir, 'test.cpp') + migrate_file_data(json_filename=self.test_json_filepath) + # 'with' keyword will handle closing the files + with open(output_filepath, 'rb') as cpp_file: + # Reopen the file and check if what we read back is the same + line = cpp_file.readline() + self.assertTrue('static const char {}[]'.format(symbol) in line) + + def test_default_output_filename(self): + """ + Does the output filename default as expected? + :return: + """ + migrate_file_data(json_filename=self.test_json_filepath) + output_filepath = path.join(self.test_dir, 'test.cpp') + self.assertTrue(path.isfile(output_filepath)) + + def test_set_output_filename(self): + """ + Does the output filename set as expected? + :return: + """ + output_filename = 'test_output' + migrate_file_data(json_filename=self.test_json_filepath, output_filename=output_filename) + output_filepath = path.join(self.test_dir, '{}.cpp'.format(output_filename)) + print "input_filepath: {}".format(self.test_json_filepath) + print "output_filepath: {}".format(output_filepath) + # TODO: This assertion fails. Fix it. + self.assertTrue(path.isfile(output_filepath)) + + def test_file_read_error(self): + """ + Does migrate_file_data() return an IOError as expected if the input file is unreadable? + :return: + """ + # TODO: Intentionally break this unit test as a placeholder until it can be properly implemented + self.assertTrue(False) + +if __name__ == '__main__': + unittest.main() From aff7298d29a085ded9b4b82a951870ce7015fc77 Mon Sep 17 00:00:00 2001 From: malcolm Date: Sat, 3 Dec 2016 17:05:50 -0500 Subject: [PATCH 5/7] Fixed IOError message. Fixed unit tests test_set_output_filename and test_file_read_error. --- devtools/osvr_json_to_c.py | 2 +- devtools/test_osvr_json_to_c.py | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/devtools/osvr_json_to_c.py b/devtools/osvr_json_to_c.py index d91240da4..93eec425b 100644 --- a/devtools/osvr_json_to_c.py +++ b/devtools/osvr_json_to_c.py @@ -97,7 +97,7 @@ def migrate_file_data(json_filename, variable_name='json', output_filename=None) cpp_output.write("0x{}".format('\n'.encode("hex"))) cpp_output.write('};\n') except IOError: - print "Could not read file:", json_filename + return "Could not read file: {}".format(json_filename) def main(argv): diff --git a/devtools/test_osvr_json_to_c.py b/devtools/test_osvr_json_to_c.py index 9300f7921..13284eeec 100644 --- a/devtools/test_osvr_json_to_c.py +++ b/devtools/test_osvr_json_to_c.py @@ -66,20 +66,17 @@ def test_set_output_filename(self): :return: """ output_filename = 'test_output' - migrate_file_data(json_filename=self.test_json_filepath, output_filename=output_filename) output_filepath = path.join(self.test_dir, '{}.cpp'.format(output_filename)) - print "input_filepath: {}".format(self.test_json_filepath) - print "output_filepath: {}".format(output_filepath) - # TODO: This assertion fails. Fix it. + migrate_file_data(json_filename=self.test_json_filepath, output_filename=output_filepath) self.assertTrue(path.isfile(output_filepath)) def test_file_read_error(self): """ - Does migrate_file_data() return an IOError as expected if the input file is unreadable? + Does migrate_file_data() return an error message as expected if the input file is unreadable? :return: """ - # TODO: Intentionally break this unit test as a placeholder until it can be properly implemented - self.assertTrue(False) + error_message = migrate_file_data(json_filename='') + self.assertTrue('Could not read file:' in error_message) if __name__ == '__main__': unittest.main() From 9bcb5c92117a030ec6aef20dc4b0907e5cbe5c76 Mon Sep 17 00:00:00 2001 From: malcolm Date: Sat, 3 Dec 2016 17:21:56 -0500 Subject: [PATCH 6/7] Added test_hello_world_json unit test. --- devtools/osvr_json_to_c.py | 2 +- devtools/test_osvr_json_to_c.py | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/devtools/osvr_json_to_c.py b/devtools/osvr_json_to_c.py index 93eec425b..a15689ad9 100644 --- a/devtools/osvr_json_to_c.py +++ b/devtools/osvr_json_to_c.py @@ -71,7 +71,7 @@ def migrate_file_data(json_filename, variable_name='json', output_filename=None) filename_parts.append('cpp') output_filename = '.'.join(filename_parts) - # 'with' keyword will handle closing the files + # 'with' keyword will handle the closing of files with open(json_filename, 'rb') as json_input: with open(output_filename, 'w') as cpp_output: end_of_file = '' diff --git a/devtools/test_osvr_json_to_c.py b/devtools/test_osvr_json_to_c.py index 13284eeec..73b8709fb 100644 --- a/devtools/test_osvr_json_to_c.py +++ b/devtools/test_osvr_json_to_c.py @@ -13,6 +13,7 @@ def setUp(self): self.test_dir = tempfile.mkdtemp() self.test_json_filename = 'test.json' self.test_json_filepath = path.join(self.test_dir, self.test_json_filename) + # 'with' keyword will handle the closing of files with open(self.test_json_filepath, 'w') as json_file: json_file_contents = """{ "hello": "world" @@ -31,7 +32,7 @@ def test_set_variable_name_in_output(self): symbol = 'variable_name' temp_output_filepath = path.join(self.test_dir, 'test.cpp') migrate_file_data(json_filename=self.test_json_filepath, variable_name=symbol) - # 'with' keyword will handle closing the files + # 'with' keyword will handle the closing of files with open(temp_output_filepath, 'rb') as cpp_file: # Reopen the file and check if what we read back is the same line = cpp_file.readline() @@ -45,7 +46,7 @@ def test_default_variable_name_in_output(self): symbol = 'json' output_filepath = path.join(self.test_dir, 'test.cpp') migrate_file_data(json_filename=self.test_json_filepath) - # 'with' keyword will handle closing the files + # 'with' keyword will handle the closing of files with open(output_filepath, 'rb') as cpp_file: # Reopen the file and check if what we read back is the same line = cpp_file.readline() @@ -78,5 +79,20 @@ def test_file_read_error(self): error_message = migrate_file_data(json_filename='') self.assertTrue('Could not read file:' in error_message) + def test_hello_world_json(self): + """ + Does migrate_file_data() create the proper cpp file contents given the hello world test data? + :return: + """ + expected_cpp_contents = "static const char json[] = {0x7b, 0x22, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x22, 0x3a, 0x22, 0x77, 0x6f," \ + " 0x72, 0x6c, 0x64, 0x22, 0x7d, 0x0a};\n" + output_filepath = path.join(self.test_dir, 'test.cpp') + migrate_file_data(json_filename=self.test_json_filepath) + # 'with' keyword will handle the closing of files + with open(output_filepath, 'rb') as cpp_file: + # Reopen the file and check if what we read back is the same + cpp_contents = cpp_file.read() + self.assertEqual(cpp_contents, expected_cpp_contents) + if __name__ == '__main__': unittest.main() From 791b0dbb9d4042f17a21b505f7a1d123cdb5c22b Mon Sep 17 00:00:00 2001 From: malcolm Date: Sat, 3 Dec 2016 23:58:26 -0500 Subject: [PATCH 7/7] Added Apache License to test_osvr_json_to_c.py. --- devtools/test_osvr_json_to_c.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/devtools/test_osvr_json_to_c.py b/devtools/test_osvr_json_to_c.py index 73b8709fb..d27cb66ae 100644 --- a/devtools/test_osvr_json_to_c.py +++ b/devtools/test_osvr_json_to_c.py @@ -1,3 +1,17 @@ +""" + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http:www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + """ + from os import path import shutil import tempfile