From 623e24bb2c944ae9de281cdd4ab40d803775ae31 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Mon, 3 Apr 2017 11:54:25 +0200 Subject: [PATCH 01/12] Workaround for solving unbounded recursion when hlookup a value in same region as the value is located --- src/compile/c/compile_to_c.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/compile/c/compile_to_c.rb b/src/compile/c/compile_to_c.rb index c3de876..987d554 100644 --- a/src/compile/c/compile_to_c.rb +++ b/src/compile/c/compile_to_c.rb @@ -52,11 +52,17 @@ def rewrite(formulae, sheet_names, output) output.puts "#{static_or_not}ExcelValue #{name}() {" output.puts " static ExcelValue result;" output.puts " if(variable_set[#{@variable_set_counter}] == 1) { return result;}" + output.puts " static int recursion_prevention;" + output.puts " if(recursion_prevention == 1) { return result;}" + output.puts " recursion_prevention = 1;" + mapper.initializers.each do |i| - output.puts " #{i}" + output.puts " #{i}".gsub("#{worksheet_c_name}_#{cell.downcase}()", "ZERO") end + output.puts " result = #{calculation};" output.puts " variable_set[#{@variable_set_counter}] = 1;" + output.puts " recursion_prevention = 0;" output.puts " return result;" output.puts "}" output.puts From 6f7f40c9a81e11980bc3f47e151df32e0fc7735f Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Thu, 6 Apr 2017 08:44:11 +0200 Subject: [PATCH 02/12] Fix most errors from named references being replaced by blanks in the inliner --- src/commands/excel_to_x.rb | 1 + src/simplify/inline_formulae.rb | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/commands/excel_to_x.rb b/src/commands/excel_to_x.rb index 27956c2..c16623b 100644 --- a/src/commands/excel_to_x.rb +++ b/src/commands/excel_to_x.rb @@ -1006,6 +1006,7 @@ def replace_formulae_with_their_results inline_replacer = InlineFormulaeAst.new inline_replacer.references = @formulae inline_replacer.inline_ast = inline_ast_decision + inline_replacer.named_references = @named_references value_replacer = MapFormulaeToValues.new value_replacer.original_excel_filename = excel_file diff --git a/src/simplify/inline_formulae.rb b/src/simplify/inline_formulae.rb index bd3f835..63ad574 100644 --- a/src/simplify/inline_formulae.rb +++ b/src/simplify/inline_formulae.rb @@ -1,10 +1,10 @@ class InlineFormulaeAst - attr_accessor :references, :current_sheet_name, :inline_ast + attr_accessor :references, :current_sheet_name, :inline_ast, :named_references attr_accessor :count_replaced - def initialize(references = nil, current_sheet_name = nil, inline_ast = nil) - @references, @current_sheet_name, @inline_ast = references, [current_sheet_name], inline_ast + def initialize(references = nil, current_sheet_name = nil, inline_ast = nil, named_references = nil) + @references, @current_sheet_name, @inline_ast, @named_references = references, [current_sheet_name], inline_ast, named_references @count_replaced = 0 @inline_ast ||= lambda { |sheet, ref, references| true } # Default is to always inline end @@ -50,9 +50,17 @@ def sheet_reference(ast) return unless ast[2][0] == :cell sheet = ast[1].to_sym ref = ast[2][1].to_s.upcase.gsub('$','').to_sym - # FIXME: Need to check if valid worksheet and return [:error, "#REF!"] if not - # Now check user preference on this - return unless inline_ast.call(sheet,ref, references) + ref_org = ast[2][1].to_s.gsub('$','').to_sym + + if named_references[ref_org] # Fix issue for named_reference that is mapped to blank + sheet = named_references[ref_org][1] + ref = named_references[ref_org][2][1] + else + # FIXME: Need to check if valid worksheet and return [:error, "#REF!"] if not + # Now check user preference on this + return unless inline_ast.call(sheet,ref, references) + end + ast_to_inline = ast_or_blank(sheet, ref) @count_replaced += 1 current_sheet_name.push(sheet) @@ -79,6 +87,11 @@ def cell(ast) def ast_or_blank(sheet, ref) ast_to_inline = references[[sheet, ref]] return ast_to_inline if ast_to_inline + + if named_references.key?(ref.downcase) + return named_references[ref.downcase] + end + # Need to add a new blank cell and return ast for an inlined blank references[[sheet, ref]] = [:blank] [:inlined_blank] From 5d622f344f7e3b6ee2bfd8a105a6e54bfb22db60 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Thu, 6 Apr 2017 08:59:22 +0200 Subject: [PATCH 03/12] Make the rewrite of shared formulae aware of named references --- src/commands/excel_to_x.rb | 2 +- src/rewrite/ast_copy_formula.rb | 7 ++++++- src/rewrite/rewrite_shared_formulae.rb | 8 ++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/commands/excel_to_x.rb b/src/commands/excel_to_x.rb index c16623b..f9268ef 100644 --- a/src/commands/excel_to_x.rb +++ b/src/commands/excel_to_x.rb @@ -704,7 +704,7 @@ def rewrite_row_and_column_references # them so every cell has its own definition def rewrite_shared_formulae_into_normal_formulae log.info "Rewriting shared formulae" - @formulae_shared = RewriteSharedFormulae.rewrite( @formulae_shared, @formulae_shared_targets) + @formulae_shared = RewriteSharedFormulae.rewrite( @formulae_shared, @formulae_shared_targets, @named_references) @shared_formulae_targets = :no_longer_needed # Allow the targets to be garbage collected. end diff --git a/src/rewrite/ast_copy_formula.rb b/src/rewrite/ast_copy_formula.rb index 5c47866..7b8f121 100644 --- a/src/rewrite/ast_copy_formula.rb +++ b/src/rewrite/ast_copy_formula.rb @@ -4,10 +4,12 @@ class AstCopyFormula attr_accessor :rows_to_move attr_accessor :columns_to_move + attr_accessor :named_references - def initialize + def initialize(named_references) self.rows_to_move = 0 self.columns_to_move = 0 + self.named_references = named_references end DO_NOT_MAP = {:number => true, :string => true, :blank => true, :null => true, :error => true, :boolean_true => true, :boolean_false => true, :operator => true, :comparator => true} @@ -26,6 +28,9 @@ def copy(ast) def cell(reference) r = Reference.for(reference) + if self.named_references.key?(reference) + return [:cell, reference] + end [:cell,r.offset(rows_to_move,columns_to_move)] end diff --git a/src/rewrite/rewrite_shared_formulae.rb b/src/rewrite/rewrite_shared_formulae.rb index 2978e15..831c6cc 100644 --- a/src/rewrite/rewrite_shared_formulae.rb +++ b/src/rewrite/rewrite_shared_formulae.rb @@ -5,7 +5,7 @@ def self.rewrite(*args) new.rewrite(*args) end - def rewrite(formula_shared, formula_shared_targets) + def rewrite(formula_shared, formula_shared_targets, named_references) @output = {} @formula_shared_targets = formula_shared_targets @@ -13,13 +13,13 @@ def rewrite(formula_shared, formula_shared_targets) copy_range = a[0] shared_formula_identifier = a[1] shared_ast = a[2] - share_formula(ref, shared_ast, copy_range, shared_formula_identifier) + share_formula(ref, shared_ast, copy_range, shared_formula_identifier, named_references) end @output end - def share_formula(ref, shared_ast, copy_range, shared_formula_identifier) - copier = AstCopyFormula.new + def share_formula(ref, shared_ast, copy_range, shared_formula_identifier, named_references) + copier = AstCopyFormula.new(named_references) copy_range = Area.for(copy_range) copy_range.calculate_excel_variables From 57573d578cf16890b2b013fca13f1e277ab786d6 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Thu, 6 Apr 2017 10:26:09 +0200 Subject: [PATCH 04/12] Fix that avoid indirect references to be resolved as blanks and thereby be replaced with zeros --- src/commands/excel_to_x.rb | 2 +- .../replace_references_to_blanks_with_zeros.rb | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/commands/excel_to_x.rb b/src/commands/excel_to_x.rb index f9268ef..bce2c1b 100644 --- a/src/commands/excel_to_x.rb +++ b/src/commands/excel_to_x.rb @@ -919,7 +919,7 @@ def simplify(cells = @formulae) @replace_arrays_with_single_cells_replacer ||= ReplaceArraysWithSingleCellsAst.new @replace_string_joins_on_ranges_replacer ||= ReplaceStringJoinOnRangesAST.new @sheetless_cell_reference_replacer ||= RewriteCellReferencesToIncludeSheetAst.new - @replace_references_to_blanks_with_zeros ||= ReplaceReferencesToBlanksWithZeros.new(@formulae, nil, inline_ast_decision) + @replace_references_to_blanks_with_zeros ||= ReplaceReferencesToBlanksWithZeros.new(@formulae, nil, inline_ast_decision, @named_references) @fix_subtotal_of_subtotals ||= FixSubtotalOfSubtotals.new(@formulae) # FIXME: Bodge to put it here as well, but seems to be required column_and_row_function_replacement = ReplaceColumnAndRowFunctionsAST.new diff --git a/src/simplify/replace_references_to_blanks_with_zeros.rb b/src/simplify/replace_references_to_blanks_with_zeros.rb index b06d007..e678333 100644 --- a/src/simplify/replace_references_to_blanks_with_zeros.rb +++ b/src/simplify/replace_references_to_blanks_with_zeros.rb @@ -1,10 +1,10 @@ class ReplaceReferencesToBlanksWithZeros - attr_accessor :references, :current_sheet_name, :inline_ast + attr_accessor :references, :current_sheet_name, :inline_ast, :named_references attr_accessor :count_replaced - def initialize(references = nil, current_sheet_name = nil, inline_ast = nil) - @references, @current_sheet_name, @inline_ast = references, [current_sheet_name], inline_ast + def initialize(references = nil, current_sheet_name = nil, inline_ast = nil, named_references) + @references, @current_sheet_name, @inline_ast, @named_references = references, [current_sheet_name], inline_ast, named_references @count_replaced = 0 @inline_ast ||= lambda { |sheet, ref, references| true } # Default is to always inline end @@ -18,8 +18,13 @@ def map(ast) case ast.first when :sheet_reference return ast unless ast[2][0] == :cell - sheet = ast[1].to_sym - ref = ast[2][1].to_s.upcase.gsub('$','').to_sym + if @named_references.key?(ast[2][1]) + sheet = named_references[ast[2][1]][1] + ref = named_references[ast[2][1]][2].last + else + sheet = ast[1].to_sym + ref = ast[2][1].to_s.upcase.gsub('$','').to_sym + end when :cell sheet = current_sheet_name.last ref = ast[1].to_s.upcase.gsub('$', '').to_sym From 2ed656f63e0fd8a3fd5174ac067218997414d542 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Fri, 7 Apr 2017 12:00:42 +0200 Subject: [PATCH 05/12] Fix for hlookup with strings --- src/compile/c/excel_to_c_runtime.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/compile/c/excel_to_c_runtime.c b/src/compile/c/excel_to_c_runtime.c index a22e10d..e96d343 100644 --- a/src/compile/c/excel_to_c_runtime.c +++ b/src/compile/c/excel_to_c_runtime.c @@ -2359,11 +2359,18 @@ static ExcelValue hlookup(ExcelValue lookup_value_v,ExcelValue lookup_table_v, E for(i=0; i< columns; i++) { possible_match_v = array[i]; if(lookup_value_v.type != possible_match_v.type) continue; - if(more_than(possible_match_v,lookup_value_v).number == true) { - if(i == 0) return NA; - return array[((((int) row_number_v.number)-1)*columns)+(i-1)]; - } else { - last_good_match = i; + if(lookup_value_v.type == ExcelString && possible_match_v.type == ExcelString) + { + if (strstr(possible_match_v.string, lookup_value_v.string) != NULL) + last_good_match = i; + } else + { + if(more_than(possible_match_v,lookup_value_v).number == true) { + if(i == 0) return NA; + return array[((((int) row_number_v.number)-1)*columns)+(i-1)]; + } else { + last_good_match = i; + } } } return array[((((int) row_number_v.number)-1)*columns)+(last_good_match)]; From ae830a255cedffdf78be27470e54069860c88746 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Thu, 11 May 2017 09:02:13 +0200 Subject: [PATCH 06/12] Extend recursion prevention such that sheets does not need to be reloaded --- src/commands/excel_to_c.rb | 1 + src/compile/c/compile_to_c.rb | 9 +++++---- src/compile/c/excel_to_c_runtime.c | 6 ++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/commands/excel_to_c.rb b/src/commands/excel_to_c.rb index eaf96fb..5e697ec 100644 --- a/src/commands/excel_to_c.rb +++ b/src/commands/excel_to_c.rb @@ -43,6 +43,7 @@ def write_out_excel_as_code o.puts "// #{excel_file} approximately translated into C" o.puts "// definitions" o.puts "#define NUMBER_OF_REFS #{number_of_refs}" + o.puts "#define NUMBER_OF_RECURSION_PREVENT_VARS #{number_of_refs}" # This is only known after creating CompileToC, number_of_refs overapproximate it o.puts "#define EXCEL_FILENAME #{excel_file.inspect}" o.puts "// end of definitions" o.puts diff --git a/src/compile/c/compile_to_c.rb b/src/compile/c/compile_to_c.rb index 987d554..461cf1d 100644 --- a/src/compile/c/compile_to_c.rb +++ b/src/compile/c/compile_to_c.rb @@ -14,6 +14,7 @@ def rewrite(formulae, sheet_names, output) self.settable ||= lambda { |ref| false } self.gettable ||= lambda { |ref| true } @variable_set_counter ||= 0 + @recursion_prevention_counter ||= 0 mapper = MapFormulaeToC.new mapper.sheet_names = sheet_names @@ -52,9 +53,8 @@ def rewrite(formulae, sheet_names, output) output.puts "#{static_or_not}ExcelValue #{name}() {" output.puts " static ExcelValue result;" output.puts " if(variable_set[#{@variable_set_counter}] == 1) { return result;}" - output.puts " static int recursion_prevention;" - output.puts " if(recursion_prevention == 1) { return result;}" - output.puts " recursion_prevention = 1;" + output.puts " if(recursion_prevention[#{@recursion_prevention_counter}] == 1) { return result;}" + output.puts " recursion_prevention[#{@recursion_prevention_counter}] = 1;" mapper.initializers.each do |i| output.puts " #{i}".gsub("#{worksheet_c_name}_#{cell.downcase}()", "ZERO") @@ -62,13 +62,14 @@ def rewrite(formulae, sheet_names, output) output.puts " result = #{calculation};" output.puts " variable_set[#{@variable_set_counter}] = 1;" - output.puts " recursion_prevention = 0;" + output.puts " recursion_prevention[#{@recursion_prevention_counter}] = 0;" output.puts " return result;" output.puts "}" output.puts end end @variable_set_counter += 1 + @recursion_prevention_counter += 1 mapper.reset rescue Exception => e puts "Exception at #{ref} #{ast}" diff --git a/src/compile/c/excel_to_c_runtime.c b/src/compile/c/excel_to_c_runtime.c index e96d343..846366a 100644 --- a/src/compile/c/excel_to_c_runtime.c +++ b/src/compile/c/excel_to_c_runtime.c @@ -10,6 +10,10 @@ #define NUMBER_OF_REFS 0 #endif + + + + #ifndef EXCEL_FILENAME #define EXCEL_FILENAME "NoExcelFilename" #endif @@ -146,11 +150,13 @@ static void free_all_allocated_memory() { } static int variable_set[NUMBER_OF_REFS]; +static int recursion_prevention[NUMBER_OF_RECURSION_PREVENT_VARS]; // Resets all cached and malloc'd values void reset() { free_all_allocated_memory(); memset(variable_set, 0, sizeof(variable_set)); + memset(recursion_prevention, 0, sizeof(recursion_prevention)); } // Handy macros From f82be5262091d1559e2076231541c6fbd7c53419 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Mon, 22 May 2017 15:54:30 +0200 Subject: [PATCH 07/12] Disabling of optimization in debug mode --- bin/excel_to_c | 1 + src/commands/excel_to_x.rb | 44 ++++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/bin/excel_to_c b/bin/excel_to_c index 07c5880..3b3f071 100755 --- a/bin/excel_to_c +++ b/bin/excel_to_c @@ -55,6 +55,7 @@ END opts.on('-d','--debug', "Does not perform final optimisations of spreadsheet, leaving the resulting code more similar to the original worksheet, but potentially slower") do |sheet| command.should_inline_formulae_that_are_only_used_once = false + command.replace_reference_to_blanks_with_zeros = false command.extract_repeated_parts_of_formulae = false end diff --git a/src/commands/excel_to_x.rb b/src/commands/excel_to_x.rb index bce2c1b..7e4880e 100644 --- a/src/commands/excel_to_x.rb +++ b/src/commands/excel_to_x.rb @@ -108,6 +108,11 @@ class ExcelToX # * false - the compiler leaves calculations in their original cells expanded. This may make debugging easier attr_accessor :should_inline_formulae_that_are_only_used_once + # Optional attribute, Boolean. + # * true (default) - the compiler replaces references to blanks with zeros. This should increase performance + # * false - the compiler leaves references to blanks. This make setable cells more predicatable and easing debugging. + attr_accessor :replace_reference_to_blanks_with_zeros + # Optional attribute, Boolean. # * true (default) - the compiler attempts to extract bits of calculation that appear in more than one formula into separate methods. This should increase performance # * false - the compiler leaves calculations fully expanded. This may make debugging easier @@ -247,6 +252,7 @@ def set_defaults # Setting this to false may make it easier to figure out errors self.extract_repeated_parts_of_formulae = true if @extract_repeated_parts_of_formulae == nil self.should_inline_formulae_that_are_only_used_once = true if @should_inline_formulae_that_are_only_used_once == nil + self.replace_reference_to_blanks_with_zeros = true if @should_inline_formulae_that_are_only_used_once == nil # This setting is used for debugging, and makes the system only do the conversion on a subset of the worksheets if self.isolate @@ -947,8 +953,10 @@ def simplify(cells = @formulae) @wrap_formulae_that_return_arrays_replacer.map(ast) column_and_row_function_replacement.current_reference = ref.last column_and_row_function_replacement.replace(ast) - @replace_references_to_blanks_with_zeros.current_sheet_name = ref.first - @replace_references_to_blanks_with_zeros.map(ast) + if replace_reference_to_blanks_with_zeros + @replace_references_to_blanks_with_zeros.current_sheet_name = ref.first + @replace_references_to_blanks_with_zeros.map(ast) + end @fix_subtotal_of_subtotals.map(ast) rescue Exception => e log.fatal "Exception when simplifying #{ref}: #{ast}" @@ -1041,9 +1049,11 @@ def replace_formulae_with_their_results if offset_replacement.replace(ast) references_that_need_updating[ref] = ast end - # FIXME: Shouldn't need to wrap ref.fist in an array - inline_replacer.current_sheet_name = [ref.first] - inline_replacer.map(ast) + if should_inline_formulae_that_are_only_used_once + # FIXME: Shouldn't need to wrap ref.fist in an array + inline_replacer.current_sheet_name = [ref.first] + inline_replacer.map(ast) + end # If a formula references a cell containing a value, the reference is replaced with the value (e.g., if A1 := 2 and A2 := A1 + 1 then becomes: A2 := 2 + 1) #require 'pry'; binding.pry if ref == [:"Outputs - Summary table", :E77] value_replacer.map(ast) @@ -1058,9 +1068,11 @@ def replace_formulae_with_their_results end - @named_references.each do |ref, ast| - inline_replacer.current_sheet_name = ref.is_a?(Array) ? [ref.first] : [] - inline_replacer.map(ast) + if should_inline_formulae_that_are_only_used_once + @named_references.each do |ref, ast| + inline_replacer.current_sheet_name = ref.is_a?(Array) ? [ref.first] : [] + inline_replacer.map(ast) + end end simplify(references_that_need_updating) @@ -1258,6 +1270,16 @@ def ensure_there_is_a_good_set_of_cells_that_can_be_set_at_runtime @cells_that_can_be_set_at_runtime = cells_with_settable_values end + def make_blank_referenced_cells_settable(ref, ast) + unless replace_reference_to_blanks_with_zeros + if ast == nil + ast = [:number, 0.0] + @formulae[ref] = ast + end + end + ast + end + # Returns a list of cells that are: # 1. Simple values (e.g., a string or a number) # 2. That are referenced in other formulae @@ -1267,13 +1289,13 @@ def cells_with_settable_values log.info "Generating a good set of cells that should be settable" counter = CountFormulaReferences.new - count = counter.count(@formulae) + countList = counter.count(@formulae) settable_cells = {} settable_types = [:blank,:number,:null,:string,:shared_string,:constant,:percentage,:error,:boolean_true,:boolean_false] - count.each do |ref,count| + countList.each do |ref,count| next unless count >= 1 # No point making a cell that isn't reference settable - ast = @formulae[ref] + ast = make_blank_referenced_cells_settable(ref, @formulae[ref]) next unless ast # Sometimes empty cells are referenced. next unless settable_types.include?(ast.first) settable_cells[ref.first] ||= [] From 918304b4466f35b7a32c52b15448ccebc5464005 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Mon, 29 May 2017 15:08:00 +0200 Subject: [PATCH 08/12] Return 0 for cells that have been optimized away --- src/commands/excel_to_c.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/excel_to_c.rb b/src/commands/excel_to_c.rb index 5e697ec..4b7ec34 100644 --- a/src/commands/excel_to_c.rb +++ b/src/commands/excel_to_c.rb @@ -218,7 +218,7 @@ def method_missing(name, *arguments) end def get(name) - return 0 unless C.respond_to?(name) + return nil unless C.respond_to?(name) ruby_value_from_excel_value(C.send(name)) end From e7688c454c05314912dace68ad08157e273a02bb Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Tue, 30 May 2017 09:56:46 +0200 Subject: [PATCH 09/12] Fixes to make changes pass rspec tests --- Gemfile | 1 + Gemfile.lock | 4 +++- src/rewrite/ast_copy_formula.rb | 2 +- src/rewrite/rewrite_shared_formulae.rb | 2 +- src/simplify/inline_formulae.rb | 2 +- src/simplify/replace_references_to_blanks_with_zeros.rb | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 5c5ea3d..83cb019 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,7 @@ gem 'rubypeg' group :test do gem 'rspec' + gem 'minitest' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 89d13a0..674b724 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,6 +9,7 @@ GEM ffi (1.9.17) interception (0.5) method_source (0.8.2) + minitest (5.10.1) ox (2.4.9) pry (0.10.4) coderay (~> 1.1.0) @@ -43,6 +44,7 @@ PLATFORMS DEPENDENCIES ffi + minitest ox pry-rescue pry-stack_explorer @@ -50,4 +52,4 @@ DEPENDENCIES rubypeg BUNDLED WITH - 1.13.7 + 1.14.6 diff --git a/src/rewrite/ast_copy_formula.rb b/src/rewrite/ast_copy_formula.rb index 7b8f121..aa41073 100644 --- a/src/rewrite/ast_copy_formula.rb +++ b/src/rewrite/ast_copy_formula.rb @@ -6,7 +6,7 @@ class AstCopyFormula attr_accessor :columns_to_move attr_accessor :named_references - def initialize(named_references) + def initialize(named_references = {}) self.rows_to_move = 0 self.columns_to_move = 0 self.named_references = named_references diff --git a/src/rewrite/rewrite_shared_formulae.rb b/src/rewrite/rewrite_shared_formulae.rb index 831c6cc..7d4bb64 100644 --- a/src/rewrite/rewrite_shared_formulae.rb +++ b/src/rewrite/rewrite_shared_formulae.rb @@ -5,7 +5,7 @@ def self.rewrite(*args) new.rewrite(*args) end - def rewrite(formula_shared, formula_shared_targets, named_references) + def rewrite(formula_shared, formula_shared_targets, named_references = {}) @output = {} @formula_shared_targets = formula_shared_targets diff --git a/src/simplify/inline_formulae.rb b/src/simplify/inline_formulae.rb index 63ad574..fcc3377 100644 --- a/src/simplify/inline_formulae.rb +++ b/src/simplify/inline_formulae.rb @@ -3,7 +3,7 @@ class InlineFormulaeAst attr_accessor :references, :current_sheet_name, :inline_ast, :named_references attr_accessor :count_replaced - def initialize(references = nil, current_sheet_name = nil, inline_ast = nil, named_references = nil) + def initialize(references = nil, current_sheet_name = nil, inline_ast = nil, named_references = {}) @references, @current_sheet_name, @inline_ast, @named_references = references, [current_sheet_name], inline_ast, named_references @count_replaced = 0 @inline_ast ||= lambda { |sheet, ref, references| true } # Default is to always inline diff --git a/src/simplify/replace_references_to_blanks_with_zeros.rb b/src/simplify/replace_references_to_blanks_with_zeros.rb index e678333..da23513 100644 --- a/src/simplify/replace_references_to_blanks_with_zeros.rb +++ b/src/simplify/replace_references_to_blanks_with_zeros.rb @@ -3,7 +3,7 @@ class ReplaceReferencesToBlanksWithZeros attr_accessor :references, :current_sheet_name, :inline_ast, :named_references attr_accessor :count_replaced - def initialize(references = nil, current_sheet_name = nil, inline_ast = nil, named_references) + def initialize(references = nil, current_sheet_name = nil, inline_ast = nil, named_references = {}) @references, @current_sheet_name, @inline_ast, @named_references = references, [current_sheet_name], inline_ast, named_references @count_replaced = 0 @inline_ast ||= lambda { |sheet, ref, references| true } # Default is to always inline From 9995524b41014b552eb461aae5ab0b12e6f294f5 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Wed, 23 Aug 2017 16:29:02 +0200 Subject: [PATCH 10/12] Add support for resetting individual sheets --- src/commands/excel_to_c.rb | 17 ++++++++++++++++- src/compile/c/compile_to_c.rb | 29 ++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/commands/excel_to_c.rb b/src/commands/excel_to_c.rb index 4b7ec34..16a763b 100644 --- a/src/commands/excel_to_c.rb +++ b/src/commands/excel_to_c.rb @@ -84,6 +84,7 @@ def write_out_excel_as_code c.settable = settable c.gettable = gettable c.rewrite(@formulae, @worksheet_c_names, o) + c.reset_sheets(@worksheet_c_names, o) # Output the named references @@ -191,7 +192,15 @@ def write_fuby_ffi_interface name = output_name.downcase o = output("#{name}.rb") - + + sheet_resets = [] + worksheets do |sheet_name, _| + sheet_name = c_name_for_worksheet_name(sheet_name) + sheet_resets.push("def reset_#{sheet_name}") + sheet_resets.push(" C.reset_#{sheet_name}") + sheet_resets.push("end\n") + end + code = < Date: Tue, 15 Jan 2019 10:56:35 +0100 Subject: [PATCH 11/12] Cleanup for publishing code --- src/commands/common_command_line_options.rb | 10 +- src/commands/excel_to_x.rb | 230 +++++++++--------- src/compile/c/compile_to_c.rb | 6 +- src/compile/c/excel_to_c_runtime.c | 6 +- ...replace_references_to_blanks_with_zeros.rb | 2 +- 5 files changed, 125 insertions(+), 129 deletions(-) diff --git a/src/commands/common_command_line_options.rb b/src/commands/common_command_line_options.rb index 9279b04..12e160e 100644 --- a/src/commands/common_command_line_options.rb +++ b/src/commands/common_command_line_options.rb @@ -31,7 +31,7 @@ def self.set(command:, options:, generates:, extension:) end options.on('-s', '--settable INPUT_WORKSHEET', "Translate value cells in INPUT_WORKSHEET as settable variables in the #{generates}.") do |sheet| - + command.cells_that_can_be_set_at_runtime = { sheet => :all } end @@ -57,16 +57,16 @@ def self.set(command:, options:, generates:, extension:) puts options exit end - + options.set_summary_width 35 end - + def self.parse(options:, command:, arguments:) begin options.parse!(arguments) - rescue OptionParser::ParseError => e - STDERR.puts e.message, "\n", options + rescue OptionParser::ParseError => e + STDERR.puts e.message, "\n", options return false end diff --git a/src/commands/excel_to_x.rb b/src/commands/excel_to_x.rb index fa4c219..e4cdfd6 100644 --- a/src/commands/excel_to_x.rb +++ b/src/commands/excel_to_x.rb @@ -14,27 +14,27 @@ class XMLFileNotFoundException < Exception; end # FIXME: Replacing with c compatible names everywhere class ExcelToX - + # Required attribute. The source excel file. This must be .xlsx not .xls attr_accessor :excel_file - + # Optional attribute. The output directory. # If not specified, will be '#{excel_file_name}/c' attr_accessor :output_directory - + # Optional attribute. The name of the resulting ruby or c file and ruby or ruby ffi module name. Defaults to excelspreadsheet attr_accessor :output_name # Optional attribute. The excel file will be translated to xml and stored here. # If not specified, will be '#{excel_file_name}/xml' attr_accessor :xml_directory - + # Optional attribute. Specifies which cells have setters created in the c code so their values can be altered at runtime. - # It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet + # It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet # should be setable, or an array of cell names on that sheet that should be settable (e.g., A1) attr_accessor :cells_that_can_be_set_at_runtime - # Optional attribute. Specifies which named references to be turned into setters. + # Optional attribute. Specifies which named references to be turned into setters. # # NB: Named references are assumed to include table names. # @@ -52,7 +52,7 @@ class ExcelToX # # By default no named references are output attr_accessor :named_references_that_can_be_set_at_runtime - + # Optional attribute. Specifies which cells must appear in the final generated code. # The default is that all cells in the original spreadsheet appear in the final code. # @@ -62,10 +62,10 @@ class ExcelToX # * specified as a cell that can be set at runtime # may be excluded from the final generated code. # - # It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet + # It is a hash. The keys are the sheet names. The values are either the symbol :all to specify that all cells on that sheet # should be lept, or an array of cell names on that sheet that should be kept (e.g., A1) attr_accessor :cells_to_keep - + # Optional attribute. Specifies which named references should be included in the output # # NB: Named references are assumed to include table names. @@ -85,17 +85,17 @@ class ExcelToX # # By default, no named references are output attr_accessor :named_references_to_keep - + # Optional attribute. Boolean. Not relevant to all types of code output # * true - the generated c code is compiled # * false - the generated c code is not compiled (default, unless actuall_run_tests is specified as true) attr_accessor :actually_compile_code - # Optional attribute. Boolean. + # Optional attribute. Boolean. # * true - the generated tests are run # * false (default) - the generated tests are not run attr_accessor :actually_run_tests - + # This is the log file, if set it needs to respond to the same methods as the standard logger library attr_accessor :log @@ -108,7 +108,7 @@ class ExcelToX # * true (default) - the compiler attempts to inline any calculation that is done in another cell, but only referred to by this cell. This should increase performance # * false - the compiler leaves calculations in their original cells expanded. This may make debugging easier attr_accessor :should_inline_formulae_that_are_only_used_once - + # Optional attribute, Boolean. # * true (default) - the compiler replaces references to blanks with zeros. This should increase performance # * false - the compiler leaves references to blanks. This make setable cells more predicatable and easing debugging. @@ -122,18 +122,18 @@ class ExcelToX # Optional attribute, Array. Default nil # This is used to help debug large spreadsheets that aren't working correctly. - # If set to the name of a worksheet then ONLY that worksheet will be run through the + # If set to the name of a worksheet then ONLY that worksheet will be run through the # optimisation and simplification code. Will also override cells_to_keep to keep all # cells on tha sheet and nothing else. attr_accessor :isolate # Optional attribute, Boolean. Default false # If set to true, will persevere through some errors where it can rather than aborting - # immediately. This can be helpful in getting to grips with conversion errors on a - # really messy sheet, since it allows you to see all the errors at once and which are + # immediately. This can be helpful in getting to grips with conversion errors on a + # really messy sheet, since it allows you to see all the errors at once and which are # really fatal. attr_accessor :persevere - + # This is a private method, default is a hash, keys are cell references. # The code will dump debuging information about the given cells as they # progress through the conversion @@ -149,17 +149,17 @@ def go! set_defaults log.info "Excel to Code version #{ExcelToCode.version}\n\n" - + # These turn the excel into xml on disk sort_out_output_directories unzip_excel - + # These gets the named references, worksheet names and shared strings out of the excel extract_data_from_workbook - + # This gets all the formulae, values and tables out of the worksheets extract_data_from_worksheets - + # This checks that the user inputs of which cells to keep are in the right # format and refer to sheets and references that actually exist clean_cells_that_can_be_set_at_runtime @@ -168,7 +168,7 @@ def go! clean_named_references_to_keep clean_named_references_that_can_be_set_at_runtime - # This is an early check that the functions in the extracted data have + # This is an early check that the functions in the extracted data have # all got an implementation in, at least, the ruby code check_all_functions_implemented @@ -176,7 +176,7 @@ def go! # into a series of required cell references transfer_named_references_to_keep_into_cells_to_keep transfer_named_references_that_can_be_set_at_runtime_into_cells_that_can_be_set_at_runtime - + # These perform some translations to tsimplify the excel # Including: # * Turning row and column references (e.g., A:A) to areas, based on the size of the worksheet @@ -189,14 +189,14 @@ def go! rewrite_shared_formulae_into_normal_formulae rewrite_array_formulae combine_formulae_types - + # These perform a series of transformations to the information # with the intent of removing any redundant calculations that are in the excel. # Replacing shared strings and named references with their actual values, tidying arithmetic simplify_arithmetic simplify - # If nothing has been specified in named_references_that_can_be_set_at_runtime + # If nothing has been specified in named_references_that_can_be_set_at_runtime # or in cells_that_can_be_set_at_runtime, then we assume that # all value cells should be settable if they are referenced by # any other forumla. @@ -221,7 +221,7 @@ def go! # This actually creates the code (implemented in subclasses) write_code - + # These compile and run the code version of the excel (implemented in subclasses) compile_code run_tests @@ -230,28 +230,28 @@ def go! log.info "The generated code is available in #{File.join(output_directory)}" end - + # If an attribute hasn't been specified, specifies a good default value here. def set_defaults raise ExcelToCodeException.new("No excel file has been specified") unless excel_file - + self.output_directory ||= Dir.pwd unless self.xml_directory self.xml_directory ||= Dir.mktmpdir @delete_xml_directory_at_end = true end - + self.output_name ||= "Excelspreadsheet" - + self.cells_that_can_be_set_at_runtime ||= {} - + # Make sure the relevant directories exist self.excel_file = File.expand_path(excel_file) self.output_directory = File.expand_path(output_directory) # For debugging self.dump_steps ||= {} - + # Set up our log file unless self.log self.log = Logger.new(STDOUT) @@ -287,11 +287,11 @@ def set_defaults end # Creates any directories that are needed - def sort_out_output_directories + def sort_out_output_directories FileUtils.mkdir_p(output_directory) FileUtils.mkdir_p(xml_directory) end - + # FIXME: Replace these with pure ruby versions? def unzip_excel log.info "Removing old folders" @@ -300,7 +300,7 @@ def unzip_excel execute_system_command 'unzip', '-q', excel_file, '-d', xml_directory end - def execute_system_command(*args) + def execute_system_command(*args) c = args.shelljoin output = `#{c}` unless $?.exitstatus == 0 @@ -309,12 +309,12 @@ def execute_system_command(*args) exit 1 end end - + # The excel workbook.xml and allied relationship files knows about # shared strings, named references and the actual human readable - # names of each of the worksheets. + # names of each of the worksheets. # - # In this method we also loop through each of the individual + # In this method we also loop through each of the individual # worksheet files to work out their dimensions def extract_data_from_workbook extract_shared_strings @@ -330,7 +330,7 @@ def extract_shared_strings @shared_strings = ExtractSharedStrings.extract(i) end end - + # Excel keeps a central list of named references. This includes those # that are local to a specific worksheet. # They are put in a @named_references hash @@ -357,7 +357,7 @@ def extract_named_references if e.respond_to?(:'ref=') e.ref = ['Named reference', name] end - if persevere + if persevere $stderr.puts e.message $stderr.puts "--persevere true, so setting #{name} = #REF!" @named_references[name] = [:error, "#REF!"] @@ -378,7 +378,7 @@ def deep_copy(ast) end end - # Named references can be simple cell references, + # Named references can be simple cell references, # or they can be ranges, or errors, or table references # this function converts all the different types into # arrays of cell references @@ -396,18 +396,18 @@ def convert_named_references_into_simple_form end # Excel keeps a list of worksheet names. To get the mapping between - # human and computer name correct we have to look in the workbook + # human and computer name correct we have to look in the workbook # relationships files. We also need to mangle the name into something # that will work ok as a filesystem or program name def extract_worksheet_names log.info "Extracting worksheet names" - + worksheet_rids = {} xml('workbook.xml') do |i| worksheet_rids = ExtractWorksheetNames.extract(i) # {'worksheet_name' => 'rId3' ...} end - + xml_for_rids = {} xml('_rels','workbook.xml.rels') do |i| xml_for_rids = ExtractRelationships.extract(i) #{ 'rId3' => "worlsheets/sheet1.xml" } @@ -460,7 +460,7 @@ def clean_cells_that_can_be_set_at_runtime # Make sure references are of the form A1, not a1 or A$1 cells_that_can_be_set_at_runtime.keys.each do |sheet| next unless cells_that_can_be_set_at_runtime[sheet].is_a?(Array) - cells_that_can_be_set_at_runtime[sheet] = cells_that_can_be_set_at_runtime[sheet].map do |reference| + cells_that_can_be_set_at_runtime[sheet] = cells_that_can_be_set_at_runtime[sheet].map do |reference| reference.gsub('$','').upcase.to_sym end end @@ -470,13 +470,13 @@ def clean_cells_that_can_be_set_at_runtime # Make sure that all the cell names are upcase symbols and don't have any $ in them def clean_cells_to_keep return unless cells_to_keep - + # Make sure sheet names are symbols cells_to_keep.keys.each do |sheet| next if sheet.is_a?(Symbol) cells_to_keep[sheet.to_sym] = cells_to_keep.delete(sheet) end - + # Make sure the sheets actually exist cells_to_keep.keys.each do |sheet| next if @worksheet_xmls.has_key?(sheet) @@ -489,7 +489,7 @@ def clean_cells_to_keep next unless cells_to_keep[sheet].is_a?(Array) cells_to_keep[sheet] = cells_to_keep[sheet].map { |reference| reference.gsub('$','').upcase.to_sym } end - end + end # Make sure named_references_to_keep are lowercase symbols def clean_named_references_to_keep @@ -548,19 +548,19 @@ def clean_named_references_that_can_be_set_at_runtime end end - + # For each worksheet, extract the useful bits from the excel xml def extract_data_from_worksheets # All are hashes of the format ["SheetName", "A1"] => [:number, "1"] # This one has a series of table references extractor = ExtractDataFromWorksheet.new extractor.persevere = persevere - + # Loop through the worksheets # FIXME: make xml_filename be the IO object? worksheets do |name, xml_filename| - # This is used in debugging large worksheets to limit + # This is used in debugging large worksheets to limit # the optimisation to a particular worksheet if isolate log.info "Only extracting values from #{name}: #{!isolate.include?(name)}" @@ -575,7 +575,7 @@ def extract_data_from_worksheets @values = extractor.values @formulae_simple = extractor.formulae_simple @formulae_shared = extractor.formulae_shared - @formulae_shared_targets = extractor.formulae_shared_targets + @formulae_shared_targets = extractor.formulae_shared_targets @formulae_array = extractor.formulae_array @worksheets_dimensions = extractor.worksheets_dimensions @table_rids = extractor.table_rids @@ -584,7 +584,7 @@ def extract_data_from_worksheets @table_data = {} extract_tables end - + # To extract a table we need to look in the worksheet for table references # then we look in the relationships file for the filename that matches that # reference and contains the table data. Then we consolidate all the data @@ -599,9 +599,9 @@ def extract_tables xml(File.join('worksheets','_rels',"#{File.basename(xml_filename)}.rels")) do |i| xml_for_rids = ExtractRelationships.extract(i) end - + # Then extract the individual tables - array_of_table_rids.each do |rid| + array_of_table_rids.each do |rid| xml(File.join('worksheets', xml_for_rids[rid])) do |i| ExtractTable.extract(worksheet_name, i).each do |table_name, details| name = table_name.downcase @@ -624,7 +624,7 @@ def extract_tables @table_data.each do |name, reference| @table_data[name] = @replace_ranges_with_array_literals_replacer.map(reference) end - + end def check_all_functions_implemented @@ -638,25 +638,25 @@ def check_all_functions_implemented end unless functions_used.empty? - + log.fatal "The following functions have not been implemented in excel_to_code #{ExcelToCode.version}:" functions_used.each do |f| log.fatal f.to_s - end - + end + log.fatal "Check for a new version of excel_to_code at https://github.com/tamc/excel_to_code" log.fatal "Or follow the instractions at https://github.com/tamc/excel_to_code/blob/master/doc/How_to_add_a_missing_function.md to implement the function yourself" exit end end - + # This makes sure that cells_to_keep includes named_references_to_keep def transfer_named_references_to_keep_into_cells_to_keep log.info "Transfering named references to keep into cells to keep" return unless @named_references_to_keep if @named_references_to_keep == :all - @named_references_to_keep = @named_references.keys + @table_areas.keys + @named_references_to_keep = @named_references.keys + @table_areas.keys # If the user has specified named_references_to_keep == :all, but there are none, fall back if @named_references_to_keep.empty? log.warn "named_references_to_keep == :all, but no named references found" @@ -714,7 +714,7 @@ def add_ref_to_hash(ref, hash) log.error "Weird reference in named reference #{ref}" end end - + # Excel can include references to strings rather than the strings # themselves. This harmonises so the strings themselves are always # used. @@ -725,17 +725,17 @@ def rewrite_values_to_remove_shared_strings r.map(ast) end end - - # In Excel we can have references like A:Z and 5:20 which mean all cells in columns + + # In Excel we can have references like A:Z and 5:20 which mean all cells in columns # A to Z and all cells in rows 5 to 20 respectively. This function translates these - # into more conventional references (e.g., A5:Z20) based on the maximum area that + # into more conventional references (e.g., A5:Z20) based on the maximum area that # has been used on a worksheet def rewrite_row_and_column_references log.info "Rewriting row and column references" # FIXME: Refactor dimension_objects = {} - @worksheets_dimensions.map do |sheet_name, dimension| - dimension_objects[sheet_name] = WorksheetDimension.new(dimension) + @worksheets_dimensions.map do |sheet_name, dimension| + dimension_objects[sheet_name] = WorksheetDimension.new(dimension) end mapper = MapColumnAndRowRangeAst.new(nil, dimension_objects) @@ -755,7 +755,7 @@ def rewrite_row_and_column_references end # FIXME: Could we now nil off the dimensions? Or do we need for indirects? end - + # Excel can share formula definitions across cells. This function unshares # them so every cell has its own definition def rewrite_shared_formulae_into_normal_formulae @@ -784,7 +784,7 @@ def rewrite_array_formulae emergency_indirect_replacement_bodge.references = @values emergency_indirect_replacement_bodge.tables = @tables emergency_indirect_replacement_bodge.named_references = @named_references - + @formulae_array.each do |ref, details| begin @shared_string_replacer.map(details.last) @@ -829,7 +829,7 @@ def combine_formulae_types log.info "Sheet contains #{@formulae.size} cells" end - + # Turns aritmetic with many arguments (1+2+3+4) into arithmetic with only # two arguments (((1+2)+3)+4), taking into account operator precedence. def simplify_arithmetic @@ -839,7 +839,7 @@ def simplify_arithmetic simplify_arithmetic_replacer.map(ast) end end - + # This ensures that all gettable and settable values appear in the output # even if they are blank in the underlying excel def required_references @@ -866,7 +866,7 @@ def required_references end end - # In some situations also need to add the named references + # In some situations also need to add the named references if @named_references_to_keep @named_references_to_keep.each do |name| ref = @named_references[name] || @table_areas[name] @@ -918,7 +918,7 @@ def work_out_which_named_references_can_be_set_at_runtime cell = Reference.for(ref[2][1]).unfix.to_sym s = cells_that_can_be_set[sheet] if s && ( s == :all || s.include?(cell) ) - @named_references_that_can_be_set_at_runtime << name + @named_references_that_can_be_set_at_runtime << name cells_that_can_be_set_due_to_named_reference[sheet] << cell.to_sym cells_that_can_be_set_due_to_named_reference[sheet].uniq! end @@ -932,8 +932,8 @@ def work_out_which_named_references_can_be_set_at_runtime s && s.include?(cell) end if settable - @named_references_that_can_be_set_at_runtime << name - ref.each do |r| + @named_references_that_can_be_set_at_runtime << name + ref.each do |r| sheet = r[1] cell = r[2][1].gsub('$','') cells_that_can_be_set_due_to_named_reference[sheet] << cell.to_sym @@ -965,16 +965,16 @@ def filter_named_references def debug_dump(ref, ast, location = "") return unless dump_steps[ref] - puts "#{location}: #{ref} = #{ast}" + puts "#{location}: #{ref} = #{ast}" end - + def simplify(cells = @formulae) log.info "Simplifying cells" @shared_string_replacer ||= ReplaceSharedStringAst.new(@shared_strings) @replace_arithmetic_on_ranges_replacer ||= ReplaceArithmeticOnRangesAst.new @wrap_formulae_that_return_arrays_replacer ||= WrapFormulaeThatReturnArraysAndAReNotInArraysAst.new - @named_reference_replacer ||= ReplaceNamedReferencesAst.new(@named_references, nil, @table_data) + @named_reference_replacer ||= ReplaceNamedReferencesAst.new(@named_references, nil, @table_data) @table_reference_replacer ||= ReplaceTableReferenceAst.new(@tables) @replace_ranges_with_array_literals_replacer ||= ReplaceRangesWithArrayLiteralsAst.new @replace_arrays_with_single_cells_replacer ||= ReplaceArraysWithSingleCellsAst.new @@ -1046,8 +1046,8 @@ def must_keep?(ref) end must_keep_in_sheet.include?(ref.last) end - - def inline_ast_decision + + def inline_ast_decision @inline_ast_decision ||= lambda do |sheet, cell, references| ref = [sheet,cell] if must_keep?(ref) @@ -1097,7 +1097,7 @@ def replace_formulae_with_their_results offset_replacement = ReplaceOffsetsWithReferencesAst.new cell_address_replacement = ReplaceCellAddressesWithReferencesAst.new - begin + begin number_of_passes += 1 log.info "Starting pass #{number_of_passes} on #{@cells_with_formulae.size} cells" @@ -1160,8 +1160,8 @@ def replace_formulae_with_their_results end while replacements_made_in_the_last_pass > 0 && number_of_passes < 50 end - - + + # If 'cells to keep' are specified, then other cells are removed, unless # they are required to calculate the value of a cell in 'cells to keep'. def remove_any_cells_not_needed_for_outputs @@ -1169,8 +1169,8 @@ def remove_any_cells_not_needed_for_outputs # If 'cells to keep' isn't specified, then ALL cells are kept return unless cells_to_keep && !cells_to_keep.empty? - - # Work out what cells the cells in 'cells to keep' need + + # Work out what cells the cells in 'cells to keep' need # in order to be able to calculate their values identifier = IdentifyDependencies.new identifier.references = @formulae @@ -1183,7 +1183,7 @@ def remove_any_cells_not_needed_for_outputs end end end - + # On top of that, we don't want to remove any cells # that have been specified as 'settable' worksheets do |name,xml_filename| @@ -1191,13 +1191,13 @@ def remove_any_cells_not_needed_for_outputs next unless s if s == :all identifier.add_depedencies_for(name) - else + else s.each do |ref| identifier.add_depedencies_for(name,ref) end end end - + # Now we actually go ahead and remove the cells r = RemoveCells.new r.cells_to_keep = identifier.dependencies @@ -1206,7 +1206,7 @@ def remove_any_cells_not_needed_for_outputs r.rewrite(@values) r.rewrite(@cells_with_formulae) end - + # If a cell is only referenced from one other cell, then it is inlined into that other cell # e.g., A1 := B3+B6 ; B1 := A1 + B3 becomes: B1 := (B3 + B6) + B3. A1 is removed. def inline_formulae_that_are_only_used_once @@ -1215,7 +1215,7 @@ def inline_formulae_that_are_only_used_once # First step is to calculate how many times each cell is referenced by another cell counter = CountFormulaReferences.new count = counter.count(@formulae) - + # This takes the decision: # 1. If a cell is in the list of cells to keep, then it is never inlined # 2. Otherwise, it is inlined if only one other cell refers to it. @@ -1227,7 +1227,7 @@ def inline_formulae_that_are_only_used_once count[[sheet,cell]] == 1 # i.e., inline if used only once end end - + r = InlineFormulaeAst.new r.references = @formulae r.inline_ast = inline_ast_decision @@ -1241,11 +1241,11 @@ def inline_formulae_that_are_only_used_once end end end - + # This comes up with a list of references to test, in the form of a file called 'References to test'. # It is structured to contain one reference per row: # worksheet_c_name \t ref \t value_ast - # These will be sorted so that later refs depend on earlier refs. This should mean that the first test that + # These will be sorted so that later refs depend on earlier refs. This should mean that the first test that # fails will be the root cause of the problem def create_sorted_references_to_test log.info "Creating references to test" @@ -1254,8 +1254,8 @@ def create_sorted_references_to_test # First get the list of references we should test @values.each do |ref, value| - if !cells_to_keep || - cells_to_keep.empty? || + if !cells_to_keep || + cells_to_keep.empty? || (cells_to_keep[ref.first] && ( cells_to_keep[ref.first] == :all || cells_to_keep[ref.first].include?(ref.last) @@ -1280,23 +1280,23 @@ def create_sorted_references_to_test # e.g., A1 := (B1 + B3) + B10; A2 := (B1 + B3) + 3 gets transformed to: Common1 := B1 + B3 ; A1 := Common1 + B10 ; A2 := Common1 + 3 def separate_formulae_elements log.info "Looking for repeated bits of formulae" - - + + identifier = IdentifyRepeatedFormulaElements.new repeated_elements = identifier.count(@cells_with_formulae) - - # We apply a threshold that something needs to be used twice for us to bother separating it out. + + # We apply a threshold that something needs to be used twice for us to bother separating it out. # FIXME: This threshold is arbitrary repeated_elements.delete_if do |element,count| count < 2 end - + # Translate the repeated elements into a code of the form [:cell, "common#{1}"] index = 0 repeated_element_ast = {} repeated_elements.each do |ast, count| repeated_element_ast[ast.dup] = [:cell, "common#{index}"] - index +=1 + index +=1 end r = ReplaceCommonElementsInFormulae.new @@ -1313,12 +1313,12 @@ def separate_formulae_elements end end - + # This puts back in an optimisation that excel carries out by making sure that # two copies of the same value actually refer to the same underlying spot in memory def replace_values_with_constants log.info "Replacing values with constants" - + # First do it in the formulae r = MapValuesToConstants.new @formulae.each do |ref, ast| @@ -1331,13 +1331,13 @@ def replace_values_with_constants @constants = r.constants.invert end - - # If nothing has been specified in named_references_that_can_be_set_at_runtime + + # If nothing has been specified in named_references_that_can_be_set_at_runtime # or in cells_that_can_be_set_at_runtime, then we assume that # all value cells should be settable if they are referenced by # any other forumla. def ensure_there_is_a_good_set_of_cells_that_can_be_set_at_runtime - # By this stage, if named_references were set, then cells_that_can_be_set_at_runtime will + # By this stage, if named_references were set, then cells_that_can_be_set_at_runtime will # have been set to match return unless @cells_that_can_be_set_at_runtime.empty? @cells_that_can_be_set_at_runtime = cells_with_settable_values @@ -1376,13 +1376,13 @@ def cells_with_settable_values end return settable_cells end - + # UTILITY FUNCTIONS def settable settable_refs = @cells_that_can_be_set_at_runtime if settable_refs - lambda { |ref| + lambda { |ref| sheet = ref.first cell = ref.last if settable_refs[sheet] @@ -1399,11 +1399,11 @@ def settable lambda { |ref| false } end end - + def gettable if @cells_to_keep gettable_refs = @cells_to_keep - lambda { |ref| + lambda { |ref| sheet = ref.first cell = ref.last if gettable_refs[sheet] @@ -1420,17 +1420,17 @@ def gettable lambda { |ref| true } end end - + def c_name_for_worksheet_name(name) @worksheet_c_names[name.to_s] end - + def worksheets @worksheet_xmls.each do |name, filename| yield name, filename end end - + def xml(*args, &block) args.flatten! filename = File.join(xml_directory,'xl',*args) @@ -1447,7 +1447,7 @@ def xml(*args, &block) f end end - + def output(*args) args.flatten! File.open(File.join(output_directory,*args),'w') @@ -1460,7 +1460,7 @@ def close(*args) f.close end end - + def ruby_module_name @ruby_module_name = output_name.sub(/^[a-z\d]*/) { $&.capitalize } @ruby_module_name = @ruby_module_name.gsub(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" }.gsub('/', '::') diff --git a/src/compile/c/compile_to_c.rb b/src/compile/c/compile_to_c.rb index 6e64ed8..7994b42 100644 --- a/src/compile/c/compile_to_c.rb +++ b/src/compile/c/compile_to_c.rb @@ -2,14 +2,14 @@ require 'set' class CompileToC - + attr_accessor :settable attr_accessor :gettable attr_accessor :variable_set_counter attr_accessor :variable_set_sheet_hash attr_accessor :recursion_prevention_sheet_hash attr_accessor :allow_unknown_functions - + def self.rewrite(*args) self.new.rewrite(*args) end @@ -37,7 +37,7 @@ def rewrite(formulae, sheet_names, output) worksheet_c_name = mapper.sheet_names[worksheet.to_s] || worksheet.to_s calculation = mapper.map(ast) name = worksheet_c_name.length > 0 ? "#{worksheet_c_name}_#{cell.downcase}" : cell.downcase - + # Declare function as static so it can be inlined where possible static_or_not = gettable.call(ref) ? "" : "static " diff --git a/src/compile/c/excel_to_c_runtime.c b/src/compile/c/excel_to_c_runtime.c index 77655a3..b4347e5 100644 --- a/src/compile/c/excel_to_c_runtime.c +++ b/src/compile/c/excel_to_c_runtime.c @@ -10,10 +10,6 @@ #define NUMBER_OF_REFS 0 #endif - - - - #ifndef EXCEL_FILENAME #define EXCEL_FILENAME "NoExcelFilename" #endif @@ -3027,7 +3023,7 @@ static ExcelValue hlookup(ExcelValue lookup_value_v,ExcelValue lookup_table_v, E { if (strstr(possible_match_v.string, lookup_value_v.string) != NULL) last_good_match = i; - } else + } else { if(more_than(possible_match_v,lookup_value_v).number == true) { if(i == 0) return NA; diff --git a/src/simplify/replace_references_to_blanks_with_zeros.rb b/src/simplify/replace_references_to_blanks_with_zeros.rb index da23513..ad50ff5 100644 --- a/src/simplify/replace_references_to_blanks_with_zeros.rb +++ b/src/simplify/replace_references_to_blanks_with_zeros.rb @@ -18,7 +18,7 @@ def map(ast) case ast.first when :sheet_reference return ast unless ast[2][0] == :cell - if @named_references.key?(ast[2][1]) + if @named_references.key?(ast[2][1]) sheet = named_references[ast[2][1]][1] ref = named_references[ast[2][1]][2].last else From 448f0546893bb71ca319d73ff5de2c7392c79a07 Mon Sep 17 00:00:00 2001 From: "Andreas E. Dalsgaard" Date: Mon, 14 Jan 2019 14:06:51 +0100 Subject: [PATCH 12/12] Use c worksheet name to make test case pass --- src/compile/c/compile_to_c.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/compile/c/compile_to_c.rb b/src/compile/c/compile_to_c.rb index 7994b42..05b1232 100644 --- a/src/compile/c/compile_to_c.rb +++ b/src/compile/c/compile_to_c.rb @@ -2,14 +2,14 @@ require 'set' class CompileToC - + attr_accessor :settable attr_accessor :gettable attr_accessor :variable_set_counter attr_accessor :variable_set_sheet_hash attr_accessor :recursion_prevention_sheet_hash attr_accessor :allow_unknown_functions - + def self.rewrite(*args) self.new.rewrite(*args) end @@ -37,7 +37,7 @@ def rewrite(formulae, sheet_names, output) worksheet_c_name = mapper.sheet_names[worksheet.to_s] || worksheet.to_s calculation = mapper.map(ast) name = worksheet_c_name.length > 0 ? "#{worksheet_c_name}_#{cell.downcase}" : cell.downcase - + # Declare function as static so it can be inlined where possible static_or_not = gettable.call(ref) ? "" : "static " @@ -79,10 +79,12 @@ def rewrite(formulae, sheet_names, output) output.puts end end - @variable_set_sheet_hash[worksheet.to_s.downcase].add(@variable_set_counter) - @recursion_prevention_sheet_hash[worksheet.to_s.downcase].add(@variable_set_counter) - @variable_set_counter += 1 - @recursion_prevention_counter += 1 + unless worksheet_c_name.empty? + @variable_set_sheet_hash[worksheet_c_name].add(@variable_set_counter) + @recursion_prevention_sheet_hash[worksheet_c_name].add(@variable_set_counter) + @variable_set_counter += 1 + @recursion_prevention_counter += 1 + end mapper.reset rescue Exception => e puts "Exception at #{ref} #{ast}"