Merge pull request #1778 from nicklewis/GH-1759-non-utf-8-task-output

(GH-1759) Handle non-UTF-8 results

Author: lucywyman

lib/bolt/outputter/human.rb

@@ -107,13 +107,14 @@ def print_result(result)
 
           # Use special handling if the result looks like a command or script result
           if result.generic_value.keys == %w[stdout stderr exit_code]
-            unless result['stdout'].strip.empty?
+            safe_value = result.safe_value
+            unless safe_value['stdout'].strip.empty?
               @stream.puts(indent(2, "STDOUT:"))
-              @stream.puts(indent(4, result['stdout']))
+              @stream.puts(indent(4, safe_value['stdout']))
             end
-            unless result['stderr'].strip.empty?
+            unless safe_value['stderr'].strip.empty?
               @stream.puts(indent(2, "STDERR:"))
-              @stream.puts(indent(4, result['stderr']))
+              @stream.puts(indent(4, safe_value['stderr']))
             end
           elsif result.generic_value.any?
             @stream.puts(indent(2, ::JSON.pretty_generate(result.generic_value)))

lib/bolt/outputter/json.rb

@@ -28,7 +28,7 @@ def handle_event(event)
 
       def print_result(result)
         @stream.puts ',' if @preceding_item
-        @stream.puts result.status_hash.to_json
+        @stream.puts result.to_json
         @preceding_item = true
       end
 

lib/bolt/rerun.rb

@@ -45,7 +45,7 @@ def update(result_set)
         end
 
         if result_set.is_a?(Bolt::ResultSet)
-          data = result_set.map { |res| res.status_hash.select { |k, _| %i[target status].include? k } }
+          data = result_set.map { |res| { target: res.target.name, status: res.status } }
           FileUtils.mkdir_p(File.dirname(@path))
           File.write(@path, data.to_json)
         elsif File.exist?(@path)

lib/bolt/result.rb

@@ -40,18 +40,23 @@ def self.for_command(target, stdout, stderr, exit_code, action, command)
     end
 
     def self.for_task(target, stdout, stderr, exit_code, task)
-      begin
-        value = JSON.parse(stdout)
-        unless value.is_a? Hash
-          value = nil
-        end
-      rescue JSON::ParserError
-        value = nil
-      end
-      value ||= { '_output' => stdout }
+      stdout.force_encoding('utf-8') unless stdout.encoding == Encoding::UTF_8
+      value = if stdout.valid_encoding?
+                parse_hash(stdout) || { '_output' => stdout }
+              else
+                { '_error' => { 'kind' => 'puppetlabs.tasks/task-error',
+                                'issue_code' => 'TASK_ERROR',
+                                'msg' => 'The task result contained invalid UTF-8 on stdout',
+                                'details' => {} } }
+              end
+
       if exit_code != 0 && value['_error'].nil?
         msg = if stdout.empty?
-                "The task failed with exit code #{exit_code}:\n#{stderr}"
+                if stderr.empty?
+                  "The task failed with exit code #{exit_code} and no output"
+                else
+                  "The task failed with exit code #{exit_code} and no stdout, but stderr contained:\n#{stderr}"
+                end
               else
                 "The task failed with exit code #{exit_code}"
               end
@@ -63,6 +68,13 @@ def self.for_task(target, stdout, stderr, exit_code, task)
       new(target, value: value, action: 'task', object: task)
     end
 
+    def self.parse_hash(string)
+      value = JSON.parse(string)
+      value if value.is_a? Hash
+    rescue JSON::ParserError
+      nil
+    end
+
     def self.for_upload(target, source, destination)
       new(target, message: "Uploaded '#{source}' to '#{target.host}:#{destination}'", action: 'upload', object: source)
     end
@@ -110,18 +122,8 @@ def message?
       message && !message.strip.empty?
     end
 
-    def status_hash
-      {
-        target: @target.name,
-        action: action,
-        object: object,
-        status: status,
-        value: @value
-      }
-    end
-
     def generic_value
-      value.reject { |k, _| %w[_error _output].include? k }
+      safe_value.reject { |k, _| %w[_error _output].include? k }
     end
 
     def eql?(other)
@@ -139,15 +141,36 @@ def ==(other)
     end
 
     def to_json(opts = nil)
-      status_hash.to_json(opts)
+      to_data.to_json(opts)
     end
 
     def to_s
       to_json
     end
 
+    # This is the value with all non-UTF-8 characters removed, suitable for
+    # printing or converting to JSON. It *should* only be possible to have
+    # non-UTF-8 characters in stdout/stderr keys as they are not allowed from
+    # tasks but we scrub the whole thing just in case.
+    def safe_value
+      Bolt::Util.walk_vals(value) do |val|
+        if val.is_a?(String)
+          # Replace invalid bytes with hex codes, ie. \xDE\xAD\xBE\xEF
+          val.scrub { |c| c.bytes.map { |b| "\\x" + b.to_s(16).upcase }.join }
+        else
+          val
+        end
+      end
+    end
+
     def to_data
-      Bolt::Util.walk_keys(status_hash, &:to_s)
+      {
+        "target" => @target.name,
+        "action" => action,
+        "object" => object,
+        "status" => status,
+        "value" => safe_value
+      }
     end
 
     def status

lib/bolt/result_set.rb

@@ -99,17 +99,14 @@ def eql?(other)
       self.class == other.class && @results == other.results
     end
 
-    def to_a
-      @results.map(&:status_hash)
-    end
-
     def to_json(opts = nil)
-      @results.map(&:status_hash).to_json(opts)
+      to_data.to_json(opts)
     end
 
     def to_data
       @results.map(&:to_data)
     end
+    alias to_a to_data
 
     def to_s
       to_json

lib/bolt/shell/powershell.rb

@@ -267,10 +267,18 @@ def execute(command)
 
         result = Bolt::Node::Output.new
         inp.close
-        out.binmode
-        err.binmode
-        stdout = Thread.new { result.stdout << out.read }
-        stderr = Thread.new { result.stderr << err.read }
+        stdout = Thread.new do
+          # Set to binmode to preserve \r\n line endings, but save and restore
+          # the proper encoding so the string isn't later misinterpreted
+          encoding = out.external_encoding
+          out.binmode
+          result.stdout << out.read.force_encoding(encoding)
+        end
+        stderr = Thread.new do
+          encoding = err.external_encoding
+          err.binmode
+          result.stderr << err.read.force_encoding(encoding)
+        end
 
         stdout.join
         stderr.join

lib/bolt/transport/winrm/connection.rb

@@ -108,8 +108,8 @@ def execute(command)
           # it will fail if the shell attempts to provide stdin
           inp.close
 
-          out_rd, out_wr = IO.pipe
-          err_rd, err_wr = IO.pipe
+          out_rd, out_wr = IO.pipe('UTF-8')
+          err_rd, err_wr = IO.pipe('UTF-8')
           th = Thread.new do
             result = @session.run(command)
             out_wr << result.stdout

lib/bolt_server/transport_app.rb

@@ -57,8 +57,8 @@ def initialize(config)
     end
 
     def scrub_stack_trace(result)
-      if result.dig(:value, '_error', 'details', 'stack_trace')
-        result[:value]['_error']['details'].reject! { |k| k == 'stack_trace' }
+      if result.dig('value', '_error', 'details', 'stack_trace')
+        result['value']['_error']['details'].reject! { |k| k == 'stack_trace' }
       end
       result
     end
@@ -87,14 +87,14 @@ def validate_schema(schema, body)
     # If the `result_set` contains only one item, it will be returned
     # as a single result object. Set `aggregate` to treat it as a set
     # of results with length 1 instead.
-    def result_set_to_status_hash(result_set, aggregate: false)
+    def result_set_to_data(result_set, aggregate: false)
       scrubbed_results = result_set.map do |result|
-        scrub_stack_trace(result.status_hash)
+        scrub_stack_trace(result.to_data)
       end
 
       if aggregate || scrubbed_results.length > 1
         # For actions that act on multiple targets, construct a status hash for the aggregate result
-        all_succeeded = scrubbed_results.all? { |r| r[:status] == 'success' }
+        all_succeeded = scrubbed_results.all? { |r| r['status'] == 'success' }
         {
           status: all_succeeded ? 'success' : 'failure',
           result: scrubbed_results
@@ -297,7 +297,7 @@ def make_ssh_target(target_hash)
       return [400, error.to_json] unless error.nil?
 
       aggregate = body['target'].nil?
-      [200, result_set_to_status_hash(result_set, aggregate: aggregate).to_json]
+      [200, result_set_to_data(result_set, aggregate: aggregate).to_json]
     end
 
     def make_winrm_target(target_hash)
@@ -337,7 +337,7 @@ def make_winrm_target(target_hash)
       return [400, error.to_json] if error
 
       aggregate = body['target'].nil?
-      [200, result_set_to_status_hash(result_set, aggregate: aggregate).to_json]
+      [200, result_set_to_data(result_set, aggregate: aggregate).to_json]
     end
 
     # Fetches the metadata for a single plan

spec/bolt/rerun_spec.rb

@@ -32,8 +32,8 @@
 
   let(:failure_array) do
     result_set.map do |r|
-      r = r.status_hash
-      { 'target' => r[:target], 'status' => r[:status] }
+      r = r.to_data
+      { 'target' => r['target'], 'status' => r['status'] }
     end
   end
 

spec/bolt/result_spec.rb

@@ -109,5 +109,25 @@
       expect(result.value).to eq(obj)
       expect(result.error_hash).to eq(obj['_error'])
     end
+
+    it 'uses the unparsed value of stdout if it is not valid JSON' do
+      stdout = 'just some string'
+      result = Bolt::Result.for_task(target, stdout, '', 0, 'atask')
+      expect(result.value).to eq('_output' => 'just some string')
+    end
+
+    it 'generates an error for binary data' do
+      stdout = "\xFC].\xF9\xA8\x85f\xDF{\x11d\xD5\x8E\xC6\xA6"
+      result = Bolt::Result.for_task(target, stdout, '', 0, 'atask')
+      expect(result.value.keys).to eq(['_error'])
+      expect(result.error_hash['msg']).to match(/The task result contained invalid UTF-8/)
+    end
+
+    it 'generates an error for non-UTF-8 output' do
+      stdout = "☃".encode('utf-32')
+      result = Bolt::Result.for_task(target, stdout, '', 0, 'atask')
+      expect(result.value.keys).to eq(['_error'])
+      expect(result.error_hash['msg']).to match(/The task result contained invalid UTF-8/)
+    end
   end
 end