From fb80f2c320687943bc4fae1503ed15f9dff4ce64 Mon Sep 17 00:00:00 2001 From: Justin Coyne Date: Fri, 11 Jan 2013 23:33:31 -0600 Subject: [PATCH] Encode response body with specified charset header --- lib/restclient/response.rb | 20 +++++++++++++- spec/request_spec.rb | 8 ++---- spec/response_spec.rb | 54 +++++++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/lib/restclient/response.rb b/lib/restclient/response.rb index e934d20b..1297e438 100644 --- a/lib/restclient/response.rb +++ b/lib/restclient/response.rb @@ -13,12 +13,30 @@ def body end def Response.create body, net_http_res, args - result = body || '' + result = encoded_body(body, net_http_res) result.extend Response result.net_http_res = net_http_res result.args = args result end + + private + + def self.encoded_body(body, net_http_res) + result = body || '' + if ((200..207).include?(net_http_res.code.to_i) && content_type = net_http_res.content_type) + charset = if set = net_http_res.type_params['charset'] + set + elsif content_type == 'text/xml' + 'us-ascii' + elsif content_type.split('/').first == 'text' + 'iso-8859-1' + end + result.force_encoding(charset) if charset + end + result + end + end end diff --git a/spec/request_spec.rb b/spec/request_spec.rb index 4505f0c3..dca8af6a 100644 --- a/spec/request_spec.rb +++ b/spec/request_spec.rb @@ -50,9 +50,7 @@ end it "processes a successful result" do - res = mock("result") - res.stub!(:code).and_return("200") - res.stub!(:body).and_return('body') + res = stub("result", :code=>"200", :body=>"body", :content_type=>'text/plain', :type_params=>{}) res.stub!(:[]).with('content-encoding').and_return(nil) @request.process_result(res).body.should == 'body' @request.process_result(res).to_s.should == 'body' @@ -60,9 +58,7 @@ it "doesn't classify successful requests as failed" do 203.upto(207) do |code| - res = mock("result") - res.stub!(:code).and_return(code.to_s) - res.stub!(:body).and_return("") + res = stub("result", :code=>"200", :body=>"", :content_type=>'text/plain', :type_params=>{}) res.stub!(:[]).with('content-encoding').and_return(nil) @request.process_result(res).should be_empty end diff --git a/spec/response_spec.rb b/spec/response_spec.rb index 5048d19d..34776623 100644 --- a/spec/response_spec.rb +++ b/spec/response_spec.rb @@ -1,3 +1,4 @@ +# encoding: ASCII-8BIT # make sure this runs in binary mode require File.join( File.dirname(File.expand_path(__FILE__)), 'base') require 'webmock/rspec' @@ -5,7 +6,7 @@ describe RestClient::Response do before do - @net_http_res = mock('net http response', :to_hash => {"Status" => ["200 OK"]}, :code => 200) + @net_http_res = mock('net http response', :to_hash => {"Status" => ["200 OK"]}, :code => 200, :type_params => {}, :content_type=>'text/html') @request = mock('http request', :user => nil, :password => nil) @response = RestClient::Response.create('abc', @net_http_res, {}) end @@ -27,14 +28,14 @@ describe "cookie processing" do it "should correctly deal with one Set-Cookie header with one cookie inside" do - net_http_res = mock('net http response', :to_hash => {"etag" => ["\"e1ac1a2df945942ef4cac8116366baad\""], "set-cookie" => ["main_page=main_page_no_rewrite; path=/; expires=Tue, 20-Jan-2015 15:03:14 GMT"]}) + net_http_res = mock('net http response', :to_hash => {"etag" => ["\"e1ac1a2df945942ef4cac8116366baad\""], "set-cookie" => ["main_page=main_page_no_rewrite; path=/; expires=Tue, 20-Jan-2015 15:03:14 GMT"]}, :code=>200, :type_params=>{}, :content_type=>'text/html') response = RestClient::Response.create('abc', net_http_res, {}) response.headers[:set_cookie].should == ["main_page=main_page_no_rewrite; path=/; expires=Tue, 20-Jan-2015 15:03:14 GMT"] response.cookies.should == { "main_page" => "main_page_no_rewrite" } end it "should correctly deal with multiple cookies [multiple Set-Cookie headers]" do - net_http_res = mock('net http response', :to_hash => {"etag" => ["\"e1ac1a2df945942ef4cac8116366baad\""], "set-cookie" => ["main_page=main_page_no_rewrite; path=/; expires=Tue, 20-Jan-2015 15:03:14 GMT", "remember_me=; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT", "user=somebody; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT"]}) + net_http_res = mock('net http response', :to_hash => {"etag" => ["\"e1ac1a2df945942ef4cac8116366baad\""], "set-cookie" => ["main_page=main_page_no_rewrite; path=/; expires=Tue, 20-Jan-2015 15:03:14 GMT", "remember_me=; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT", "user=somebody; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT"]}, :code=>200, :type_params=>{}, :content_type=>'text/html') response = RestClient::Response.create('abc', net_http_res, {}) response.headers[:set_cookie].should == ["main_page=main_page_no_rewrite; path=/; expires=Tue, 20-Jan-2015 15:03:14 GMT", "remember_me=; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT", "user=somebody; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT"] response.cookies.should == { @@ -45,7 +46,7 @@ end it "should correctly deal with multiple cookies [one Set-Cookie header with multiple cookies]" do - net_http_res = mock('net http response', :to_hash => {"etag" => ["\"e1ac1a2df945942ef4cac8116366baad\""], "set-cookie" => ["main_page=main_page_no_rewrite; path=/; expires=Tue, 20-Jan-2015 15:03:14 GMT, remember_me=; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT, user=somebody; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT"]}) + net_http_res = mock('net http response', :to_hash => {"etag" => ["\"e1ac1a2df945942ef4cac8116366baad\""], "set-cookie" => ["main_page=main_page_no_rewrite; path=/; expires=Tue, 20-Jan-2015 15:03:14 GMT, remember_me=; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT, user=somebody; path=/; expires=Thu, 01-Jan-1970 00:00:00 GMT"]}, :code=>200, :type_params=>{}, :content_type=>'text/html') response = RestClient::Response.create('abc', net_http_res, {}) response.cookies.should == { "main_page" => "main_page_no_rewrite", @@ -58,7 +59,7 @@ describe "exceptions processing" do it "should return itself for normal codes" do (200..206).each do |code| - net_http_res = mock('net http response', :code => '200') + net_http_res = mock('net http response', :code => '200', :type_params => {}, :content_type=>'text/html') response = RestClient::Response.create('abc', net_http_res, {}) response.return! @request end @@ -76,6 +77,41 @@ end + describe "charset" do + it "does nothing when Content-Type is missing" do + stub_request(:get, 'http://some/resource').to_return(:body => 'Foo') + RestClient::Request.execute(:url => 'http://some/resource', :method => :get).body.encoding.should == Encoding.find('ASCII-8BIT') + end + it "does nothing when charset is missing and it's not a text type" do + stub_request(:get, 'http://some/resource').to_return(:body => 'Foo', :headers => {'Content-Type' => 'image/png'}) + RestClient::Request.execute(:url => 'http://some/resource', :method => :get).body.encoding.should == Encoding.find('ASCII-8BIT') + end + it "defaults to us-ascii when content-type is text/xml and charset is missing" do + # @see http://tools.ietf.org/html/rfc3023#page-19 + # > Conformant with [RFC2046], if a text/xml entity is received with + # > the charset parameter omitted, MIME processors and XML processors + # > MUST use the default charset value of "us-ascii"[ASCII]. In cases + # > where the XML MIME entity is transmitted via HTTP, the default + # > charset value is still "us-ascii". (Note: There is an + # > inconsistency between this specification and HTTP/1.1, which uses + # > ISO-8859-1[ISO8859] as the default for a historical reason. Since + # > XML is a new format, a new default should be chosen for better + # > I18N. US-ASCII was chosen, since it is the intersection of UTF-8 + # > and ISO-8859-1 and since it is already used by MIME.) + stub_request(:get, 'http://some/resource').to_return(:body => 'Foo', :headers => {'Content-Type' => 'text/xml'}) + RestClient::Request.execute(:url => 'http://some/resource', :method => :get).body.encoding.should == Encoding.find('us-ascii') + end + it "defaults to iso-8859-1 when content-type is a subtype of text and charset is missing" do + stub_request(:get, 'http://some/resource').to_return(:body => 'Foo', :headers => {'Content-Type' => 'text/plain'}) + RestClient::Request.execute(:url => 'http://some/resource', :method => :get).body.encoding.should == Encoding.find('iso-8859-1') + end + it "forces the encoding to the charset header" do + stub_request(:get, 'http://some/resource').to_return(:body => 'Foo', :headers => {'Content-Type' => 'text/html; charset=UTF-8'}) + RestClient::Request.execute(:url => 'http://some/resource', :method => :get).body.encoding.should == Encoding.find('utf-8') + stub_request(:get, 'http://some/resource').to_return(:body => 'Foo', :headers => {'Content-Type' => 'text/html; charset=iso-8859-1'}) + RestClient::Request.execute(:url => 'http://some/resource', :method => :get).body.encoding.should == Encoding.find('iso-8859-1') + end + end describe "redirection" do it "follows a redirection when the request is a get" do @@ -122,25 +158,25 @@ it "follows a redirection when the request is a post and result is a 303" do stub_request(:put, 'http://some/resource').to_return(:body => '', :status => 303, :headers => {'Location' => 'http://new/resource'}) - stub_request(:get, 'http://new/resource').to_return(:body => 'Foo') + stub_request(:get, 'http://new/resource').to_return(:body => 'Foo', :type_params => {}) RestClient::Request.execute(:url => 'http://some/resource', :method => :put).body.should == 'Foo' end it "follows a redirection when the request is a head" do stub_request(:head, 'http://some/resource').to_return(:body => '', :status => 301, :headers => {'Location' => 'http://new/resource'}) - stub_request(:head, 'http://new/resource').to_return(:body => 'Foo') + stub_request(:head, 'http://new/resource').to_return(:body => 'Foo', :type_params => {}) RestClient::Request.execute(:url => 'http://some/resource', :method => :head).body.should == 'Foo' end it "handles redirects with relative paths" do stub_request(:get, 'http://some/resource').to_return(:body => '', :status => 301, :headers => {'Location' => 'index'}) - stub_request(:get, 'http://some/index').to_return(:body => 'Foo') + stub_request(:get, 'http://some/index').to_return(:body => 'Foo', :type_params => {}) RestClient::Request.execute(:url => 'http://some/resource', :method => :get).body.should == 'Foo' end it "handles redirects with relative path and query string" do stub_request(:get, 'http://some/resource').to_return(:body => '', :status => 301, :headers => {'Location' => 'index?q=1'}) - stub_request(:get, 'http://some/index?q=1').to_return(:body => 'Foo') + stub_request(:get, 'http://some/index?q=1').to_return(:body => 'Foo', :type_params => {}) RestClient::Request.execute(:url => 'http://some/resource', :method => :get).body.should == 'Foo' end