From e9deb2bfa8c66bd49bdc9538f530428faac08fab Mon Sep 17 00:00:00 2001 From: Nadir Cohen Date: Tue, 9 Nov 2021 06:48:22 +0200 Subject: [PATCH 01/72] updates Linux Firefox user-agent to rev94 --- lib/mechanize.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mechanize.rb b/lib/mechanize.rb index 10cae760..457d9eec 100644 --- a/lib/mechanize.rb +++ b/lib/mechanize.rb @@ -115,7 +115,7 @@ class Error < RuntimeError AGENT_ALIASES = { 'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/sparklemotion/mechanize/)", - 'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0', + 'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0', 'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)', 'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624', 'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0', From 4a0dfe57123ff78066d72fe1106db3abc547cb65 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Thu, 11 Nov 2021 15:32:36 -0500 Subject: [PATCH 02/72] version bump to v2.8.3 --- CHANGELOG.md | 7 +++++++ lib/mechanize/version.rb | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1de920af..61fd9174 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Mechanize CHANGELOG +## 2.8.3 / 2021-11-11 + +### Update + +* Update the "Linux Firefox" user agent string to rev94 (#587) Thank you, @ncs1! + + ## 2.8.2 / 2021-08-06 ### Dependencies diff --git a/lib/mechanize/version.rb b/lib/mechanize/version.rb index 8dbba67e..83961dc3 100644 --- a/lib/mechanize/version.rb +++ b/lib/mechanize/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true class Mechanize - VERSION = "2.8.2" + VERSION = "2.8.3" end From 1c099a6e44428ac343fdb3513e240072b10521a0 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 17 Jan 2022 13:04:07 -0500 Subject: [PATCH 03/72] use safe_load when using Psych >= 3.1 see related https://github.com/sparklemotion/http-cookie/pull/34 --- lib/mechanize/cookie_jar.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/mechanize/cookie_jar.rb b/lib/mechanize/cookie_jar.rb index 64a325b9..82bd942b 100644 --- a/lib/mechanize/cookie_jar.rb +++ b/lib/mechanize/cookie_jar.rb @@ -149,7 +149,7 @@ def load(input, *options) return super(input, opthash) if opthash[:format] != :yaml begin - data = YAML.load(input) # rubocop:disable Security/YAMLLoad + data = load_yaml(input) rescue ArgumentError @logger.warn "unloadable YAML cookie data discarded" if @logger return self @@ -174,6 +174,18 @@ def load(input, *options) return self end end + + private + + if YAML.name == "Psych" && Gem::Requirement.new(">= 3.1").satisfied_by?(Gem::Version.new(Psych::VERSION)) + def load_yaml(yaml) + YAML.safe_load(yaml, aliases: true, permitted_classes: ["Mechanize::Cookie", "Time"]) + end + else + def load_yaml(yaml) + YAML.load(yaml) # rubocop:disable Security/YAMLLoad + end + end end class ::HTTP::CookieJar From 8302ec557ec2a364d764fc948c0792737ade5d83 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 17 Jan 2022 12:32:08 -0500 Subject: [PATCH 04/72] ci: update to cover Ruby 3.1 --- .github/workflows/ci-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 2f7a9a8b..841ac988 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.0" + ruby-version: "3.1" bundler-cache: true - run: bundle exec rake rubocop @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ["2.5", "2.6", "2.7", "3.0", "jruby", "truffleruby-head"] + ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "head", "jruby", "truffleruby-head"] runs-on: ubuntu-latest steps: @@ -48,6 +48,6 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.0" + ruby-version: "3.1" bundler-cache: true - run: bundle exec rake test From 70ebc34112973b6c9e1b300c963a6fb4c3fedd19 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 17 Jan 2022 13:20:00 -0500 Subject: [PATCH 05/72] version bump to v2.8.4 --- CHANGELOG.md | 7 +++++++ lib/mechanize/version.rb | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61fd9174..4d55d5de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Mechanize CHANGELOG +## 2.8.4 / 2022-01-17 + +### Fix + +* `Mechanize::CookieJar#load` calls `Psych.safe_load` when using Psych >= 3.1 + + ## 2.8.3 / 2021-11-11 ### Update diff --git a/lib/mechanize/version.rb b/lib/mechanize/version.rb index 83961dc3..3b862e21 100644 --- a/lib/mechanize/version.rb +++ b/lib/mechanize/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true class Mechanize - VERSION = "2.8.3" + VERSION = "2.8.4" end From 907c778001625cb9daa686d5019c939cb416e45b Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 5 Jun 2022 21:25:48 -0400 Subject: [PATCH 06/72] fix: clear credentials when redirecting to a different port Note that in this case we treat cookies differently from credentials per RFC 6265 section 8.5: https://datatracker.ietf.org/doc/html/rfc6265#section-8.5 > Cookies do not provide isolation by port. If a cookie is readable > by a service running on one port, the cookie is also readable by a > service running on another port of the same server. If a cookie is > writable by a service on one port, the cookie is also writable by a > service running on another port of the same server. For this > reason, servers SHOULD NOT both run mutually distrusting services on > different ports of the same host and use cookies to store security- > sensitive information. --- lib/mechanize/http/agent.rb | 13 +++++++++---- test/test_mechanize_http_agent.rb | 25 +++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/mechanize/http/agent.rb b/lib/mechanize/http/agent.rb index 5b1b2266..9e433a3f 100644 --- a/lib/mechanize/http/agent.rb +++ b/lib/mechanize/http/agent.rb @@ -9,7 +9,8 @@ class Mechanize::HTTP::Agent - CREDENTIAL_HEADERS = ['Authorization', 'Cookie'] + CREDENTIAL_HEADERS = ['Authorization'] + COOKIE_HEADERS = ['Cookie'] POST_HEADERS = ['Content-Length', 'Content-MD5', 'Content-Type'] # :section: Headers @@ -998,10 +999,14 @@ def response_redirect(response, method, page, redirects, headers, end # Make sure we clear credential headers if being redirected to another site - if new_uri.host != page.uri.host - CREDENTIAL_HEADERS.each do |ch| - headers.delete_if { |h| h.casecmp?(ch) } + if new_uri.host == page.uri.host + if new_uri.port != page.uri.port + # https://datatracker.ietf.org/doc/html/rfc6265#section-8.5 + # cookies are OK to be shared across ports on the same host + CREDENTIAL_HEADERS.each { |ch| headers.delete_if { |h| h.casecmp?(ch) } } end + else + (COOKIE_HEADERS + CREDENTIAL_HEADERS).each { |ch| headers.delete_if { |h| h.casecmp?(ch) } } end fetch new_uri, redirect_method, headers, [], referer, redirects + 1 diff --git a/test/test_mechanize_http_agent.rb b/test/test_mechanize_http_agent.rb index bc8c0580..8a2681f5 100644 --- a/test/test_mechanize_http_agent.rb +++ b/test/test_mechanize_http_agent.rb @@ -1569,7 +1569,7 @@ def test_response_redirect_to_cross_site_with_credential refute_includes(headers.keys, "AUTHORIZATION") refute_includes(headers.keys, "cookie") - assert_match 'range|bytes=0-9999', page.body + assert_match("range|bytes=0-9999", page.body) refute_match("authorization|Basic xxx", page.body) refute_match("cookie|name=value", page.body) end @@ -1590,11 +1590,32 @@ def test_response_redirect_to_same_site_with_credential assert_includes(headers.keys, "AUTHORIZATION") assert_includes(headers.keys, "cookie") - assert_match 'range|bytes=0-9999', page.body + assert_match("range|bytes=0-9999", page.body) assert_match("authorization|Basic xxx", page.body) assert_match("cookie|name=value", page.body) end + def test_response_redirect_to_same_site_diff_port_with_credential + @agent.redirect_ok = true + + headers = { + 'Range' => 'bytes=0-9999', + 'AUTHORIZATION' => 'Basic xxx', + 'cookie' => 'name=value', + } + + page = html_page '' + page = @agent.response_redirect({ 'Location' => 'http://example:81/http_headers' }, :get, + page, 0, headers) + + refute_includes(headers.keys, "AUTHORIZATION") + assert_includes(headers.keys, "cookie") + + assert_match("range|bytes=0-9999", page.body) + refute_match("authorization|Basic xxx", page.body) + assert_match("cookie|name=value", page.body) + end + def test_response_redirect_not_ok @agent.redirect_ok = false From c1091fd4e3bce84f018b37a212dda6f670060154 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Thu, 9 Jun 2022 13:30:59 -0400 Subject: [PATCH 07/72] version bump to v2.8.5 --- CHANGELOG.md | 7 +++++++ lib/mechanize/version.rb | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d55d5de..7588f18b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Mechanize CHANGELOG +## 2.8.5 / 2022-06-09 + +### Security + +Fixes low-severity CVE-2022-31033, "Authorization header leak on port redirect." See [GHSA-64qm-hrgp-pgr9](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-64qm-hrgp-pgr9) for more details. + + ## 2.8.4 / 2022-01-17 ### Fix diff --git a/lib/mechanize/version.rb b/lib/mechanize/version.rb index 3b862e21..60a7e6e9 100644 --- a/lib/mechanize/version.rb +++ b/lib/mechanize/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true class Mechanize - VERSION = "2.8.4" + VERSION = "2.8.5" end From ab33104ca11cd2e6bc416f7c3dd3a453d51d39bc Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 7 Feb 2023 08:52:31 -0500 Subject: [PATCH 08/72] ci: include ruby 3.2 --- .github/workflows/ci-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 841ac988..7315cc7c 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.1" + ruby-version: "3.2" bundler-cache: true - run: bundle exec rake rubocop @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "head", "jruby", "truffleruby-head"] + ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby", "truffleruby-head"] runs-on: ubuntu-latest steps: @@ -48,6 +48,6 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.1" + ruby-version: "3.2" bundler-cache: true - run: bundle exec rake test From 1cf35a5f71fb7c42fa26fc35b57bea1cd9fcb795 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 7 Feb 2023 08:56:00 -0500 Subject: [PATCH 09/72] ci: run CI in a cron job and cancel concurrent jobs --- .github/workflows/ci-test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 7315cc7c..2dcd37c7 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -1,5 +1,9 @@ name: "ci" +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + cancel-in-progress: true + on: push: branches: @@ -8,6 +12,8 @@ on: types: [opened, synchronize] branches: - main + schedule: + - cron: "0 8 * * 5" # At 08:00 on Friday # https://crontab.guru/#0_8_*_*_5 jobs: rubocop: From 802ce5c563bc91397fec11167e3aae7ea62c0b17 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 7 Feb 2023 09:06:22 -0500 Subject: [PATCH 10/72] test: skip NTLM test if openssl doesn't support MD4 This seems to be common with Ruby 3.1+ (OpenSSL 3.0+). --- test/test_mechanize_http_agent.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_mechanize_http_agent.rb b/test/test_mechanize_http_agent.rb index 8a2681f5..a640dab8 100644 --- a/test/test_mechanize_http_agent.rb +++ b/test/test_mechanize_http_agent.rb @@ -823,7 +823,11 @@ def test_response_authenticate_ntlm @res.instance_variable_set(:@header, 'www-authenticate' => ['Negotiate, NTLM']) - page = @agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil + begin + page = @agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil + rescue OpenSSL::Digest::DigestError + skip "It looks like OpenSSL is not configured to support MD4" + end assert_equal 'ok', page.body # lame test end From 23f8a161ff42f779a7ace43978f6ea2221d64272 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 7 Feb 2023 09:16:53 -0500 Subject: [PATCH 11/72] test: simplify jruby skips --- test/test_mechanize_http_agent.rb | 21 +++++++++++---------- test/test_mechanize_page_link.rb | 15 ++++++--------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/test/test_mechanize_http_agent.rb b/test/test_mechanize_http_agent.rb index a640dab8..f6beb06d 100644 --- a/test/test_mechanize_http_agent.rb +++ b/test/test_mechanize_http_agent.rb @@ -27,13 +27,10 @@ def auth_realm uri, scheme, type realm end - def jruby_zlib? + def skip_if_jruby_zlib if RUBY_ENGINE == 'jruby' meth = caller[0][/`(\w+)/, 1] - warn "#{meth}: skipped because how Zlib handles error is different in JRuby" - true - else - false + skip "#{meth}: skipped because how Zlib handles error is different in JRuby" end end @@ -935,7 +932,7 @@ def test_response_content_encoding_gzip_corrupt body_io = StringIO.new \ "\037\213\b\0002\002\225M\000\003+H,*\001" - skip if jruby_zlib? + skip_if_jruby_zlib e = assert_raises Mechanize::Error do @agent.response_content_encoding @res, body_io @@ -969,7 +966,8 @@ def test_response_content_encoding_gzip_checksum_corrupt_crc assert_match %r%invalid compressed data -- crc error%, log.string rescue IOError - raise unless jruby_zlib? + skip_if_jruby_zlib + raise end def test_response_content_encoding_gzip_checksum_corrupt_length @@ -987,7 +985,8 @@ def test_response_content_encoding_gzip_checksum_corrupt_length assert_match %r%invalid compressed data -- length error%, log.string rescue IOError - raise unless jruby_zlib? + skip_if_jruby_zlib + raise end def test_response_content_encoding_gzip_checksum_truncated @@ -1005,7 +1004,8 @@ def test_response_content_encoding_gzip_checksum_truncated assert_match %r%unable to gunzip response: footer is not found%, log.string rescue IOError - raise unless jruby_zlib? + skip_if_jruby_zlib + raise end def test_response_content_encoding_gzip_empty @@ -1046,7 +1046,8 @@ def test_response_content_encoding_gzip_no_footer assert body_io.closed? rescue IOError - raise unless jruby_zlib? + skip_if_jruby_zlib + raise end def test_response_content_encoding_none diff --git a/test/test_mechanize_page_link.rb b/test/test_mechanize_page_link.rb index 32c63bb9..5e77dcc0 100644 --- a/test/test_mechanize_page_link.rb +++ b/test/test_mechanize_page_link.rb @@ -51,13 +51,10 @@ def util_page body = @body, res = @res Mechanize::Page.new @uri, res, body && body.force_encoding(Encoding::BINARY), 200, @mech end - def nkf_dependency? - if RUBY_ENGINE == 'ruby' - false - else + def skip_if_nkf_dependency + if RUBY_ENGINE == 'jruby' meth = caller[0][/`(\w+)/, 1] - warn "#{meth}: skipped because this feature currently depends on NKF" - true + skip "#{meth}: skipped because this feature currently depends on NKF" end end @@ -112,7 +109,7 @@ def test_encoding_charset_after_title end def test_encoding_charset_after_title_bad - return if nkf_dependency? + skip_if_nkf_dependency page = util_page UTF8 @@ -122,7 +119,7 @@ def test_encoding_charset_after_title_bad end def test_encoding_charset_after_title_double_bad - return if nkf_dependency? + skip_if_nkf_dependency page = util_page SJIS_BAD_AFTER_TITLE @@ -132,7 +129,7 @@ def test_encoding_charset_after_title_double_bad end def test_encoding_charset_bad - return if nkf_dependency? + skip_if_nkf_dependency page = util_page "#{UTF8_TITLE}" page.encodings.replace %w[ From 848084afbcae4b3b8a7378d24a919f971e0a2d4d Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 7 Feb 2023 09:22:06 -0500 Subject: [PATCH 12/72] ci: pin jruby to 9.3 for now until https://github.com/jruby/jruby/issues/7481 is in a release --- .github/workflows/ci-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 2dcd37c7..1e9a00df 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -31,7 +31,9 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby", "truffleruby-head"] + # TODO update jruby once 9.4.1.0 is out + # 9.4.0.0 is affected by https://github.com/jruby/jruby/issues/7481 + ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby-9.3", "truffleruby-head"] runs-on: ubuntu-latest steps: From 84ca53e01b5a788fde96c6214fc5ac0683a8d699 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 7 Feb 2023 09:35:46 -0500 Subject: [PATCH 13/72] example: fix wikipedia script Closes #604 --- examples/wikipedia_links_to_philosophy.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/wikipedia_links_to_philosophy.rb b/examples/wikipedia_links_to_philosophy.rb index 0b744766..1907d222 100644 --- a/examples/wikipedia_links_to_philosophy.rb +++ b/examples/wikipedia_links_to_philosophy.rb @@ -58,10 +58,10 @@ def finished? # the article. def follow_first_link - puts @title + puts "#{@title} (#{@page.uri})" # > p > a rejects italics - links = @page.root.css('.mw-content-ltr > p > a[href^="/wiki/"]') + links = @page.root.css('.mw-content-ltr p > a[href^="/wiki/"]') # reject disambiguation and special pages, images and files links = links.reject do |link_node| @@ -74,10 +74,9 @@ def follow_first_link link = links.first - unless link then - # disambiguation page? try the first item in the list - link = - @page.root.css('.mw-content-ltr > ul > li > a[href^="/wiki/"]').first + if link.nil? + puts "Could not parse #{@page.uri}" + exit 1 end # convert a Nokogiri HTML element back to a mechanize link From 427f9be1bb5c31158fe2c135a5950fdcbbdbd9c0 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 5 Mar 2023 08:55:54 -0500 Subject: [PATCH 14/72] ci: update to jruby 9.4 https://github.com/jruby/jruby/issues/7481 is fixed in 9.4.1.0 --- .github/workflows/ci-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 1e9a00df..6fb7141a 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -31,9 +31,7 @@ jobs: strategy: fail-fast: false matrix: - # TODO update jruby once 9.4.1.0 is out - # 9.4.0.0 is affected by https://github.com/jruby/jruby/issues/7481 - ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby-9.3", "truffleruby-head"] + ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby-9.4", "truffleruby-head"] runs-on: ubuntu-latest steps: From cb9da398bef0ca476e988857f05aeb70863ccc57 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sat, 4 Mar 2023 21:33:30 -0500 Subject: [PATCH 15/72] ci: use modern bundler ruby 2.5 just started failing --- .github/workflows/ci-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 6fb7141a..135e3d29 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -40,6 +40,7 @@ jobs: with: ruby-version: ${{matrix.ruby-version}} bundler-cache: true + bundler: 2.3.26 # https://github.com/rubygems/rubygems/issues/6435 - run: bundle exec rake test test-platform: @@ -56,4 +57,5 @@ jobs: with: ruby-version: "3.2" bundler-cache: true + bundler: latest - run: bundle exec rake test From 3584eeba1c7383eb70ace1398ef1a35d4f9a5aa3 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 14 Mar 2023 10:54:12 -0400 Subject: [PATCH 16/72] drop Ruby 2.5 support because the work to get CI to build properly is not worth it, IMHO context: ruby 2.5 CI started failing again, https://github.com/sparklemotion/mechanize/actions/runs/4382526288/jobs/7741821736#step:3:346 --- .github/workflows/ci-test.yml | 2 +- CHANGELOG.md | 7 +++++++ README.md | 2 +- mechanize.gemspec | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 135e3d29..42ff238c 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby-9.4", "truffleruby-head"] + ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby-9.4", "truffleruby-head"] runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 7588f18b..9f58e9c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Mechanize CHANGELOG +## next / unreleased + +### Requirements + +* Mechanize now requires Ruby 2.6 or newer. + + ## 2.8.5 / 2022-06-09 ### Security diff --git a/README.md b/README.md index 39539fcd..869631fb 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The Mechanize library is used for automating interaction with websites. Mechaniz ## Dependencies -* Ruby >= 2.5 +* Ruby >= 2.6 * Gems: * `addressable` * `domain_name` diff --git a/mechanize.gemspec b/mechanize.gemspec index 53d28dc6..80cf3d92 100644 --- a/mechanize.gemspec +++ b/mechanize.gemspec @@ -51,7 +51,7 @@ Gem::Specification.new do |spec| spec.extra_rdoc_files += Dir['*.rdoc', '*.md'] spec.rdoc_options = ["--main", "README.md"] - spec.required_ruby_version = ">= 2.5.0" + spec.required_ruby_version = ">= 2.6.0" spec.add_runtime_dependency("addressable", "~> 2.8") spec.add_runtime_dependency("domain_name", ">= 0.5.20190701", "~> 0.5") From 96a28e79c39d5f74d33cd0f7fc85dec46378114e Mon Sep 17 00:00:00 2001 From: Ruby Date: Thu, 6 Apr 2023 16:13:51 -0400 Subject: [PATCH 17/72] fix: stop force_encoding the page body which allows us to handle frozen strings --- lib/mechanize/page.rb | 4 ---- test/test_mechanize_page.rb | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/mechanize/page.rb b/lib/mechanize/page.rb index 65df949f..c50d62eb 100644 --- a/lib/mechanize/page.rb +++ b/lib/mechanize/page.rb @@ -41,10 +41,6 @@ def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil) @encodings.concat self.class.response_header_charset(response) if body - # Force the encoding to be 8BIT so we can perform regular expressions. - # We'll set it to the detected encoding later - body.force_encoding(Encoding::ASCII_8BIT) - @encodings.concat self.class.meta_charset body meta_content_type = self.class.meta_content_type body diff --git a/test/test_mechanize_page.rb b/test/test_mechanize_page.rb index 4aeb1035..014cb50b 100644 --- a/test/test_mechanize_page.rb +++ b/test/test_mechanize_page.rb @@ -276,5 +276,19 @@ def test_multiple_titles assert_equal page.title, "HTML>TITLE" end + def test_frozen_string_body + html = (<<~HTML).freeze + + + Page Title + + +

Hello World

+ + + HTML + + html_page(html) # refute_raises + end end From cfa06d2fc560665a67c16348671b67780ebb2b15 Mon Sep 17 00:00:00 2001 From: Ruby Date: Fri, 7 Apr 2023 09:11:20 -0400 Subject: [PATCH 18/72] doc: update CHANGELOG [skip ci] --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f58e9c4..1b570c41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ * Mechanize now requires Ruby 2.6 or newer. +### Improvement + +* Mechanize can now parse frozen strings. (#610) + + ## 2.8.5 / 2022-06-09 ### Security From 77ab71c346e11c4e0c9ca8d34561f5a276555f45 Mon Sep 17 00:00:00 2001 From: Ruby Date: Fri, 7 Apr 2023 09:13:53 -0400 Subject: [PATCH 19/72] ci: update to actions/checkout@v3 to quash the node 12 warnings --- .github/workflows/ci-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 42ff238c..366fab6a 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -19,7 +19,7 @@ jobs: rubocop: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ruby/setup-ruby@v1 with: ruby-version: "3.2" @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{matrix.ruby-version}} @@ -52,7 +52,7 @@ jobs: runs-on: ${{matrix.platform}} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ruby/setup-ruby@v1 with: ruby-version: "3.2" From cf041d7892cb2b1da6b7b74237a9b50155d35b9c Mon Sep 17 00:00:00 2001 From: Ruby Date: Fri, 7 Apr 2023 09:17:58 -0400 Subject: [PATCH 20/72] version bump to v2.9.0 --- CHANGELOG.md | 2 +- lib/mechanize/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b570c41..318a4b3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Mechanize CHANGELOG -## next / unreleased +## 2.9.0 / 2023-04-07 ### Requirements diff --git a/lib/mechanize/version.rb b/lib/mechanize/version.rb index 60a7e6e9..1fa6c789 100644 --- a/lib/mechanize/version.rb +++ b/lib/mechanize/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true class Mechanize - VERSION = "2.8.5" + VERSION = "2.9.0" end From 78791c5844d73706eb05ebedcc2dc866c6091f52 Mon Sep 17 00:00:00 2001 From: Naoki Takashima Date: Fri, 14 Apr 2023 10:46:40 +0900 Subject: [PATCH 21/72] add example to fetch latest user agents --- examples/latest_user_agents.rb | 91 ++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 examples/latest_user_agents.rb diff --git a/examples/latest_user_agents.rb b/examples/latest_user_agents.rb new file mode 100644 index 00000000..5773d003 --- /dev/null +++ b/examples/latest_user_agents.rb @@ -0,0 +1,91 @@ +require 'mechanize' + +class LatestUAFetcher + attr_reader :user_agents + + BASE_URL = 'https://www.whatismybrowser.com/guides/the-latest-user-agent' + + def initialize + @agent = Mechanize.new.tap { |a| a.user_agent_alias = 'Mac Firefox' } + @user_agents = {} + end + + def run + sleep_time = 1 + + puts 'get chrome UA...' + chrome + puts "sleeping... (#{sleep_time}s)" + sleep 1 + + puts 'get firefox UA...' + firefox + puts "sleeping... (#{sleep_time}s)" + sleep 1 + + puts 'get safari UA...' + safari + puts "sleeping... (#{sleep_time}s)" + sleep 1 + + puts 'get edge UA...' + edge + end + + private + + def edge + page = @agent.get("#{BASE_URL}/edge") + + windows_dom = page.css("h2:contains('Latest Edge on Windows User Agents')") + @user_agents[:edge] = { + windows: windows_dom.css('+ .listing-of-useragents .code').first.text + } + end + + def firefox + page = @agent.get("#{BASE_URL}/firefox") + + desktop_dom = page.css("h2:contains('Latest Firefox on Desktop User Agents')") + table_dom = desktop_dom.css('+ .listing-of-useragents') + + @user_agents[:firefox] = { + windows: table_dom.css('td:contains("Windows")').css('+ td .code').text, + macOS: table_dom.css('td:contains("Macos")').css('+ td .code').text, + linux: table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text + } + end + + def safari + page = @agent.get("#{BASE_URL}/safari") + + macos_dom = page.css("h2:contains('Latest Safari on macOS User Agents')") + ios_dom = page.css("h2:contains('Latest Safari on iOS User Agents')") + + @user_agents[:safari] = { + mac_os: macos_dom.css('+ .listing-of-useragents .code').first.text, + iphone: ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text, + ipad: ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text + } + end + + def chrome + page = @agent.get("#{BASE_URL}/chrome") + + windows_dom = page.css("h2:contains('Latest Chrome on Windows 10 User Agents')") + linux_dom = page.css("h2:contains('Latest Chrome on Linux User Agents')") + macos_dom = page.css("h2:contains('Latest Chrome on macOS User Agents')") + android_dom = page.css("h2:contains('Latest Chrome on Android User Agents')") + + @user_agents[:chrome] = { + windows: windows_dom.css('+ .listing-of-useragents .code').first.text, + linux: linux_dom.css('+ .listing-of-useragents .code').first.text, + mac_os: macos_dom.css('+ .listing-of-useragents .code').first.text, + android: android_dom.css('+ .listing-of-useragents .code').first.text + } + end +end + +agent = LatestUAFetcher.new +agent.run +p agent.user_agents From 96d21aae1be09fd7fb04e24e16b496a2c37dcdfc Mon Sep 17 00:00:00 2001 From: Naoki Takashima Date: Fri, 14 Apr 2023 11:03:30 +0900 Subject: [PATCH 22/72] update user agent strings for agent aliases --- lib/mechanize.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/mechanize.rb b/lib/mechanize.rb index 457d9eec..19fe9a01 100644 --- a/lib/mechanize.rb +++ b/lib/mechanize.rb @@ -115,26 +115,26 @@ class Error < RuntimeError AGENT_ALIASES = { 'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/sparklemotion/mechanize/)", - 'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0', + 'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:112.0) Gecko/20100101 Firefox/112.0', 'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)', 'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624', - 'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0', + 'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13.3; rv:112.0) Gecko/20100101 Firefox/112.0', 'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401', 'Mac Safari 4' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; de-at) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10', - 'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9', - 'Windows Chrome' => 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.125 Safari/537.36', + 'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', + 'Windows Chrome' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', 'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', 'Windows IE 8' => 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', 'Windows IE 9' => 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)', 'Windows IE 10' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)', 'Windows IE 11' => 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586', + 'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.46', 'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6', - 'Windows Firefox' => 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0', - 'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B5110e Safari/601.1', - 'iPad' => 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1', - 'Android' => 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.76 Safari/537.36', + 'Windows Firefox' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:112.0) Gecko/20100101 Firefox/112.0', + 'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', + 'iPad' => 'Mozilla/5.0 (iPad; CPU OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', + 'Android' => 'Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.48 Mobile Safari/537.36', } AGENT_ALIASES.default_proc = proc { |hash, key| From 18b7efdc761ff18dc68ece5ec5a2c908adb19c0a Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sat, 15 Apr 2023 15:57:58 -0400 Subject: [PATCH 23/72] doc: update docstring for AGENT_ALIASES and CHANGELOG --- CHANGELOG.md | 5 ++++ lib/mechanize.rb | 69 ++++++++++++++++++++++++++++++------------------ 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 318a4b3d..ed90503a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Mechanize CHANGELOG +## next / unreleased + +* Updated User-Agent strings to represent modern browser versions. (#612) Thank you, @takatea! + + ## 2.9.0 / 2023-04-07 ### Requirements diff --git a/lib/mechanize.rb b/lib/mechanize.rb index 19fe9a01..6ae4b49f 100644 --- a/lib/mechanize.rb +++ b/lib/mechanize.rb @@ -87,54 +87,73 @@ class Error < RuntimeError # description in parenthesis is for informative purposes and is not part of # the alias name. # - # * Linux Firefox (43.0 on Ubuntu Linux) - # * Linux Konqueror (3) - # * Linux Mozilla - # * Mac Firefox (43.0) - # * Mac Mozilla - # * Mac Safari (9.0 on OS X 10.11.2) - # * Mac Safari 4 - # * Mechanize (default) - # * Windows IE 6 - # * Windows IE 7 - # * Windows IE 8 - # * Windows IE 9 - # * Windows IE 10 (Windows 8 64bit) - # * Windows IE 11 (Windows 8.1 64bit) - # * Windows Edge - # * Windows Mozilla - # * Windows Firefox (43.0) - # * iPhone (iOS 9.1) - # * iPad (iOS 9.1) - # * Android (5.1.1) + # The default User-Agent alias: + # + # * "Mechanize" + # + # Linux User-Agent aliases: + # + # * "Linux Firefox" + # * "Linux Konqueror" + # * "Linux Mozilla" + # + # Mac User-Agent aliases: + # + # * "Mac Firefox" + # * "Mac Mozilla" + # * "Mac Safari 4" + # * "Mac Safari" + # + # Windows User-Agent aliases: + # + # * "Windows Edge" + # * "Windows Firefox" + # * "Windows IE 6" + # * "Windows IE 7" + # * "Windows IE 8" + # * "Windows IE 9" + # * "Windows IE 10" + # * "Windows IE 11" + # * "Windows Mozilla" + # + # Mobile User-Agent aliases: + # + # * "Android" + # * "iPad" + # * "iPhone" # # Example: # # agent = Mechanize.new # agent.user_agent_alias = 'Mac Safari' - + # AGENT_ALIASES = { + # TODO: use output from examples/latest_user_agents.rb as the underling data structure 'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/sparklemotion/mechanize/)", + 'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:112.0) Gecko/20100101 Firefox/112.0', 'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)', 'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624', + 'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13.3; rv:112.0) Gecko/20100101 Firefox/112.0', 'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401', 'Mac Safari 4' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; de-at) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10', 'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', + 'Windows Chrome' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', + 'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.46', + 'Windows Firefox' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:112.0) Gecko/20100101 Firefox/112.0', 'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', 'Windows IE 8' => 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', 'Windows IE 9' => 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)', 'Windows IE 10' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)', 'Windows IE 11' => 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.46', 'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6', - 'Windows Firefox' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:112.0) Gecko/20100101 Firefox/112.0', - 'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', - 'iPad' => 'Mozilla/5.0 (iPad; CPU OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', + 'Android' => 'Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.48 Mobile Safari/537.36', + 'iPad' => 'Mozilla/5.0 (iPad; CPU OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', + 'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', } AGENT_ALIASES.default_proc = proc { |hash, key| From 59cc064c85ff408d73d688f1127fd37f461a6ba6 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 17 Apr 2023 17:19:01 -0400 Subject: [PATCH 24/72] version bump to v2.9.1 --- CHANGELOG.md | 4 +++- lib/mechanize/version.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed90503a..4f9c22a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Mechanize CHANGELOG -## next / unreleased +## 2.9.1 / 2023-04-17 + +### Update * Updated User-Agent strings to represent modern browser versions. (#612) Thank you, @takatea! diff --git a/lib/mechanize/version.rb b/lib/mechanize/version.rb index 1fa6c789..db25d2ab 100644 --- a/lib/mechanize/version.rb +++ b/lib/mechanize/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true class Mechanize - VERSION = "2.9.0" + VERSION = "2.9.1" end From 762df0c71705d1c054aca4abd95a638774a47242 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 7 Jun 2023 10:30:37 -0400 Subject: [PATCH 25/72] test: work around libxml2 encoding changes versions 2.11.0..2.11.4 do not recover from encoding errors. upcoming release will change the encoding of the resulting doc as a result of recovery changes in libxml2. See https://gitlab.gnome.org/GNOME/libxml2/-/issues/543 for more context. --- test/test_mechanize_page_link.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/test_mechanize_page_link.rb b/test/test_mechanize_page_link.rb index 5e77dcc0..988328cc 100644 --- a/test/test_mechanize_page_link.rb +++ b/test/test_mechanize_page_link.rb @@ -1,5 +1,7 @@ # coding: utf-8 +puts "Nokogiri::VERSION_INFO: #{Nokogiri::VERSION_INFO}" + require 'mechanize/test_case' class TestMechanizePageLink < Mechanize::TestCase @@ -111,11 +113,15 @@ def test_encoding_charset_after_title def test_encoding_charset_after_title_bad skip_if_nkf_dependency + # https://gitlab.gnome.org/GNOME/libxml2/-/issues/543 + skip if Nokogiri.uses_libxml?([">= 2.11.0", "<= 2.11.4"]) + expected_encoding = Nokogiri.uses_libxml?("< 2.11.0") ? 'UTF-8' : 'Shift_JIS' + page = util_page UTF8 assert_equal false, page.encoding_error? - assert_equal 'UTF-8', page.encoding + assert_equal expected_encoding, page.encoding end def test_encoding_charset_after_title_double_bad @@ -131,6 +137,10 @@ def test_encoding_charset_after_title_double_bad def test_encoding_charset_bad skip_if_nkf_dependency + # https://gitlab.gnome.org/GNOME/libxml2/-/issues/543 + skip if Nokogiri.uses_libxml?([">= 2.11.0", "<= 2.11.4"]) + expected_encoding = Nokogiri.uses_libxml?("< 2.11.0") ? 'UTF-8' : 'Shift_JIS' + page = util_page "#{UTF8_TITLE}" page.encodings.replace %w[ UTF-8 @@ -139,7 +149,7 @@ def test_encoding_charset_bad assert_equal false, page.encoding_error? - assert_equal 'UTF-8', page.encoding + assert_equal expected_encoding, page.encoding end def test_encoding_meta_charset From 0ba09f46be6f1ce4adb94baca9867ed769f6535a Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 11 Aug 2023 09:29:06 -0400 Subject: [PATCH 26/72] test: work around libxml2 encoding changes Related to 762df0c7, we're seeing the upstream fixes did not ship in v2.11.5 so let's change the version test to assume it won't land until v2.12.0. --- test/test_mechanize_page_link.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_mechanize_page_link.rb b/test/test_mechanize_page_link.rb index 988328cc..04294420 100644 --- a/test/test_mechanize_page_link.rb +++ b/test/test_mechanize_page_link.rb @@ -114,7 +114,7 @@ def test_encoding_charset_after_title_bad skip_if_nkf_dependency # https://gitlab.gnome.org/GNOME/libxml2/-/issues/543 - skip if Nokogiri.uses_libxml?([">= 2.11.0", "<= 2.11.4"]) + skip if Nokogiri.uses_libxml?([">= 2.11.0", "< 2.12.0"]) expected_encoding = Nokogiri.uses_libxml?("< 2.11.0") ? 'UTF-8' : 'Shift_JIS' page = util_page UTF8 @@ -138,7 +138,7 @@ def test_encoding_charset_bad skip_if_nkf_dependency # https://gitlab.gnome.org/GNOME/libxml2/-/issues/543 - skip if Nokogiri.uses_libxml?([">= 2.11.0", "<= 2.11.4"]) + skip if Nokogiri.uses_libxml?([">= 2.11.0", "< 2.12.0"]) expected_encoding = Nokogiri.uses_libxml?("< 2.11.0") ? 'UTF-8' : 'Shift_JIS' page = util_page "#{UTF8_TITLE}" From 37785c0c90a7c25c24d546f190dd2fa8c2c1a694 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sat, 16 Sep 2023 13:48:09 -0400 Subject: [PATCH 27/72] ci: test_mechanize_page_link requires nokogiri before using it --- test/test_mechanize_page_link.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_mechanize_page_link.rb b/test/test_mechanize_page_link.rb index 04294420..520a13c1 100644 --- a/test/test_mechanize_page_link.rb +++ b/test/test_mechanize_page_link.rb @@ -1,9 +1,9 @@ # coding: utf-8 -puts "Nokogiri::VERSION_INFO: #{Nokogiri::VERSION_INFO}" - require 'mechanize/test_case' +puts "Nokogiri::VERSION_INFO: #{Nokogiri::VERSION_INFO}" + class TestMechanizePageLink < Mechanize::TestCase WINDOWS_1255 = <<-HTML From c037db67a9637d400e47cf71cbe3859e7565fd00 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 13 Dec 2023 17:16:39 -0500 Subject: [PATCH 28/72] ci: enable manual workflow --- .github/workflows/ci-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 366fab6a..b926db8c 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -5,6 +5,9 @@ concurrency: cancel-in-progress: true on: + workflow_dispatch: + schedule: + - cron: "0 8 * * 5" # At 08:00 on Friday # https://crontab.guru/#0_8_*_*_5 push: branches: - main @@ -12,8 +15,6 @@ on: types: [opened, synchronize] branches: - main - schedule: - - cron: "0 8 * * 5" # At 08:00 on Friday # https://crontab.guru/#0_8_*_*_5 jobs: rubocop: From 13c8c39028627bb3175c674e67ba2fb426dbd2a8 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 13 Dec 2023 17:32:18 -0500 Subject: [PATCH 29/72] dep: remove mutex_m which is unused since 4d074f4d (2010) --- lib/mechanize.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/mechanize.rb b/lib/mechanize.rb index 6ae4b49f..60e52860 100644 --- a/lib/mechanize.rb +++ b/lib/mechanize.rb @@ -2,7 +2,6 @@ require 'mechanize/version' require 'fileutils' require 'forwardable' -require 'mutex_m' require 'net/http/digest_auth' require 'net/http/persistent' require 'nokogiri' From 245cc7856b33c59191b2f542c4fe12c977d09656 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 29 Dec 2023 10:20:01 -0500 Subject: [PATCH 30/72] ci: add ruby 3.3 to the matrix --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b926db8c..37103b0c 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby-9.4", "truffleruby-head"] + ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3", "head", "jruby-9.4", "truffleruby-head"] runs-on: ubuntu-latest steps: From 46482f1869352d240bd9809c86bff6057c316d9d Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 29 Dec 2023 10:35:16 -0500 Subject: [PATCH 31/72] test: accept "UTF-8" as an encoding alias for "UTF8" As of libxml 2.12.0, the `page.encoding` value inspected by this test is dependent on how libiconv is built (specifically: which aliases are supported). - if alias "UTF8" is defined, then the page encoding will be "UTF-8" - if alias "UTF8" is not defined, then the page encoding will be "UTF8" The modified test started failing on Windows and MacOS (where Nokogiri compiles libiconv and statically links it into the native (precompiled) gem) but still passed on Linux (where libc dynamically provides libiconv symbols). --- test/test_mechanize_http_agent.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test_mechanize_http_agent.rb b/test/test_mechanize_http_agent.rb index f6beb06d..f2fea1e7 100644 --- a/test/test_mechanize_http_agent.rb +++ b/test/test_mechanize_http_agent.rb @@ -1309,8 +1309,13 @@ def test_response_parse_content_type_encoding_broken_utf_8 page = @agent.response_parse @res, body, @uri assert_instance_of Mechanize::Page, page - assert_equal 'UTF8', page.encoding - assert_equal 'UTF8', page.parser.encoding + + # as of libxml 2.12.0, the result is dependent on how libiconv is built (which aliases are supported) + # if the alias "UTF8" is defined, then the result will be "UTF-8". + # if the alias "UTF8" is not defined, then the result will be "UTF8". + # note that this alias may be defined by Nokogiri itself in its EncodingHandler class. + assert_includes ["UTF8", "UTF-8"], page.encoding + assert_includes ["UTF8", "UTF-8"], page.parser.encoding end def test_response_parse_content_type_encoding_garbage From b3e9a0d0c984222e182e6923c4ac8ff9c81fbbcd Mon Sep 17 00:00:00 2001 From: Patrick Linnane Date: Fri, 12 Jan 2024 14:11:00 -0800 Subject: [PATCH 32/72] various: fix miscellaneous typos Signed-off-by: Patrick Linnane --- CHANGELOG.md | 18 +++++++++--------- lib/mechanize.rb | 4 ++-- lib/mechanize/form.rb | 6 +++--- lib/mechanize/form/multi_select_list.rb | 2 +- lib/mechanize/http/agent.rb | 2 +- .../http/content_disposition_parser.rb | 3 +-- lib/mechanize/page.rb | 2 +- test/test_mechanize_cookie_jar.rb | 2 +- test/test_mechanize_form.rb | 2 +- 9 files changed, 20 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f9c22a7..f754178f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,7 +109,7 @@ Fixes low-severity CVE-2022-31033, "Authorization header leak on port redirect." * Fix element(s)_with(search: selector) methods not working for forms, form fields and frames. (#444) * Improve the filename parser for the `Content-Disposition` header. (#496, #517) * Accept `Content-Encoding: identity`. (#515) - * Mechanize::Page#title no longer picks a title in an embeded SVG/RDF element. (#503) + * Mechanize::Page#title no longer picks a title in an embedded SVG/RDF element. (#503) * Make Mechanize::Form#has_field? boolean. (#501) ## 2.7.5 @@ -415,8 +415,8 @@ Fixes low-severity CVE-2022-31033, "Authorization header leak on port redirect." * When given multiple HTTP authentication options mechanize now picks the strongest method. * Improvements to HTTP authorization: - * mechanize raises Mechanize::UnathorizedError for 401 responses which is - a sublcass of Mechanize::ResponseCodeError. + * mechanize raises Mechanize::UnauthorizedError for 401 responses which is + a subclass of Mechanize::ResponseCodeError. * Added support for NTLM authentication, but this has not been tested. * Mechanize::Cookie.new accepts attributes in a hash. * Mechanize::CookieJar#<<(cookie) (alias: add!) is added. Issue #139 @@ -507,7 +507,7 @@ Mechanize is now under the MIT license * New Features - * Add header reference methods to Mechanize::File so that a reponse + * Add header reference methods to Mechanize::File so that a response object gets compatible with Net::HTTPResponse. * Mechanize#click accepts a regexp or string to click a button/link in the current page. It works as expected when not passed a string or regexp. @@ -732,7 +732,7 @@ Mechanize is now under the MIT license http://d.hatena.ne.jp/kitamomonga/20080410/ruby_mechanize_percent_url_bug * #21132 Not checking for EOF errors on redirect * Fixed a weird gzipping error. - * #21233 Smarter multipart boundry. Thanks Todd Willey! + * #21233 Smarter multipart boundary. Thanks Todd Willey! * #20097 Supporting meta tag cookies. ## 0.7.6 @@ -841,7 +841,7 @@ Mechanize is now under the MIT license * [#9877] Moved last request time. Thanks Max Stepanov * Added WWW::Mechanize::File#save * Defaulting file name to URI or Content-Disposition -* Updating compatability with hpricot +* Updating compatibility with hpricot * Added more unit tests ## 0.6.7 @@ -853,7 +853,7 @@ Mechanize is now under the MIT license * Removing hpricot overrides * Fixed a bug where alt text can be nil. Thanks Yannick! -* Unparseable expiration dates in cookies are now treated as session cookies +* Unparsable expiration dates in cookies are now treated as session cookies * Caching connections * Requests now default to keep alive * [#9434] Fixed bug where html entities weren't decoded @@ -919,7 +919,7 @@ Mechanize is now under the MIT license * Added a method to Form called "submit". Now forms can be submitted by calling a method on the form. * Added a click method to links -* Added an REXML pluggable parser for backwards compatability. To use it, +* Added an REXML pluggable parser for backwards compatibility. To use it, just do this: agent.pluggable_parser.html = WWW::Mechanize::REXMLPage * Fixed a bug with referrers by adding a page attribute to forms and links. @@ -1046,7 +1046,7 @@ Mechanize is now under the MIT license ## 0.4.4 -* Fixed error in method signature, basic_authetication is now basic_auth +* Fixed error in method signature, basic_authentication is now basic_auth * Fixed bug with encoding names in file uploads (Big thanks to Alex Young) * Added options to the select list diff --git a/lib/mechanize.rb b/lib/mechanize.rb index 60e52860..08511d02 100644 --- a/lib/mechanize.rb +++ b/lib/mechanize.rb @@ -200,7 +200,7 @@ def self.start # # If you need segregated SSL connections give each agent a unique # name. Otherwise the connections will be shared. This is - # particularly important if you are using certifcates. + # particularly important if you are using certificates. # # agent_1 = Mechanize.new 'conn1' # agent_2 = Mechanize.new 'conn2' @@ -956,7 +956,7 @@ def read_timeout= read_timeout # allowed: # # :all, true:: All 3xx redirects are followed (default) - # :permanent:: Only 301 Moved Permanantly redirects are followed + # :permanent:: Only 301 Moved Permanently redirects are followed # false:: No redirects are followed def redirect_ok diff --git a/lib/mechanize/form.rb b/lib/mechanize/form.rb index 8ecd0e0f..eb87cc5d 100644 --- a/lib/mechanize/form.rb +++ b/lib/mechanize/form.rb @@ -482,7 +482,7 @@ def delete_field!(field_name) # form.file_upload_with(:file_name => /picture/).value = 'foo' ## - # :mehtod: file_upload_with!(criteria) + # :method: file_upload_with!(criteria) # # Same as +file_upload_with+ but raises an ElementNotFoundError if no button # matches +criteria+ @@ -506,7 +506,7 @@ def delete_field!(field_name) # form.radiobutton_with(:name => /woo/).check ## - # :mehtod: radiobutton_with!(criteria) + # :method: radiobutton_with!(criteria) # # Same as +radiobutton_with+ but raises an ElementNotFoundError if no button # matches +criteria+ @@ -530,7 +530,7 @@ def delete_field!(field_name) # form.checkbox_with(:name => /woo/).check ## - # :mehtod: checkbox_with!(criteria) + # :method: checkbox_with!(criteria) # # Same as +checkbox_with+ but raises an ElementNotFoundError if no button # matches +criteria+ diff --git a/lib/mechanize/form/multi_select_list.rb b/lib/mechanize/form/multi_select_list.rb index d7e718b4..62e789c8 100644 --- a/lib/mechanize/form/multi_select_list.rb +++ b/lib/mechanize/form/multi_select_list.rb @@ -36,7 +36,7 @@ def initialize node # select_list.option_with(:value => '1').value = 'foo' ## - # :mehtod: option_with!(criteria) + # :method: option_with!(criteria) # # Same as +option_with+ but raises an ElementNotFoundError if no button # matches +criteria+ diff --git a/lib/mechanize/http/agent.rb b/lib/mechanize/http/agent.rb index 9e433a3f..4740d460 100644 --- a/lib/mechanize/http/agent.rb +++ b/lib/mechanize/http/agent.rb @@ -66,7 +66,7 @@ class Mechanize::HTTP::Agent # allowed: # # :all, true:: All 3xx redirects are followed (default) - # :permanent:: Only 301 Moved Permanantly redirects are followed + # :permanent:: Only 301 Moved Permanently redirects are followed # false:: No redirects are followed attr_accessor :redirect_ok diff --git a/lib/mechanize/http/content_disposition_parser.rb b/lib/mechanize/http/content_disposition_parser.rb index 3ea90971..f8b8f9a9 100644 --- a/lib/mechanize/http/content_disposition_parser.rb +++ b/lib/mechanize/http/content_disposition_parser.rb @@ -81,7 +81,7 @@ def parse content_disposition, header = false end ## - # Extracts disposition-parm and returns a Hash. + # Extracts disposition-param and returns a Hash. def parse_parameters parameters = {} @@ -200,4 +200,3 @@ def spaces end end - diff --git a/lib/mechanize/page.rb b/lib/mechanize/page.rb index c50d62eb..96ffac06 100644 --- a/lib/mechanize/page.rb +++ b/lib/mechanize/page.rb @@ -95,7 +95,7 @@ def encoding end # Return whether parser result has errors related to encoding or not. - # false indicates just parser has no encoding errors, not encoding is vaild. + # false indicates just parser has no encoding errors, not encoding is valid. def encoding_error?(parser=nil) parser = self.parser unless parser return false if parser.errors.empty? diff --git a/test/test_mechanize_cookie_jar.rb b/test/test_mechanize_cookie_jar.rb index 56259702..2af06dfe 100644 --- a/test/test_mechanize_cookie_jar.rb +++ b/test/test_mechanize_cookie_jar.rb @@ -540,7 +540,7 @@ def test_save_and_read_expired_cookies @jar.add url, Mechanize::Cookie.new(cookie_values) - # HACK no asertion + # HACK no assertion end def test_ssl_cookies diff --git a/test/test_mechanize_form.rb b/test/test_mechanize_form.rb index 610a4bdd..b6247ddc 100644 --- a/test/test_mechanize_form.rb +++ b/test/test_mechanize_form.rb @@ -357,7 +357,7 @@ def test_submit_first_field_wins assert_equal 'ticky=1&ticky=0', submitted.parser.at('#query').text end - def test_submit_takes_arbirary_headers + def test_submit_takes_arbitrary_headers page = @mech.get('http://localhost:2000/form_no_action.html') assert form = page.forms.first form.action = '/http_headers' From 8f0bfa0017026b7ca5710f5799a7d11885b863e4 Mon Sep 17 00:00:00 2001 From: takatea Date: Sat, 13 Jan 2024 19:17:39 +0900 Subject: [PATCH 33/72] update example of latest_user_agents Mechanize::AGENT_ALIASES can easily update by using examples/latest_user_agents.rb example: ``` require './examples/latest_user_agents' agent = LatestUAFetcher.new updated_user_agents = agent.user_agents.dup.delete_if do |key, value| Mechanize::AGENT_ALIASES[key] == value end pp updated_user_agents ``` --- examples/latest_user_agents.rb | 70 +++++++++++++++------------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/examples/latest_user_agents.rb b/examples/latest_user_agents.rb index 5773d003..0cfb1194 100644 --- a/examples/latest_user_agents.rb +++ b/examples/latest_user_agents.rb @@ -1,5 +1,7 @@ require 'mechanize' +# LatestUAFetcher fetches latest user agents from `WhatIsMyBrowser.com`. +# It can use to update `Mechanize::AGENT_ALIASES`. class LatestUAFetcher attr_reader :user_agents @@ -11,36 +13,31 @@ def initialize end def run + return unless user_agents.empty? + sleep_time = 1 - puts 'get chrome UA...' - chrome - puts "sleeping... (#{sleep_time}s)" - sleep 1 + fetch_user_agents('chrome') + fetch_user_agents('firefox') + fetch_user_agents('safari') + fetch_user_agents('edge') + end - puts 'get firefox UA...' - firefox - puts "sleeping... (#{sleep_time}s)" - sleep 1 + private - puts 'get safari UA...' - safari + def fetch_user_agents(browser_name, sleep_time = 1) + puts "fetch #{browser_name} UA..." + send(browser_name) puts "sleeping... (#{sleep_time}s)" - sleep 1 - - puts 'get edge UA...' - edge + sleep sleep_time end - private - def edge page = @agent.get("#{BASE_URL}/edge") windows_dom = page.css("h2:contains('Latest Edge on Windows User Agents')") - @user_agents[:edge] = { - windows: windows_dom.css('+ .listing-of-useragents .code').first.text - } + + @user_agents['Windows Edge'] = windows_dom.css('+ .listing-of-useragents .code').first.text end def firefox @@ -49,11 +46,9 @@ def firefox desktop_dom = page.css("h2:contains('Latest Firefox on Desktop User Agents')") table_dom = desktop_dom.css('+ .listing-of-useragents') - @user_agents[:firefox] = { - windows: table_dom.css('td:contains("Windows")').css('+ td .code').text, - macOS: table_dom.css('td:contains("Macos")').css('+ td .code').text, - linux: table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text - } + @user_agents['Linux Firefox'] = table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text + @user_agents['Windows Firefox'] = table_dom.css('td:contains("Windows")').css('+ td .code').text + @user_agents['Mac Firefox'] = table_dom.css('td:contains("Macos")').css('+ td .code').text end def safari @@ -62,30 +57,25 @@ def safari macos_dom = page.css("h2:contains('Latest Safari on macOS User Agents')") ios_dom = page.css("h2:contains('Latest Safari on iOS User Agents')") - @user_agents[:safari] = { - mac_os: macos_dom.css('+ .listing-of-useragents .code').first.text, - iphone: ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text, - ipad: ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text - } + @user_agents['Mac Safari'] = macos_dom.css('+ .listing-of-useragents .code').first.text + @user_agents['iPhone'] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text + @user_agents['iPad'] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text end def chrome page = @agent.get("#{BASE_URL}/chrome") windows_dom = page.css("h2:contains('Latest Chrome on Windows 10 User Agents')") - linux_dom = page.css("h2:contains('Latest Chrome on Linux User Agents')") - macos_dom = page.css("h2:contains('Latest Chrome on macOS User Agents')") android_dom = page.css("h2:contains('Latest Chrome on Android User Agents')") - @user_agents[:chrome] = { - windows: windows_dom.css('+ .listing-of-useragents .code').first.text, - linux: linux_dom.css('+ .listing-of-useragents .code').first.text, - mac_os: macos_dom.css('+ .listing-of-useragents .code').first.text, - android: android_dom.css('+ .listing-of-useragents .code').first.text - } + @user_agents['Windows Chrome'] = windows_dom.css('+ .listing-of-useragents .code').first.text + @user_agents['Android'] = android_dom.css('+ .listing-of-useragents .code').first.text end end -agent = LatestUAFetcher.new -agent.run -p agent.user_agents +if $0 == __FILE__ + agent = LatestUAFetcher.new + agent.run + + pp agent.user_agents +end From baae66d1951b304b1774036451dab36eaef44fe8 Mon Sep 17 00:00:00 2001 From: takatea Date: Sat, 13 Jan 2024 22:09:58 +0900 Subject: [PATCH 34/72] feat: output ordered user agents --- examples/latest_user_agents.rb | 39 +++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/examples/latest_user_agents.rb b/examples/latest_user_agents.rb index 0cfb1194..1a55932a 100644 --- a/examples/latest_user_agents.rb +++ b/examples/latest_user_agents.rb @@ -1,10 +1,23 @@ require 'mechanize' +require 'ostruct' # LatestUAFetcher fetches latest user agents from `WhatIsMyBrowser.com`. # It can use to update `Mechanize::AGENT_ALIASES`. class LatestUAFetcher attr_reader :user_agents + USER_AGENT_TYPES = OpenStruct.new( + linux_firefox: "Linux Firefox", + mac_firefox: "Mac Firefox", + mac_safari: "Mac Safari", + windows_chrome: "Windows Chrome", + windows_edge: "Windows Edge", + windows_firefox: "Windows Firefox", + android: "Android", + iphone: "iPhone", + ipad: "iPad", + ) + BASE_URL = 'https://www.whatismybrowser.com/guides/the-latest-user-agent' def initialize @@ -23,6 +36,12 @@ def run fetch_user_agents('edge') end + def ordered_user_agents + USER_AGENT_TYPES.to_h.values.each_with_object({}) do |type, ordered_user_agents| + ordered_user_agents[type] = user_agents[type] + end + end + private def fetch_user_agents(browser_name, sleep_time = 1) @@ -37,7 +56,7 @@ def edge windows_dom = page.css("h2:contains('Latest Edge on Windows User Agents')") - @user_agents['Windows Edge'] = windows_dom.css('+ .listing-of-useragents .code').first.text + @user_agents[USER_AGENT_TYPES.windows_edge] = windows_dom.css('+ .listing-of-useragents .code').first.text end def firefox @@ -46,9 +65,9 @@ def firefox desktop_dom = page.css("h2:contains('Latest Firefox on Desktop User Agents')") table_dom = desktop_dom.css('+ .listing-of-useragents') - @user_agents['Linux Firefox'] = table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text - @user_agents['Windows Firefox'] = table_dom.css('td:contains("Windows")').css('+ td .code').text - @user_agents['Mac Firefox'] = table_dom.css('td:contains("Macos")').css('+ td .code').text + @user_agents[USER_AGENT_TYPES.linux_firefox] = table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text + @user_agents[USER_AGENT_TYPES.windows_firefox] = table_dom.css('td:contains("Windows")').css('+ td .code').text + @user_agents[USER_AGENT_TYPES.mac_firefox] = table_dom.css('td:contains("Macos")').css('+ td .code').text end def safari @@ -57,9 +76,9 @@ def safari macos_dom = page.css("h2:contains('Latest Safari on macOS User Agents')") ios_dom = page.css("h2:contains('Latest Safari on iOS User Agents')") - @user_agents['Mac Safari'] = macos_dom.css('+ .listing-of-useragents .code').first.text - @user_agents['iPhone'] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text - @user_agents['iPad'] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text + @user_agents[USER_AGENT_TYPES.mac_safari] = macos_dom.css('+ .listing-of-useragents .code').first.text + @user_agents[USER_AGENT_TYPES.iphone] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text + @user_agents[USER_AGENT_TYPES.ipad] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text end def chrome @@ -68,8 +87,8 @@ def chrome windows_dom = page.css("h2:contains('Latest Chrome on Windows 10 User Agents')") android_dom = page.css("h2:contains('Latest Chrome on Android User Agents')") - @user_agents['Windows Chrome'] = windows_dom.css('+ .listing-of-useragents .code').first.text - @user_agents['Android'] = android_dom.css('+ .listing-of-useragents .code').first.text + @user_agents[USER_AGENT_TYPES.windows_chrome] = windows_dom.css('+ .listing-of-useragents .code').first.text + @user_agents[USER_AGENT_TYPES.android] = android_dom.css('+ .listing-of-useragents .code').first.text end end @@ -77,5 +96,5 @@ def chrome agent = LatestUAFetcher.new agent.run - pp agent.user_agents + pp agent.ordered_user_agents end From 32c74eec9dc8cf30132b87c58ad2c6f37ffcde08 Mon Sep 17 00:00:00 2001 From: takatea Date: Sat, 13 Jan 2024 19:31:00 +0900 Subject: [PATCH 35/72] update user agent strings for agent aliases --- lib/mechanize.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/mechanize.rb b/lib/mechanize.rb index 60e52860..afd54806 100644 --- a/lib/mechanize.rb +++ b/lib/mechanize.rb @@ -130,18 +130,18 @@ class Error < RuntimeError # TODO: use output from examples/latest_user_agents.rb as the underling data structure 'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/sparklemotion/mechanize/)", - 'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:112.0) Gecko/20100101 Firefox/112.0', + 'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/121.0', 'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)', 'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624', - 'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13.3; rv:112.0) Gecko/20100101 Firefox/112.0', + 'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14.2; rv:109.0) Gecko/20100101 Firefox/121.0', 'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401', 'Mac Safari 4' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; de-at) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10', - 'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15', + 'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15', - 'Windows Chrome' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', - 'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.46', - 'Windows Firefox' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:112.0) Gecko/20100101 Firefox/112.0', + 'Windows Chrome' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.2210.133', + 'Windows Firefox' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0', 'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', 'Windows IE 8' => 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)', @@ -150,9 +150,9 @@ class Error < RuntimeError 'Windows IE 11' => 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko', 'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6', - 'Android' => 'Mozilla/5.0 (Linux; Android 10) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.48 Mobile Safari/537.36', - 'iPad' => 'Mozilla/5.0 (iPad; CPU OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', - 'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Mobile/15E148 Safari/604.1', + 'Android' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.210 Mobile Safari/537.36', + 'iPad' => 'Mozilla/5.0 (iPad; CPU OS 17_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1', + 'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1', } AGENT_ALIASES.default_proc = proc { |hash, key| From 34ccbd614dffe1525403f8a606d13006f42ec140 Mon Sep 17 00:00:00 2001 From: takatea Date: Sat, 13 Jan 2024 19:45:06 +0900 Subject: [PATCH 36/72] doc: update docstring for AGENT_ALIASES and remove todo comment --- lib/mechanize.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mechanize.rb b/lib/mechanize.rb index afd54806..acc9585b 100644 --- a/lib/mechanize.rb +++ b/lib/mechanize.rb @@ -105,6 +105,7 @@ class Error < RuntimeError # # Windows User-Agent aliases: # + # * "Windows Chrome" # * "Windows Edge" # * "Windows Firefox" # * "Windows IE 6" @@ -127,7 +128,6 @@ class Error < RuntimeError # agent.user_agent_alias = 'Mac Safari' # AGENT_ALIASES = { - # TODO: use output from examples/latest_user_agents.rb as the underling data structure 'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/sparklemotion/mechanize/)", 'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/121.0', From 553f6c11456eba269187aef470a2196dff4b0482 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 15 Jan 2024 13:11:54 -0500 Subject: [PATCH 37/72] version bump to v2.9.2 --- CHANGELOG.md | 6 ++++++ lib/mechanize/version.rb | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f754178f..e44200dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Mechanize CHANGELOG +## 2.9.2 / 2024-01-15 + +* Correct spelling errors in documentation. (#631) @p-linnane +* Updated User-Agent strings to represent modern browser versions. (#632) @takatea + + ## 2.9.1 / 2023-04-17 ### Update diff --git a/lib/mechanize/version.rb b/lib/mechanize/version.rb index db25d2ab..343f0eae 100644 --- a/lib/mechanize/version.rb +++ b/lib/mechanize/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true class Mechanize - VERSION = "2.9.1" + VERSION = "2.9.2" end From 9f85f2d939f1f4afab44b7f126ad45b61182253a Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 22 Jan 2024 12:45:35 -0500 Subject: [PATCH 38/72] dep: ruby 3.4 unbundled gems that ruby 3.3 is warning about. - nkf is used directly by mechanize - base64 is used by rubyntlm but that gem does not look maintained --- mechanize.gemspec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mechanize.gemspec b/mechanize.gemspec index 80cf3d92..004d35d5 100644 --- a/mechanize.gemspec +++ b/mechanize.gemspec @@ -65,10 +65,13 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency("net-http-persistent", ">= 2.5.2", "< 5.0.dev") spec.add_runtime_dependency("nokogiri", ">= 1.11.2", "~> 1.11") - spec.add_runtime_dependency("rubyntlm", ">= 0.6.3", "~> 0.6") spec.add_runtime_dependency("webrick", "~> 1.7") spec.add_runtime_dependency("webrobots", "~> 0.1.2") + spec.add_runtime_dependency("rubyntlm", ">= 0.6.3", "~> 0.6") + spec.add_runtime_dependency("base64") # removed from bundled gems in 3.4, and needed by rubyntlm (which doesn't declare this dependency) + spec.add_runtime_dependency("nkf") # removed from bundled gems in 3.4 + spec.add_development_dependency("minitest", "~> 5.14") spec.add_development_dependency("rake", "~> 13.0") spec.add_development_dependency("rdoc", "~> 6.3") From acb36b81c9add88f977352809fef694546f1f8be Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 22 Jan 2024 12:48:51 -0500 Subject: [PATCH 39/72] ci: add truffleruby (stable) and jruby-head --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 37103b0c..d2e583de 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3", "head", "jruby-9.4", "truffleruby-head"] + ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3", "jruby-9.4", "truffleruby", "head", "jruby-head", "truffleruby-head"] runs-on: ubuntu-latest steps: From 8c47e353b0dba0c36f937bfa8db5e61081ffcfbf Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 22 Jan 2024 12:57:07 -0500 Subject: [PATCH 40/72] version bump to v2.10.0 --- CHANGELOG.md | 5 +++++ lib/mechanize/version.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e44200dc..20b0a9fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Mechanize CHANGELOG +## 2.10.0 / 2024-01-22 + +* Add `nkf` and `base64` as explicit dependencies, since they are being unbundled in Ruby 3.4. (#634) @flavorjones + + ## 2.9.2 / 2024-01-15 * Correct spelling errors in documentation. (#631) @p-linnane diff --git a/lib/mechanize/version.rb b/lib/mechanize/version.rb index 343f0eae..442e458a 100644 --- a/lib/mechanize/version.rb +++ b/lib/mechanize/version.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true class Mechanize - VERSION = "2.9.2" + VERSION = "2.10.0" end From 9de5c1dd875a0def80f5386a88656bfcda9fcfd8 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 21 Feb 2024 14:14:31 -0500 Subject: [PATCH 41/72] test: backfill coverage on Cookie#set_domain deprecation message so that we have test coverage for both class and instance methods' deprecation messages. --- test/test_mechanize_cookie.rb | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/test_mechanize_cookie.rb b/test/test_mechanize_cookie.rb index fcc6606e..bb344d35 100644 --- a/test/test_mechanize_cookie.rb +++ b/test/test_mechanize_cookie.rb @@ -502,12 +502,23 @@ def test_domain= cookie.domain = 'Dom.example.com' assert 'dom.example.com', cookie.domain - cookie.domain = Object.new.tap { |o| + new_domain = Object.new.tap { |o| def o.to_str 'Example.com' end } + cookie.domain = new_domain assert 'example.com', cookie.domain + + new_domain = Object.new.tap { |o| + def o.to_str + 'Example2.com' + end + } + assert_output nil, /The call of Mechanize::Cookie#set_domain/ do + cookie.set_domain(new_domain) + end + assert 'example2.com', cookie.domain end def test_cookie_httponly From 73b8a7e9d1db104abf8516fe264c4a017f6c3e6b Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 21 Feb 2024 14:15:32 -0500 Subject: [PATCH 42/72] prefer caller_locations and Thread::Backtrace::Location methods Mechanize has still been using `Kernel#caller` whose string representation has changed enough in Ruby 3.4.0.dev to break how deprecation messages were being handrolled. Replace all uses of `caller` with `caller_locations` which is both faster and returns an object with some nice methods we can use to isolate the method, the file, and the line number. This should get us green again on ruby-head. --- lib/mechanize.rb | 4 ++-- lib/mechanize/cookie.rb | 4 ++-- test/test_mechanize_http_agent.rb | 2 +- test/test_mechanize_page_link.rb | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/mechanize.rb b/lib/mechanize.rb index 17765051..4e8d0341 100644 --- a/lib/mechanize.rb +++ b/lib/mechanize.rb @@ -717,10 +717,10 @@ class << self # authentication. def auth user, password, domain = nil - caller.first =~ /(.*?):(\d+).*?$/ + c = caller_locations(1,1).first warn <<-WARNING -At #{$1} line #{$2} +At #{c.absolute_path} line #{c.lineno} Use of #auth and #basic_auth are deprecated due to a security vulnerability. diff --git a/lib/mechanize/cookie.rb b/lib/mechanize/cookie.rb index 2d001e80..fe7eb366 100644 --- a/lib/mechanize/cookie.rb +++ b/lib/mechanize/cookie.rb @@ -7,7 +7,7 @@ class Mechanize module CookieDeprecated def __deprecated__(to = nil) $VERBOSE or return - method = caller[0][/([^`]+)(?='$)/] + method = caller_locations(1,1).first.base_label to ||= method case self when Class @@ -21,7 +21,7 @@ def __deprecated__(to = nil) this = '%s#%s' % [klass, method] that = 'HTTP::%s#%s' % [lname, to] end - warn '%s: The call of %s needs to be fixed to follow the new API (%s).' % [caller[1], this, that] + warn '%s: The call of %s needs to be fixed to follow the new API (%s).' % [caller_locations(2,1).first, this, that] end private :__deprecated__ end diff --git a/test/test_mechanize_http_agent.rb b/test/test_mechanize_http_agent.rb index f2fea1e7..0c327868 100644 --- a/test/test_mechanize_http_agent.rb +++ b/test/test_mechanize_http_agent.rb @@ -29,7 +29,7 @@ def auth_realm uri, scheme, type def skip_if_jruby_zlib if RUBY_ENGINE == 'jruby' - meth = caller[0][/`(\w+)/, 1] + meth = caller_locations(1,1).first.base_label skip "#{meth}: skipped because how Zlib handles error is different in JRuby" end end diff --git a/test/test_mechanize_page_link.rb b/test/test_mechanize_page_link.rb index 520a13c1..1be78fcf 100644 --- a/test/test_mechanize_page_link.rb +++ b/test/test_mechanize_page_link.rb @@ -55,7 +55,7 @@ def util_page body = @body, res = @res def skip_if_nkf_dependency if RUBY_ENGINE == 'jruby' - meth = caller[0][/`(\w+)/, 1] + meth = caller_locations(1,1).first.base_label skip "#{meth}: skipped because this feature currently depends on NKF" end end From 4a110852f7ba96ff0b55e3817e962cce0c8ab0d0 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sat, 23 Mar 2024 16:37:11 -0400 Subject: [PATCH 43/72] ci: skeleton upstream.yml file to which I will PR additions shortly --- .github/workflows/upstream.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/upstream.yml diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml new file mode 100644 index 00000000..e37f025a --- /dev/null +++ b/.github/workflows/upstream.yml @@ -0,0 +1,25 @@ +name: "ci" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + cancel-in-progress: true + +on: + workflow_dispatch: + schedule: + - cron: "0 8 * * 5" # At 08:00 on Friday # https://crontab.guru/#0_8_*_*_5 + push: + branches: + - main + pull_request: + types: [opened, synchronize] + branches: + - main + paths: + - .github/workflows/upstream.yml # this file + +jobs: + skeleton: + runs-on: ubuntu-latest + steps: + - run: echo "Hello, World!" From f89dc844c48dcb2bccdac81e1b08ef3455f8b2d6 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sat, 23 Mar 2024 16:22:12 -0400 Subject: [PATCH 44/72] ci: rename pipeline, update versions, drop windows and macos - rename pipeline to "ci.yml" - update to ruby 3.3 for rubocop - update actions/checkout to v4 - remove the no-longer-necessary pinning of bundler - drop windows and macos tests because this gem doesn't need it --- .github/workflows/{ci-test.yml => ci.yml} | 24 +++-------------------- 1 file changed, 3 insertions(+), 21 deletions(-) rename .github/workflows/{ci-test.yml => ci.yml} (62%) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci.yml similarity index 62% rename from .github/workflows/ci-test.yml rename to .github/workflows/ci.yml index d2e583de..45d8b5c0 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,10 @@ jobs: rubocop: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "3.2" + ruby-version: "3.3" bundler-cache: true - run: bundle exec rake rubocop @@ -36,27 +36,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{matrix.ruby-version}} bundler-cache: true - bundler: 2.3.26 # https://github.com/rubygems/rubygems/issues/6435 - - run: bundle exec rake test - - test-platform: - needs: ["rubocop"] - strategy: - fail-fast: false - matrix: - platform: ["windows-latest", "macos-latest"] - - runs-on: ${{matrix.platform}} - steps: - - uses: actions/checkout@v3 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.2" - bundler-cache: true - bundler: latest - run: bundle exec rake test From 3cff941230ef6bcaad4bddd06446e26b64ecbbd4 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sat, 23 Mar 2024 16:36:06 -0400 Subject: [PATCH 45/72] ci: extract upstream.yml pipeline to test edge rubies --- .github/workflows/ci.yml | 3 +-- .github/workflows/upstream.yml | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45d8b5c0..efb79c2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,8 +32,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3", "jruby-9.4", "truffleruby", "head", "jruby-head", "truffleruby-head"] - + ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3", "jruby-9.4", "truffleruby"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index e37f025a..b1d0f2ff 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -1,4 +1,4 @@ -name: "ci" +name: "upstream" concurrency: group: "${{github.workflow}}-${{github.ref}}" @@ -19,7 +19,16 @@ on: - .github/workflows/upstream.yml # this file jobs: - skeleton: + test: + strategy: + fail-fast: false + matrix: + ruby-version: ["head", "jruby-head", "truffleruby-head"] runs-on: ubuntu-latest steps: - - run: echo "Hello, World!" + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{matrix.ruby-version}} + bundler-cache: true + - run: bundle exec rake test From 9e72346881b4400a57a9fbeb61bca370fce67773 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sat, 23 Mar 2024 16:50:04 -0400 Subject: [PATCH 46/72] ci: add upstream nokogiri integration test --- .github/workflows/upstream.yml | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index b1d0f2ff..8a6ee609 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -26,9 +26,26 @@ jobs: ruby-version: ["head", "jruby-head", "truffleruby-head"] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{matrix.ruby-version}} - bundler-cache: true - - run: bundle exec rake test + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{matrix.ruby-version}} + bundler-cache: true + - run: bundle exec rake test + + upstream: + name: "upstream (${{matrix.name}})" + strategy: + fail-fast: false + matrix: + include: + - { name: "nokogiri", git: "https://github.com/sparklemotion/nokogiri" } + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: { ruby-version: "3.3" } + - run: | + bundle add ${{matrix.name}} --git="${{matrix.git}}" + bundle show + - run: bundle exec rake test From 608f729a774360d92054dbb7fbc3f8cfd66ec4a0 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 24 Mar 2024 11:49:37 -0400 Subject: [PATCH 47/72] test: update tests to use frozen string literals so that ruby 3.4 doesn't emit so many warnings. --- Rakefile | 2 +- test/test_mechanize.rb | 3 ++- test/test_mechanize_cookie.rb | 13 ++++++----- test/test_mechanize_cookie_jar.rb | 1 + test/test_mechanize_directory_saver.rb | 1 + test/test_mechanize_download.rb | 1 + .../test_mechanize_element_not_found_error.rb | 1 + test/test_mechanize_file.rb | 1 + test/test_mechanize_file_connection.rb | 3 ++- test/test_mechanize_file_request.rb | 1 + test/test_mechanize_file_response.rb | 1 + test/test_mechanize_file_saver.rb | 1 + test/test_mechanize_form.rb | 1 + test/test_mechanize_form_check_box.rb | 1 + test/test_mechanize_form_encoding.rb | 3 ++- test/test_mechanize_form_field.rb | 1 + test/test_mechanize_form_file_upload.rb | 1 + test/test_mechanize_form_image_button.rb | 1 + test/test_mechanize_form_keygen.rb | 1 + test/test_mechanize_form_multi_select_list.rb | 1 + test/test_mechanize_form_option.rb | 1 + test/test_mechanize_form_radio_button.rb | 1 + test/test_mechanize_form_select_list.rb | 1 + test/test_mechanize_form_textarea.rb | 1 + test/test_mechanize_headers.rb | 1 + test/test_mechanize_history.rb | 1 + test/test_mechanize_http_agent.rb | 7 +++--- test/test_mechanize_http_auth_challenge.rb | 1 + test/test_mechanize_http_auth_realm.rb | 1 + test/test_mechanize_http_auth_store.rb | 1 + ...chanize_http_content_disposition_parser.rb | 1 + ..._mechanize_http_www_authenticate_parser.rb | 1 + test/test_mechanize_image.rb | 1 + test/test_mechanize_link.rb | 1 + test/test_mechanize_page.rb | 1 + test/test_mechanize_page_encoding.rb | 11 +++++---- test/test_mechanize_page_frame.rb | 1 + test/test_mechanize_page_image.rb | 1 + test/test_mechanize_page_link.rb | 23 +++++++++---------- test/test_mechanize_page_meta_refresh.rb | 1 + test/test_mechanize_parser.rb | 1 + test/test_mechanize_pluggable_parser.rb | 1 + ..._mechanize_redirect_limit_reached_error.rb | 1 + ...echanize_redirect_not_get_or_head_error.rb | 1 + test/test_mechanize_response_read_error.rb | 1 + test/test_mechanize_subclass.rb | 1 + test/test_mechanize_util.rb | 7 +++--- test/test_mechanize_xml_file.rb | 1 + test/test_multi_select.rb | 1 + 49 files changed, 79 insertions(+), 33 deletions(-) diff --git a/Rakefile b/Rakefile index 3e2c4d30..37381990 100644 --- a/Rakefile +++ b/Rakefile @@ -49,7 +49,7 @@ namespace "rubocop" do desc "Run rubocop string literals check" task :frozen_string_literals do - sh "rubocop lib --auto-correct-all --only Style/FrozenStringLiteralComment" + sh "rubocop lib test --auto-correct-all --only Style/FrozenStringLiteralComment" end end diff --git a/test/test_mechanize.rb b/test/test_mechanize.rb index 310d7cca..1932e5d3 100644 --- a/test/test_mechanize.rb +++ b/test/test_mechanize.rb @@ -1,4 +1,5 @@ # coding: utf-8 +# frozen_string_literal: true require 'mechanize/test_case' @@ -965,7 +966,7 @@ def test_post_file_upload_nonascii }) assert_match( - "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"".force_encoding(Encoding::ASCII_8BIT), + "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"".dup.force_encoding(Encoding::ASCII_8BIT), page.body ) assert_match("Content-Type: application/zip", page.body) diff --git a/test/test_mechanize_cookie.rb b/test/test_mechanize_cookie.rb index bb344d35..5b63ad40 100644 --- a/test/test_mechanize_cookie.rb +++ b/test/test_mechanize_cookie.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' module Enumerable @@ -301,7 +302,7 @@ def test_parse_valid_cookie expires = Time.parse('Sun, 27-Sep-2037 00:00:00 GMT') cookie_params.keys.combine.each do |c| - cookie_text = "#{cookie_value}; " + cookie_text = +"#{cookie_value}; " c.each_with_index do |key, idx| if idx == (c.length - 1) cookie_text << "#{cookie_params[key]}" @@ -336,7 +337,7 @@ def test_parse_valid_cookie_empty_value expires = Time.parse('Sun, 27-Sep-2037 00:00:00 GMT') cookie_params.keys.combine.each do |c| - cookie_text = "#{cookie_value}; " + cookie_text = +"#{cookie_value}; " c.each_with_index do |key, idx| if idx == (c.length - 1) cookie_text << "#{cookie_params[key]}" @@ -373,7 +374,7 @@ def test_cookie_using_url_path cookie_params.keys.combine.each do |c| next if c.find { |k| k == 'path' } - cookie_text = "#{cookie_value}; " + cookie_text = +"#{cookie_value}; " c.each_with_index do |key, idx| if idx == (c.length - 1) cookie_text << "#{cookie_params[key]}" @@ -410,7 +411,7 @@ def test_cookie_with_secure cookie_params.keys.combine.each do |c| next unless c.find { |k| k == 'secure' } - cookie_text = "#{cookie_value}; " + cookie_text = +"#{cookie_value}; " c.each_with_index do |key, idx| if idx == (c.length - 1) cookie_text << "#{cookie_params[key]}" @@ -446,7 +447,7 @@ def test_parse_cookie_no_spaces expires = Time.parse('Sun, 27-Sep-2037 00:00:00 GMT') cookie_params.keys.combine.each do |c| - cookie_text = "#{cookie_value};" + cookie_text = +"#{cookie_value};" c.each_with_index do |key, idx| if idx == (c.length - 1) cookie_text << "#{cookie_params[key]}" @@ -530,7 +531,7 @@ def test_cookie_httponly expires = Time.parse('Sun, 27-Sep-2037 00:00:00 GMT') cookie_params.keys.combine.each do |c| - cookie_text = "#{cookie_value}; " + cookie_text = +"#{cookie_value}; " c.each_with_index do |key, idx| if idx == (c.length - 1) cookie_text << "#{cookie_params[key]}" diff --git a/test/test_mechanize_cookie_jar.rb b/test/test_mechanize_cookie_jar.rb index 2af06dfe..8371c55f 100644 --- a/test/test_mechanize_cookie_jar.rb +++ b/test/test_mechanize_cookie_jar.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' require 'fileutils' diff --git a/test/test_mechanize_directory_saver.rb b/test/test_mechanize_directory_saver.rb index f4835bf0..be3d69ce 100644 --- a/test/test_mechanize_directory_saver.rb +++ b/test/test_mechanize_directory_saver.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeDirectorySaver < Mechanize::TestCase diff --git a/test/test_mechanize_download.rb b/test/test_mechanize_download.rb index b9516dfd..5622d283 100644 --- a/test/test_mechanize_download.rb +++ b/test/test_mechanize_download.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeDownload < Mechanize::TestCase diff --git a/test/test_mechanize_element_not_found_error.rb b/test/test_mechanize_element_not_found_error.rb index e527d9d4..fab3e7dc 100644 --- a/test/test_mechanize_element_not_found_error.rb +++ b/test/test_mechanize_element_not_found_error.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeRedirectLimitReachedError < Mechanize::TestCase diff --git a/test/test_mechanize_file.rb b/test/test_mechanize_file.rb index fd22c83d..5c9676fe 100644 --- a/test/test_mechanize_file.rb +++ b/test/test_mechanize_file.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFile < Mechanize::TestCase diff --git a/test/test_mechanize_file_connection.rb b/test/test_mechanize_file_connection.rb index 4306a0ad..ef555516 100644 --- a/test/test_mechanize_file_connection.rb +++ b/test/test_mechanize_file_connection.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFileConnection < Mechanize::TestCase @@ -7,7 +8,7 @@ def test_request uri = URI.parse "file://#{file_path}" conn = Mechanize::FileConnection.new - body = '' + body = +'' conn.request uri, nil do |response| assert_equal(file_path, response.file_path) diff --git a/test/test_mechanize_file_request.rb b/test/test_mechanize_file_request.rb index e5d3e1a3..a072c462 100644 --- a/test/test_mechanize_file_request.rb +++ b/test/test_mechanize_file_request.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFileRequest < Mechanize::TestCase diff --git a/test/test_mechanize_file_response.rb b/test/test_mechanize_file_response.rb index 3aa20741..9a7c6f31 100644 --- a/test/test_mechanize_file_response.rb +++ b/test/test_mechanize_file_response.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFileResponse < Mechanize::TestCase diff --git a/test/test_mechanize_file_saver.rb b/test/test_mechanize_file_saver.rb index 3066822c..e1905cf1 100644 --- a/test/test_mechanize_file_saver.rb +++ b/test/test_mechanize_file_saver.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFileSaver < Mechanize::TestCase diff --git a/test/test_mechanize_form.rb b/test/test_mechanize_form.rb index b6247ddc..a4e4b37d 100644 --- a/test/test_mechanize_form.rb +++ b/test/test_mechanize_form.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeForm < Mechanize::TestCase diff --git a/test/test_mechanize_form_check_box.rb b/test/test_mechanize_form_check_box.rb index f11eb28d..43420f7d 100644 --- a/test/test_mechanize_form_check_box.rb +++ b/test/test_mechanize_form_check_box.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormCheckBox < Mechanize::TestCase diff --git a/test/test_mechanize_form_encoding.rb b/test/test_mechanize_form_encoding.rb index b8cadc56..d701bebf 100644 --- a/test/test_mechanize_form_encoding.rb +++ b/test/test_mechanize_form_encoding.rb @@ -1,4 +1,5 @@ # coding: utf-8 +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormEncoding < Mechanize::TestCase @@ -8,7 +9,7 @@ class TestMechanizeFormEncoding < Mechanize::TestCase INPUTTED_VALUE = "テスト" # "test" in Japanese UTF-8 encoding CONTENT_ENCODING = 'Shift_JIS' # one of Japanese encoding - encoded_value = "\x83\x65\x83\x58\x83\x67".force_encoding(::Encoding::SHIFT_JIS) # "test" in Japanese Shift_JIS encoding + encoded_value = "\x83\x65\x83\x58\x83\x67".dup.force_encoding(::Encoding::SHIFT_JIS) # "test" in Japanese Shift_JIS encoding EXPECTED_QUERY = "first_name=#{CGI.escape(encoded_value)}&first_name=&gender=&green%5Beggs%5D=" ENCODING_ERRORS = [EncodingError, Encoding::ConverterNotFoundError] # and so on diff --git a/test/test_mechanize_form_field.rb b/test/test_mechanize_form_field.rb index 8fa17a19..14ece52c 100644 --- a/test/test_mechanize_form_field.rb +++ b/test/test_mechanize_form_field.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormField < Mechanize::TestCase diff --git a/test/test_mechanize_form_file_upload.rb b/test/test_mechanize_form_file_upload.rb index a244f4fb..3a92c6d5 100644 --- a/test/test_mechanize_form_file_upload.rb +++ b/test/test_mechanize_form_file_upload.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormFileUpload < Mechanize::TestCase diff --git a/test/test_mechanize_form_image_button.rb b/test/test_mechanize_form_image_button.rb index 679ad270..4d48c3ab 100644 --- a/test/test_mechanize_form_image_button.rb +++ b/test/test_mechanize_form_image_button.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormImageButton < Mechanize::TestCase diff --git a/test/test_mechanize_form_keygen.rb b/test/test_mechanize_form_keygen.rb index 4eb17f0c..1e1f3024 100644 --- a/test/test_mechanize_form_keygen.rb +++ b/test/test_mechanize_form_keygen.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormKeygen < Mechanize::TestCase diff --git a/test/test_mechanize_form_multi_select_list.rb b/test/test_mechanize_form_multi_select_list.rb index 27fb8655..c479d63f 100644 --- a/test/test_mechanize_form_multi_select_list.rb +++ b/test/test_mechanize_form_multi_select_list.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormMultiSelectList < Mechanize::TestCase diff --git a/test/test_mechanize_form_option.rb b/test/test_mechanize_form_option.rb index 75ca6866..bd56a7be 100644 --- a/test/test_mechanize_form_option.rb +++ b/test/test_mechanize_form_option.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormOption < Mechanize::TestCase diff --git a/test/test_mechanize_form_radio_button.rb b/test/test_mechanize_form_radio_button.rb index 23872535..892a4ebd 100644 --- a/test/test_mechanize_form_radio_button.rb +++ b/test/test_mechanize_form_radio_button.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormRadioButton < Mechanize::TestCase diff --git a/test/test_mechanize_form_select_list.rb b/test/test_mechanize_form_select_list.rb index 1a61984e..80c17b65 100644 --- a/test/test_mechanize_form_select_list.rb +++ b/test/test_mechanize_form_select_list.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormSelectList < Mechanize::TestCase diff --git a/test/test_mechanize_form_textarea.rb b/test/test_mechanize_form_textarea.rb index 10599550..0783b02a 100644 --- a/test/test_mechanize_form_textarea.rb +++ b/test/test_mechanize_form_textarea.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeFormTextarea < Mechanize::TestCase diff --git a/test/test_mechanize_headers.rb b/test/test_mechanize_headers.rb index c951b0bd..f08d1211 100644 --- a/test/test_mechanize_headers.rb +++ b/test/test_mechanize_headers.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeHeaders < Mechanize::TestCase diff --git a/test/test_mechanize_history.rb b/test/test_mechanize_history.rb index a98b3f7b..02a8f6a1 100644 --- a/test/test_mechanize_history.rb +++ b/test/test_mechanize_history.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeHistory < Mechanize::TestCase diff --git a/test/test_mechanize_http_agent.rb b/test/test_mechanize_http_agent.rb index 0c327868..643c2369 100644 --- a/test/test_mechanize_http_agent.rb +++ b/test/test_mechanize_http_agent.rb @@ -1,4 +1,5 @@ # coding: utf-8 +# frozen_string_literal: true require 'mechanize/test_case' @@ -1027,7 +1028,7 @@ def test_response_content_encoding_gzip_encoding_bad body = @agent.response_content_encoding @res, body_io - expected = "test\xB2" + expected = +"test\xB2" expected.force_encoding Encoding::BINARY if have_encoding? content = body.read @@ -1432,11 +1433,11 @@ def test_response_read_file io = @agent.response_read res, req, uri - expected = "π\n".force_encoding(Encoding::BINARY) + expected = "π\n".dup.force_encoding(Encoding::BINARY) # Ruby 1.8.7 doesn't let us set the write mode of the tempfile to binary, # so we should expect an inserted carriage return on some platforms - expected_with_carriage_return = "π\r\n".force_encoding(Encoding::BINARY) + expected_with_carriage_return = "π\r\n".dup.force_encoding(Encoding::BINARY) body = io.read assert_match(/^(#{expected}|#{expected_with_carriage_return})$/m, body) diff --git a/test/test_mechanize_http_auth_challenge.rb b/test/test_mechanize_http_auth_challenge.rb index dca7e4c1..3774335e 100644 --- a/test/test_mechanize_http_auth_challenge.rb +++ b/test/test_mechanize_http_auth_challenge.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeHttpAuthChallenge < Mechanize::TestCase diff --git a/test/test_mechanize_http_auth_realm.rb b/test/test_mechanize_http_auth_realm.rb index 97219909..91ce93af 100644 --- a/test/test_mechanize_http_auth_realm.rb +++ b/test/test_mechanize_http_auth_realm.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeHttpAuthRealm < Mechanize::TestCase diff --git a/test/test_mechanize_http_auth_store.rb b/test/test_mechanize_http_auth_store.rb index b7410bf3..66bd8e53 100644 --- a/test/test_mechanize_http_auth_store.rb +++ b/test/test_mechanize_http_auth_store.rb @@ -1,4 +1,5 @@ # coding: utf-8 +# frozen_string_literal: true require 'mechanize/test_case' diff --git a/test/test_mechanize_http_content_disposition_parser.rb b/test/test_mechanize_http_content_disposition_parser.rb index c66a0420..565776d0 100644 --- a/test/test_mechanize_http_content_disposition_parser.rb +++ b/test/test_mechanize_http_content_disposition_parser.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeHttpContentDispositionParser < Mechanize::TestCase diff --git a/test/test_mechanize_http_www_authenticate_parser.rb b/test/test_mechanize_http_www_authenticate_parser.rb index 09d27018..b8e1e046 100644 --- a/test/test_mechanize_http_www_authenticate_parser.rb +++ b/test/test_mechanize_http_www_authenticate_parser.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeHttpWwwAuthenticateParser < Mechanize::TestCase diff --git a/test/test_mechanize_image.rb b/test/test_mechanize_image.rb index d4bc71d0..d7e5edce 100644 --- a/test/test_mechanize_image.rb +++ b/test/test_mechanize_image.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeImage < Mechanize::TestCase diff --git a/test/test_mechanize_link.rb b/test/test_mechanize_link.rb index 1c044ece..5b2e8bae 100644 --- a/test/test_mechanize_link.rb +++ b/test/test_mechanize_link.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizeLink < Mechanize::TestCase diff --git a/test/test_mechanize_page.rb b/test/test_mechanize_page.rb index 014cb50b..1d97bd68 100644 --- a/test/test_mechanize_page.rb +++ b/test/test_mechanize_page.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true require 'mechanize/test_case' class TestMechanizePage < Mechanize::TestCase diff --git a/test/test_mechanize_page_encoding.rb b/test/test_mechanize_page_encoding.rb index 7dc80cf9..e250a2b3 100644 --- a/test/test_mechanize_page_encoding.rb +++ b/test/test_mechanize_page_encoding.rb @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# frozen_string_literal: true require 'mechanize/test_case' # tests for Page encoding and charset and parsing @@ -12,7 +13,7 @@ def setup @uri = URI('http://localhost/') @response_headers = { 'content-type' => 'text/html' } - @body = 'hi' + @body = +'hi' end def util_page body = @body, headers = @response_headers @@ -118,7 +119,7 @@ def test_page_meta_charset_handles_whitespace end def test_meta_charset - body = '' + body = +'' page = util_page body assert_equal ['META'], page.meta_charset @@ -132,7 +133,7 @@ def test_detected_encoding def test_encodings response = {'content-type' => 'text/html;charset=HEADER'} - body = '' + body = +'' @mech.default_encoding = 'DEFAULT' page = util_page body, response @@ -175,7 +176,7 @@ def test_parser_encoding_equals_overwrites_force_default_encoding def test_parser_encoding_when_searching_elements skip "Encoding not implemented" unless have_encoding? - body = 'hi' + body = +'hi' page = util_page body, 'content-type' => 'text/html,charset=ISO-8859-1' result = page.search('#latin1') @@ -187,7 +188,7 @@ def test_parser_error_message_containing_encoding_errors skip if RUBY_ENGINE == 'jruby' # this is a libxml2-specific condition # https://github.com/sparklemotion/mechanize/issues/553 - body = <<~EOF + body = +<<~EOF