From 38bd1fd918c611cb4d7d977f26c691759895e839 Mon Sep 17 00:00:00 2001 From: Keith Layne Date: Wed, 6 Mar 2013 11:19:27 -0500 Subject: [PATCH 1/2] Remove password from mysql command string. This is one possible fix for #129 --- lib/etl/control/source/mysql_streamer.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/etl/control/source/mysql_streamer.rb b/lib/etl/control/source/mysql_streamer.rb index 5d12160..522eb5e 100644 --- a/lib/etl/control/source/mysql_streamer.rb +++ b/lib/etl/control/source/mysql_streamer.rb @@ -49,8 +49,11 @@ def each database = mandatory_option!(config, 'database') password = config['password'] # this one can omitted in some cases - mysql_command = """mysql --quick -h #{host} -u #{username} -e \"#{@query.gsub("\n","")}\" -D #{database} --password=#{password} -B""" + mysql_command = """mysql --quick -h #{host} -u #{username} -e \"#{@query.gsub("\n","")}\" -D #{database} --password -B""" Open3.popen3(mysql_command) do |stdin, out, err, external| + err.gets + stdin.puts password + stdin.close until (line = out.gets).nil? do line = line.gsub("\n","") if keys.nil? @@ -70,4 +73,4 @@ def each end end end -end \ No newline at end of file +end From 06c355ad9dfb1de55ba7ee8790c8f1c034a52fc5 Mon Sep 17 00:00:00 2001 From: Keith Layne Date: Thu, 7 Mar 2013 11:18:09 -0500 Subject: [PATCH 2/2] Put all mysql config into a temporary option file. This is (I think) a better approach towards dealing with #129. I have tested it a little bit, and I think it may provide a basis for dealing with #108. One plus is that it doesn't have to read anything magical from stderr. The Tempfile docs say that the file is created 0600. I think I read that MySQL ignores option files that are not 0600 or 0400. I would rather unlink the option file sooner, but I'm not sure how to do that without creating a race condition. I ran 'DB=mysql2 rake test' using ruby-1.9.3p392 and the only failures I got had to do with with nokogiri. I tested a little on my own use case, just to make sure it looked like it was working. --- lib/etl/control/source/mysql_streamer.rb | 35 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/lib/etl/control/source/mysql_streamer.rb b/lib/etl/control/source/mysql_streamer.rb index 522eb5e..b6dd72b 100644 --- a/lib/etl/control/source/mysql_streamer.rb +++ b/lib/etl/control/source/mysql_streamer.rb @@ -1,4 +1,5 @@ require 'open3' +require 'tempfile' # Internal: The MySQL streamer is a helper with works with the database_source # in order to allow you to use the --quick option (which stops MySQL) @@ -42,18 +43,7 @@ def mandatory_option!(hash, key) def each keys = nil - - config = ETL::Base.configurations[@name.to_s] - host = mandatory_option!(config, 'host') - username = mandatory_option!(config, 'username') - database = mandatory_option!(config, 'database') - password = config['password'] # this one can omitted in some cases - - mysql_command = """mysql --quick -h #{host} -u #{username} -e \"#{@query.gsub("\n","")}\" -D #{database} --password -B""" - Open3.popen3(mysql_command) do |stdin, out, err, external| - err.gets - stdin.puts password - stdin.close + run_mysql(:quick, :batch) do |stdin, out, err, external| until (line = out.gets).nil? do line = line.gsub("\n","") if keys.nil? @@ -73,4 +63,25 @@ def each end end end + + private + + def run_mysql(*args) + config = ETL::Base.configurations[@name.to_s] + options = args.extract_options!.merge( + 'host' => mandatory_option!(config, 'host'), + 'user' => mandatory_option!(config, 'username'), + 'database' => mandatory_option!(config, 'database'), + 'password' => config['password'], + 'execute' => "\"#{@query}\"") + + Tempfile.open('mysqlstreamer') do |option_file| + option_file.puts '[client]' + args.each {|keyword| option_file.puts keyword} + options.each {|key, value| option_file.puts "#{key}=#{value}"} + option_file.flush + + yield Open3.popen3("mysql --defaults-extra-file=#{option_file.path}") + end + end end