diff --git a/bolt.gemspec b/bolt.gemspec index 6d7f5a88d6..fa7c080785 100644 --- a/bolt.gemspec +++ b/bolt.gemspec @@ -40,6 +40,7 @@ Gem::Specification.new do |spec| spec.add_dependency "net-ssh", ">= 4.0" spec.add_dependency "orchestrator_client", "~> 0.3.1" spec.add_dependency "puppet", [">= 6.0.1", "< 7"] + spec.add_dependency "puppet-resource_api" spec.add_dependency "r10k", "~> 2.6" spec.add_dependency "terminal-table", "~> 1.8" spec.add_dependency "winrm", "~> 2.0" diff --git a/lib/bolt/config.rb b/lib/bolt/config.rb index 99e1243485..9a5a0cb4dd 100644 --- a/lib/bolt/config.rb +++ b/lib/bolt/config.rb @@ -10,6 +10,7 @@ require 'bolt/transport/orch' require 'bolt/transport/local' require 'bolt/transport/docker' +require 'bolt/transport/remote' module Bolt TRANSPORTS = { @@ -17,7 +18,8 @@ module Bolt winrm: Bolt::Transport::WinRM, pcp: Bolt::Transport::Orch, local: Bolt::Transport::Local, - docker: Bolt::Transport::Docker + docker: Bolt::Transport::Docker, + remote: Bolt::Transport::Remote, }.freeze class UnknownTransportError < Bolt::Error @@ -41,6 +43,7 @@ class Config 'tty' => false }.freeze + # TODO: move these to the transport themselves TRANSPORT_SPECIFIC_DEFAULTS = { ssh: { 'host-key-check' => true @@ -53,7 +56,10 @@ class Config 'task-environment' => 'production' }, local: {}, - docker: {} + docker: {}, + remote: { + 'run-on' => 'localhost' + } }.freeze def self.default diff --git a/lib/bolt/executor.rb b/lib/bolt/executor.rb index c4456ecb5d..9d73a703db 100644 --- a/lib/bolt/executor.rb +++ b/lib/bolt/executor.rb @@ -32,8 +32,14 @@ def initialize(concurrency = 1, @load_config = load_config @transports = Bolt::TRANSPORTS.each_with_object({}) do |(key, val), coll| - coll[key.to_s] = Concurrent::Delay.new do - val.new + if key == :remote + coll[key.to_s] = Concurrent::Delay.new do + val.new(self) + end + else + coll[key.to_s] = Concurrent::Delay.new do + val.new + end end end @reported_transports = Set.new diff --git a/lib/bolt/inventory.rb b/lib/bolt/inventory.rb index f2808bd536..96da0df520 100644 --- a/lib/bolt/inventory.rb +++ b/lib/bolt/inventory.rb @@ -136,6 +136,17 @@ def data_hash } end + # TODO: This does two things because the applicator bypasses run_task + # It would probably be cleaner to give the exectutor access to inventory + # and handle this there. + def run_on_target(target, params) + if target.remote? + [get_target(target.run_on || 'localhost'), params.merge('_target' => target.hash)] + else + [target, params] + end + end + #### PRIVATE #### # # For debugging only now @@ -207,6 +218,7 @@ def resolve_name(target) private :resolve_name def expand_targets(targets) + # TODO: inventory may not be set in this case? if targets.is_a? Bolt::Target targets elsif targets.is_a? Array diff --git a/lib/bolt/node/errors.rb b/lib/bolt/node/errors.rb index 8cbad2eedd..949c6dbc6c 100644 --- a/lib/bolt/node/errors.rb +++ b/lib/bolt/node/errors.rb @@ -7,7 +7,8 @@ class Node class BaseError < Bolt::Error attr_reader :issue_code - def initialize(message, issue_code) + # TODO can we just drop issue code here? + def initialize(message, issue_code = nil) super(message, kind, nil, issue_code) end @@ -34,6 +35,12 @@ def kind end end + class RemoteError < BaseError + def kind + 'puppetlabs.tasks/remote-task-error' + end + end + class EnvironmentVarError < BaseError def initialize(var, val) message = "Could not set environment variable '#{var}' to '#{val}'" diff --git a/lib/bolt/target.rb b/lib/bolt/target.rb index 54098df21c..0cc66639f1 100644 --- a/lib/bolt/target.rb +++ b/lib/bolt/target.rb @@ -85,6 +85,18 @@ def to_s "Target('#{@uri}', #{opts})" end + def to_h + options.merge({ + 'name' => name, + 'uri' => uri, + 'protocol' => protocol, + 'user' => user, + 'password' => password, + 'host' => host, + 'port' => port, + }) + end + def host @uri_obj.hostname end diff --git a/lib/bolt/task.rb b/lib/bolt/task.rb index fb8a12458b..640522d835 100644 --- a/lib/bolt/task.rb +++ b/lib/bolt/task.rb @@ -36,6 +36,10 @@ def module_name name.split('::').first end + def remote? + !!metadata['remote'] + end + def tasks_dir File.join(module_name, 'tasks') end diff --git a/lib/bolt/transport/base.rb b/lib/bolt/transport/base.rb index 2df8d2dc18..f1a008392d 100644 --- a/lib/bolt/transport/base.rb +++ b/lib/bolt/transport/base.rb @@ -39,6 +39,7 @@ module Transport class Base STDIN_METHODS = %w[both stdin].freeze ENVIRONMENT_METHODS = %w[both environment].freeze + DEFAULT_INPUT_METHOD = 'both' attr_reader :logger @@ -68,6 +69,33 @@ def with_events(target, callback) result end + def provided_features + [] + end + + def default_input_method + 'both' + end + + def select_implementation(target, task) + impl =task.select_implementation(target, provided_features) + impl['input_method'] ||= default_input_method + impl + end + + def add_target_param(target, task, params) + # TODO: How should metadata impact this? + #if target.remote? && !task.remote? + # raise Bolt::Node::RemoteError.new('Remote targets can only run Remotable tasks') + #end + #if !target.remote? && task.remote? + # raise Bolt::Node::RemoteError.new('Remote tasks can only be run on remote targets') + #end + + # TODO: wrap sensitive values + target.remote? ? params.merge('_target' => target.to_h) : params + end + def filter_options(target, options) if target.options['run-as'] options.reject { |k, _v| k == '_run_as' } diff --git a/lib/bolt/transport/local.rb b/lib/bolt/transport/local.rb index 43de84211c..28587f0e71 100644 --- a/lib/bolt/transport/local.rb +++ b/lib/bolt/transport/local.rb @@ -13,7 +13,9 @@ def self.options %w[tmpdir] end - PROVIDED_FEATURES = ['shell'].freeze + def provided_features + ['shell'] + end def self.validate(_options); end @@ -82,9 +84,9 @@ def run_script(target, script, arguments, _options = {}) end def run_task(target, task, arguments, _options = {}) - implementation = task.select_implementation(target, PROVIDED_FEATURES) + implementation = select_implementation(target, task) executable = implementation['path'] - input_method = implementation['input_method'] || 'both' + input_method = implementation['input_method'] extra_files = implementation['files'] in_tmpdir(target.options['tmpdir']) do |dir| diff --git a/lib/bolt/transport/orch.rb b/lib/bolt/transport/orch.rb index 07084ac2b8..bf37a48926 100644 --- a/lib/bolt/transport/orch.rb +++ b/lib/bolt/transport/orch.rb @@ -29,7 +29,9 @@ def self.options %w[service-url cacert token-file task-environment] end - PROVIDED_FEATURES = ['puppet-agent'].freeze + def provided_features + ['puppet-agent'] + end def self.validate(options); end diff --git a/lib/bolt/transport/remote.rb b/lib/bolt/transport/remote.rb new file mode 100644 index 0000000000..25a4b9dbce --- /dev/null +++ b/lib/bolt/transport/remote.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require 'bolt/transport/base' + + +module Bolt + module Transport + class Remote < Base + def self.options + # TODO: We should accept arbitrary options here + %w[host port user password connnect-timeout device-type run-on] + end + + def self.validate(options) + # This will fail when validating global config + #unless options['device-type'] + # raise Bolt::ValidationError, 'Must specify device-type for devices' + #end + end + + # TODO: this should have access to inventory so target doesn't have to + def initialize(executor) + super() + + @executor = executor + end + + def get_proxy(target) + # TODO: This needs to have access to the inventory + inventory = target.instance_variable_get(:@inventory) + raise "Target was creates without inventory? Not get_targets?" unless inventory + inventory.get_targets(target.options['run-on'] || 'localhost').first + end + + # Cannot batch because arugments differ + def run_task(target, task, arguments, options = {}) + proxy_target = get_proxy(target) + transport = @executor.transport(proxy_target.protocol) + arguments = arguments.merge('_target' => target.to_h.reject {|_, v| v.nil?}) + + # TODO: add support for device-type and feature checking here. + # * tasks/task_implementations must have the same device-type as the target + # * Does one implementation need to support multiple device types? + # * Do we need multiple implementations for the same device based on proxy features? + # TODO doesn't support transports with only batch exec(orchestrator). update orch + result = transport.run_task(proxy_target, task, arguments, options) + + Bolt::Result.new(target, value: result.value) + end + end + end +end diff --git a/lib/bolt/transport/ssh.rb b/lib/bolt/transport/ssh.rb index fdf4f90457..7051f3512e 100644 --- a/lib/bolt/transport/ssh.rb +++ b/lib/bolt/transport/ssh.rb @@ -12,7 +12,9 @@ def self.options %w[port user password sudo-password private-key host-key-check connect-timeout tmpdir run-as tty run-as-command] end - PROVIDED_FEATURES = ['shell'].freeze + def provided_features + ['shell'] + end def self.validate(options) logger = Logging.logger[self] @@ -122,11 +124,10 @@ def run_script(target, script, arguments, options = {}) end def run_task(target, task, arguments, options = {}) - implementation = task.select_implementation(target, PROVIDED_FEATURES) + implementation = select_implementation(target, task) executable = implementation['path'] input_method = implementation['input_method'] extra_files = implementation['files'] - input_method ||= 'both' # unpack any Sensitive data arguments = unwrap_sensitive_args(arguments) diff --git a/lib/bolt/transport/winrm.rb b/lib/bolt/transport/winrm.rb index de8530debd..b9096765c5 100644 --- a/lib/bolt/transport/winrm.rb +++ b/lib/bolt/transport/winrm.rb @@ -14,7 +14,13 @@ def self.options %w[port user password connect-timeout ssl ssl-verify tmpdir cacert extensions] end - PROVIDED_FEATURES = ['powershell'].freeze + def provided_features + ['powershell'] + end + + def default_input_method + 'powershell' + end def self.validate(options) ssl_flag = options['ssl'] @@ -108,7 +114,7 @@ def run_script(target, script, arguments, _options = {}) end def run_task(target, task, arguments, _options = {}) - implementation = task.select_implementation(target, PROVIDED_FEATURES) + implementation = select_implementation(target, task) executable = implementation['path'] input_method = implementation['input_method'] extra_files = implementation['files'] diff --git a/libexec/apply_catalog.rb b/libexec/apply_catalog.rb index 1e3b3eade7..44b64c5f73 100755 --- a/libexec/apply_catalog.rb +++ b/libexec/apply_catalog.rb @@ -31,6 +31,23 @@ Puppet[:default_file_terminus] = :file_server +if args['_target'] + # TODO: move some of this logic to puppet + special_keys = ['type', 'debug'] + connection = conn_info.reject { |k, _| special_keys.include?(k) } + device = OpenStruct.new(connection) + device.provider = conn_info['type'] + device.options[:debug] = true if conn_info['debug'] + Puppet[:certname] = device.name + Puppet::Util::NetworkDevice.init(device) + + Puppet[:facts_terminus] = :network_device + Puppet[:node_terminus] = :plain + Puppet[:catalog_terminus] = :compiler + # TODO this shouldn't be device speciifc + Puppet[:catalog_cache_terminus] = nil +end + exit_code = 0 begin # This happens implicitly when running the Configurer, but we make it explicit here. It creates the @@ -49,6 +66,7 @@ end # Ensure custom facts are available for provider suitability tests + # TODO: skip this for devices? facts = Puppet::Node::Facts.indirection.find(SecureRandom.uuid, environment: env) report = if Puppet::Util::Package.versioncmp(Puppet.version, '5.0.0') > 0 @@ -57,8 +75,11 @@ Puppet::Transaction::Report.new('apply') end - Puppet.override(current_environment: env, - loaders: Puppet::Pops::Loaders.new(env)) do + overrides = { current_environment: env, + loaders: Puppet::Pops::Loaders.new(env) } + ovierrides[:network_device] = true if args['_target'] + + Puppet.override(overrides) do catalog = Puppet::Resource::Catalog.from_data_hash(args['catalog']) catalog.environment = env.name.to_s catalog.environment_instance = env diff --git a/libexec/custom_facts.rb b/libexec/custom_facts.rb index 275518e93a..178bb0ebed 100755 --- a/libexec/custom_facts.rb +++ b/libexec/custom_facts.rb @@ -4,6 +4,7 @@ require 'json' require 'puppet' require 'puppet/module_tool/tar' +require 'puppet/util/network_device' require 'tempfile' args = JSON.parse(STDIN.read) @@ -19,6 +20,19 @@ cli << '--modulepath' << moduledir Puppet.initialize_settings(cli) + if (conn_info = args['_target']) + special_keys = ['type', 'debug'] + connection = conn_info.reject { |k, _| special_keys.include?(k) } + device = OpenStruct.new(connection) + device.provider = conn_info['type'] + device.options[:debug] = true if conn_info['debug'] + Puppet[:facts_terminus] = :network_device + Puppet[:certname] = device.name + Puppet::Util::NetworkDevice.init(device) + puts "device: #{device}" + exit 1 + end + Tempfile.open('plugins.tar.gz') do |plugins| File.binwrite(plugins, Base64.decode64(args['plugins'])) Puppet::ModuleTool::Tar.instance.unpack(plugins, moduledir, Etc.getlogin || Etc.getpwuid.name) @@ -30,6 +44,8 @@ end facts = Puppet::Node::Facts.indirection.find(SecureRandom.uuid, environment: env) + # TODO: the device command does this should we? + facts.name = facts.values['clientcert'] puts(facts.values.to_json) end diff --git a/pre-docs/bolt_configuration_options.md b/pre-docs/bolt_configuration_options.md index 870b4a5e89..91f2e8f103 100644 --- a/pre-docs/bolt_configuration_options.md +++ b/pre-docs/bolt_configuration_options.md @@ -92,6 +92,15 @@ ssh: `service-options`: A hash of options to configure the Docker connection. Only necessary if using a non-default URL. See https://github.com/swipely/docker-api for supported options. +## Remote transport configuration options + +*The remote transport is a new feature and currently experimental. It's configuration options and behavior may change between y releases* + +`run-on`: The proxy target the task should execute on. Default is `localhost` + +`conn-info`: A hash of connection info that will be passed to the device as `_target`. + + ## Log file configuration options Capture the results of your plan runs in a log file. diff --git a/pre-docs/inventory_file.md b/pre-docs/inventory_file.md index 2688668fb7..735fbac0ef 100644 --- a/pre-docs/inventory_file.md +++ b/pre-docs/inventory_file.md @@ -61,9 +61,9 @@ groups: ## Override a user for a specific node ``` -nodes: +nodes: - name: linux1.example.com - config: + config: ssh: user: me ``` @@ -195,10 +195,24 @@ nodes: run-as: root ``` -- **[Generating inventory files](inventory_file_generating.md)** +Configure a remote target + +```yaml +nodes: + - host1.example.com + - name: remote.example.com + config: + transport: remote + remote: + run-on: host1.example.com + conn-info: + url: https://user1:secret@remote.example.com +``` + +- **[Generating inventory files](inventory_file_generating.md)** Use the `bolt-inventory-pdb` script to generate inventory files based on PuppetDB queries. -**Related information** +**Related information** [Naming tasks](writing_tasks.md#) diff --git a/pre-docs/writing_tasks.md b/pre-docs/writing_tasks.md index fafae7d9a6..f1c2020e3f 100644 --- a/pre-docs/writing_tasks.md +++ b/pre-docs/writing_tasks.md @@ -80,7 +80,7 @@ Pattern[/\A[^\/\\]*\z/] $path In addition to these task restrictions, different scripting languages each have their own ways to validate user input. -### PowerShell +### PowerShell In PowerShell, code injection exploits calls that specifically evaluate code. Do not call `Invoke-Expression` or `Add-Type` with user input. These commands evaluate strings as C\# code. @@ -129,7 +129,7 @@ Resolve file paths with `os.realpath` and confirm them to be within another path For more information on the vulnerabilities of Python or how to escape variables, see Kevin London's blog post on [Dangerous Python Functions](https://www.kevinlondon.com/2015/07/26/dangerous-python-functions.html). -### Ruby +### Ruby In Ruby, command injection is introduced through commands like `eval`, `exec`, `system`, backtick \(\`\`\) or `%x()` execution, or the Open3 module. You can safely call these functions with user input by passing the input as additional arguments instead of a single string. @@ -296,7 +296,7 @@ To create a task that includes additional files pulled from modules, include the - the module name - one of `lib`, `files`, or `tasks` for the directory within the module -- the remaining path to a file or directory; directories must include a trailing slash `/` +- the remaining path to a file or directory; directories must include a trailing slash `/` All path separators must be forward slashes. An example would be `stdlib/lib/puppet/`. @@ -317,7 +317,7 @@ When a task includes the `files` property, all files listed in the top-level p For example, you can create a task and metadata in a new module at `~/.puppetlabs/bolt/site/mymodule/tasks/task.{json,rb}`. - **Metadata** + **Metadata** ``` { @@ -325,7 +325,7 @@ For example, you can create a task and metadata in a new module at `~/.puppetlab } ``` - **File Resource** + **File Resource** `multi_task/files/rb_helper.rb` @@ -335,7 +335,7 @@ def useful_ruby end ``` - **Task** + **Task** ``` #!/usr/bin/env ruby @@ -349,7 +349,7 @@ require_relative File.join(params['_installdir'], 'multi_task', 'files', 'rb_hel puts useful_ruby.to_json ``` - **Output** + **Output** ``` Started on localhost... @@ -361,6 +361,18 @@ Successful on 1 node: localhost Ran on 1 node in 0.12 seconds ``` +### Remote Tasks + +Some targets are hard or impossible to execute tasks on directly. For example a +network device may have a limited shell environment or a cloud service may be +driven only by HTTP APIs. In these cases it makes sense to write a task that +runs on a proxy target and remotely interacts with real target. By writing a +remote task Bolt allows users to specify connection information for remote +targets in their inventory file and injects them into the `_target` metaparam. + + +(example)[https://github.com/puppetlabs/puppetlabs-panos/pull/70] + ### Task Helpers To simplify writing tasks, Bolt includes [python\_task\_helper](https://github.com/puppetlabs/puppetlabs-python_task_helper) and [ruby\_task\_helper](https://github.com/puppetlabs/puppetlabs-ruby_task_helper). It also makes a useful demonstration of including code from another module. @@ -369,7 +381,7 @@ To simplify writing tasks, Bolt includes [python\_task\_helper](https://github.c Create task and metadata in a module at `~/.puppetlabs/bolt/site/mymodule/tasks/task.{json,py}`. - **Metadata** + **Metadata** ``` { @@ -378,7 +390,7 @@ Create task and metadata in a module at `~/.puppetlabs/bolt/site/mymodule/tasks/ } ``` - **Task** + **Task** ``` #!/usr/bin/env python @@ -395,7 +407,7 @@ if __name__ == '__main__': MyTask().run() ``` - **Output** + **Output** ``` $ bolt task run mymodule::task -n localhost name='Julia' @@ -412,7 +424,7 @@ Ran on 1 node in 0.12 seconds Create task and metadata in a new module at `~/.puppetlabs/bolt/site/mymodule/tasks/mytask.{json,rb}`. - **Metadata** + **Metadata** ``` { @@ -421,7 +433,7 @@ Create task and metadata in a new module at `~/.puppetlabs/bolt/site/mymodule/ta } ``` - **Task** + **Task** ``` #!/usr/bin/env ruby @@ -430,13 +442,13 @@ require_relative '../lib/task_helper.rb' class MyTask < TaskHelper def task(name: nil, **kwargs) { greeting: "Hi, my name is #{name}" } - end + end end MyTask.run if __FILE__ == $0 ``` - **Output** + **Output** ``` $ bolt task run mymodule::mytask -n localhost name="Robert'); DROP TABLE Students;--" @@ -469,7 +481,7 @@ For example, to add a `message` parameter to your task, read it from the environ echo your message is $PT_message ``` -### Defining parameters in Windows +### Defining parameters in Windows For Windows tasks, you can pass parameters as environment variables, but it's easier to write your task in PowerShell and use named arguments. By default tasks with a `.ps1` extension use PowerShell standard argument handling. @@ -805,16 +817,16 @@ To define a parameter as sensitive within the JSON metadata, add the `"sensitive The following table shows task metadata keys, values, and default values. -#### **Task metadata** +#### **Task metadata** |Metadata key|Description|Value|Default| |------------|-----------|-----|-------| |"description"|A description of what the task does.|String|None| -|"input\_method"|What input method the task runner should use to pass parameters to the task.| - `environment` +|"input\_method"|What input method the task runner should use to pass parameters to the task.| - `environment` -- `stdin` +- `stdin` -- `powershell` +- `powershell` | Both `environment` and `stdin` unless `.ps1` tasks, in which case `powershell` @@ -833,7 +845,7 @@ Task metadata can accept most Puppet data types. #### Common task data types -**Restriction:** +**Restriction:** Some types supported by Puppet can not be represented as JSON, such as `Hash[Integer, String]`, `Object`, or `Resource`. These should not be used in tasks, because they can never be matched. @@ -850,7 +862,7 @@ Some types supported by Puppet can not be represented as JSON, such as `Hash[Int | `Variant[Integer, Pattern[/\A\d+\Z/]]` |Matches an integer or a String of an integer| | `Boolean` |Accepts Boolean values.| -**Related information** +**Related information** [Data type syntax](https://puppet.com/docs/puppet/latest/lang_data_type.html) diff --git a/spec/bolt/transport/local_spec.rb b/spec/bolt/transport/local_spec.rb index 3d31420ff3..ee9ad64469 100644 --- a/spec/bolt/transport/local_spec.rb +++ b/spec/bolt/transport/local_spec.rb @@ -3,6 +3,7 @@ require 'spec_helper' require 'bolt/transport/local' require 'bolt/target' +require 'bolt/inventory' require_relative 'shared_examples' @@ -24,6 +25,27 @@ allow(Dir).to receive(:mktmpdir).with(no_args).and_raise('no tmpdir') end + # TODO: move to transport API examples + context 'when used as a proxy' do + let(:inventory) { Bolt::Inventory.new({}) } + let(:target) do + target = Bolt::Target.new('foo://user:pass@example.com/path/to?query=hey', + 'run-on' => 'localhost', 'device-type' => 'adevice') + target.inventory = Bolt::Inventory.new({}) + target + end + + it 'passes the correct _target' do + with_task_containing('remote', "#!/bin/sh\ncat", 'stdin') do |task| + result = local.run_task(target, task, {'param' => 'val'}).value + expect(result).to include('param' => 'val') + expect(result['_target']).to include("name"=>"foo://user:pass@example.com/path/to?query=hey") + expect(result['_target']).to include('device-type' => 'adevice') + expect(result['_target']).to include('host' => 'example.com') + end + end + end + include_examples 'transport failures' end end diff --git a/spec/fixtures/apply/device_test/lib/puppet/provider/fake_device/fake_device.rb b/spec/fixtures/apply/device_test/lib/puppet/provider/fake_device/fake_device.rb new file mode 100644 index 0000000000..e1f40d56ea --- /dev/null +++ b/spec/fixtures/apply/device_test/lib/puppet/provider/fake_device/fake_device.rb @@ -0,0 +1,21 @@ +require 'puppet/resource_api' +require 'puppet/resource_api/simple_provider' + +class Puppet::Provider::FakeDevice::FakeDevice < Puppet::ResourceApi::SimpleProvider + + def get(context) + context.device.get + end + + def update(context, name, should) + context.device.set(name, should['content']) + end + + def create(context, name, should) + update(context, name, should) + end + + def delete(context, name) + context.device.delete(name) + end +end diff --git a/spec/fixtures/apply/device_test/lib/puppet/type/fake_device.rb b/spec/fixtures/apply/device_test/lib/puppet/type/fake_device.rb new file mode 100644 index 0000000000..b18675cf30 --- /dev/null +++ b/spec/fixtures/apply/device_test/lib/puppet/type/fake_device.rb @@ -0,0 +1,29 @@ +require 'puppet/resource_api' + +Puppet::ResourceApi.register_type( + name: 'fake_device', + docs: 'fake device for testing puppet-device', + attributes: { + ensure: { + type: 'Enum[present, absent]', + desc: 'Whether this apt key should be present or absent on the target system.', + default: 'present', + }, + name: { + type: 'String', + desc: 'the path the key to modify', + behaviour: :namevar, + }, + content: { + type: 'Data', + desc: 'The value to set' + }, + merge: { + type: 'Boolean', + desc: 'whether to merge the value', + default: false, + behaviour: :parameter + } + }, + features: ['remote_resource'], +) diff --git a/spec/fixtures/apply/device_test/lib/puppet/util/network_device/fake/device.rb b/spec/fixtures/apply/device_test/lib/puppet/util/network_device/fake/device.rb new file mode 100644 index 0000000000..ad2f6dfee4 --- /dev/null +++ b/spec/fixtures/apply/device_test/lib/puppet/util/network_device/fake/device.rb @@ -0,0 +1,51 @@ +require 'puppet/util/network_device/base' + +module Puppet::Util::NetworkDevice::Fake + class Device + def initialize(url, _options = nil) + @path = URI.parse(url).path + end + + def data + if File.file? @path + JSON.load(File.open(@path)) + else + {} + end + end + + def write(new_data) + File.write(@path, new_data.to_json) + end + + def facts + { + 'operatingsystem' => 'FakeDevice', + 'exists' => File.exists?(@path), + 'size' => File.size?(@path) || 0, + } + end + + def get + data.map do |k, v| + { + name: k, + content: v, + ensure: 'present', + } + end + end + + def set(path, val, merge = false) + new = data + new[path] = val + write(new) + end + + def delete(path) + new = data + new.delete(path) + write(new) + end + end +end diff --git a/spec/fixtures/apply/device_test/plans/facts.pp b/spec/fixtures/apply/device_test/plans/facts.pp new file mode 100644 index 0000000000..c0b4a8e29c --- /dev/null +++ b/spec/fixtures/apply/device_test/plans/facts.pp @@ -0,0 +1,8 @@ +plan device_test::facts( + $nodes = 'localhost', +) { + # rely on the agent already being installed + apply_prep($nodes) + $f = get_targets($nodes).map |$t| { [$t.name, $t.facts] } + return($f) +} diff --git a/spec/fixtures/apply/device_test/plans/set_a_val.pp b/spec/fixtures/apply/device_test/plans/set_a_val.pp new file mode 100644 index 0000000000..fcf15a1b3b --- /dev/null +++ b/spec/fixtures/apply/device_test/plans/set_a_val.pp @@ -0,0 +1,15 @@ +plan device_test::set_a_val( + $key = 'key1', + $val = 'val1', + $nodes = 'localhost' +) { + # rely on the agent already being installed + run_plan('facts', nodes => $nodes) + + $r = apply($nodes) { + fake_device { $key: + content => $val + } + } + return $r +} diff --git a/spec/fixtures/modules/facts/plans/emit.pp b/spec/fixtures/modules/facts_test/plans/emit.pp similarity index 68% rename from spec/fixtures/modules/facts/plans/emit.pp rename to spec/fixtures/modules/facts_test/plans/emit.pp index 6b48ad890a..cd67d586d5 100644 --- a/spec/fixtures/modules/facts/plans/emit.pp +++ b/spec/fixtures/modules/facts_test/plans/emit.pp @@ -1,4 +1,4 @@ -plan facts::emit(String $host) { +plan facts_test::emit(String $host) { $target = get_targets($host)[0] return "Facts for ${host}: ${facts($target)}" } diff --git a/spec/fixtures/modules/facts/plans/init.pp b/spec/fixtures/modules/facts_test/plans/init.pp similarity index 83% rename from spec/fixtures/modules/facts/plans/init.pp rename to spec/fixtures/modules/facts_test/plans/init.pp index b8c953a54c..4c6a49424f 100644 --- a/spec/fixtures/modules/facts/plans/init.pp +++ b/spec/fixtures/modules/facts_test/plans/init.pp @@ -1,4 +1,4 @@ -plan facts(String $host) { +plan facts_test(String $host) { $target = get_targets($host)[0] add_facts($target, { 'kernel' => 'Linux', 'cloud' => { 'provider' => 'AWS' } }) return "Facts for ${host}: ${facts($target)}" diff --git a/spec/integration/apply_spec.rb b/spec/integration/apply_spec.rb index 6f064fb99a..ca55a235f5 100644 --- a/spec/integration/apply_spec.rb +++ b/spec/integration/apply_spec.rb @@ -60,29 +60,29 @@ def agent_version_inventory end context "when running against puppet 5 or puppet 6" do - before(:all) do - # install puppet5 - result = run_task('puppet_agent::install', 'puppet_5', { 'collection' => 'puppet5' }, - config: root_config, inventory: agent_version_inventory) - expect(result.count).to eq(1) - expect(result[0]['status']).to eq('success') - - result = run_task('puppet_agent::version', 'puppet_5', inventory: agent_version_inventory) - expect(result.count).to eq(1) - expect(result[0]['status']).to eq('success') - expect(result[0]['result']['version']).to match(/^5/) - - # install puppet6 - result = run_task('puppet_agent::install', 'puppet_6', { 'collection' => 'puppet6' }, - config: root_config, inventory: agent_version_inventory) - expect(result.count).to eq(1) - expect(result[0]['status']).to eq('success') - - result = run_task('puppet_agent::version', 'puppet_6', inventory: agent_version_inventory) - expect(result.count).to eq(1) - expect(result[0]['status']).to eq('success') - expect(result[0]['result']['version']).to match(/^6/) - end + #before(:all) do + # # install puppet5 + # result = run_task('puppet_agent::install', 'puppet_5', { 'collection' => 'puppet5' }, + # config: root_config, inventory: agent_version_inventory) + # expect(result.count).to eq(1) + # expect(result[0]).to include('status' => 'success') + + # result = run_task('puppet_agent::version', 'puppet_5', inventory: agent_version_inventory) + # expect(result.count).to eq(1) + # expect(result[0]['status']).to eq('success') + # expect(result[0]['result']['version']).to match(/^5/) + + # # install puppet6 + # result = run_task('puppet_agent::install', 'puppet_6', { 'collection' => 'puppet6' }, + # config: root_config, inventory: agent_version_inventory) + # expect(result.count).to eq(1) + # expect(result[0]['status']).to eq('success') + + # result = run_task('puppet_agent::version', 'puppet_6', inventory: agent_version_inventory) + # expect(result.count).to eq(1) + # expect(result[0]['status']).to eq('success') + # expect(result[0]['result']['version']).to match(/^6/) + #end it 'runs a ruby task' do with_tempfile_containing('inventory', YAML.dump(agent_version_inventory), '.yaml') do |inv| @@ -116,6 +116,47 @@ def agent_version_inventory expect(result['result']['stdout']).to match(/not found/) end end + + context "when running against device targets" do + let(:device_url) { "file:///tmp/#{SecureRandom.uuid}.json" } + let(:device_inventory) do + device_group = { 'name' => 'device_targets', + 'nodes' => [ + { 'name' => "puppet5_device", + 'config' => { 'remote' => { 'run-on' => 'puppet_5' }}, + }, + { 'name' => "puppet6_device", + 'config' => { 'remote' => { 'run-on' => 'puppet_6' }}, + }, + ], + 'config' => { + 'transport' => 'remote', + 'remote' => { + 'device-type' => 'fake', + 'url' => device_url, + } + } + } + inv = agent_version_inventory + inv['groups'] << device_group + inv + end + + it 'gathers facts from devices' do + with_tempfile_containing('inventory', YAML.dump(device_inventory), '.yaml') do |inv| + results = run_cli_json(%W[plan run device_test::facts --nodes device_targets + --modulepath #{modulepath} --inventoryfile #{inv.path}]) + + require 'pry'; binding.pry + expect(result).to eq([]) + results.each do |result| + expect(result['status']).to eq('success') + report = result['result']['report'] + expect(report['resource_statuses']).to include("Notify[Apply: Hi!]") + end + end + end + end end context "when installing puppet" do @@ -303,7 +344,7 @@ def config result = run_task('puppet_agent::install', conn_uri('winrm'), { 'collection' => 'puppet6' }, config: config) expect(result.count).to eq(1) - expect(result[0]['status']).to eq('success') + expect(result[0]).to include('status' => 'success') result = run_task('puppet_agent::version', conn_uri('winrm'), config: config) expect(result.count).to eq(1) diff --git a/spec/integration/device_spec.rb b/spec/integration/device_spec.rb new file mode 100644 index 0000000000..789e44d6e8 --- /dev/null +++ b/spec/integration/device_spec.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'bolt_spec/conn' +require 'bolt_spec/files' +require 'bolt_spec/integration' +require 'bolt_spec/run' + +describe "devices" do + include BoltSpec::Conn + include BoltSpec::Files + include BoltSpec::Integration + include BoltSpec::Run + + let(:modulepath) { File.join(__dir__, '../fixtures/apply') } + let(:config_flags) { %W[--format json --nodes #{uri} --password #{password} --modulepath #{modulepath}] + tflags } + + describe 'over ssh', ssh: true do + let(:uri) { conn_uri('ssh') } + let(:password) { conn_info('ssh')[:password] } + let(:tflags) { %W[--no-host-key-check --run-as root --sudo-password #{password}] } + + let(:device_url) { "file:///tmp/#{SecureRandom.uuid}.json" } + + def root_config + { 'modulepath' => File.join(__dir__, '../fixtures/apply'), + 'ssh' => { + 'run-as' => 'root', + 'sudo-password' => conn_info('ssh')[:password], + 'host-key-check' => false + } } + end + + def agent_version_inventory + { 'groups' => [ + { 'name' => 'agent_targets', + 'groups' => [ + { 'name' => 'puppet_5', + 'nodes' => [conn_uri('ssh', override_port: 20023)], + 'config' => { 'ssh' => { 'port' => 20023 } } }, + { 'name' => 'puppet_6', + 'nodes' => [conn_uri('ssh', override_port: 20024)], + 'config' => { 'ssh' => { 'port' => 20024 } } } + ], + 'config' => { + 'ssh' => { 'host' => conn_info('ssh')[:host], + 'host-key-check' => false, + 'user' => conn_info('ssh')[:user], + 'password' => conn_info('ssh')[:password], + 'key' => conn_info('ssh')[:key] } + } }, + ] } + end + + let(:device_inventory) do + device_group = { 'name' => 'device_targets', + 'nodes' => [ + # TODO map name to url in target? + { 'name' => 'p5_device', + 'url' => device_url, + 'device-type' => 'fake', + 'run-on' => 'puppet_5', + }, + { 'name' => 'p6_device', + 'url' => device_url, + 'device-type' => 'fake', + 'run-on' => 'puppet_5', + }, + ] + } + agent_version_inventory['groups'] << device_group + end + + after(:all) do + # TODO: Extract into test helper if needed in more files + uri = conn_uri('ssh') + inventory_data = conn_inventory + config_data = root_config + uninstall = '/opt/puppetlabs/bin/puppet resource package puppet-agent ensure=absent' + run_command(uninstall, uri, config: config_data, inventory: inventory_data) + end + + context "when running against puppet 5 or puppet 6" do + before(:all) do + # install puppet5 + result = run_task('puppet_agent::install', 'puppet_5', { 'collection' => 'puppet5' }, + config: root_config, inventory: agent_version_inventory) + expect(result.count).to eq(1) + expect(result[0]['status']).to eq('success') + + result = run_task('puppet_agent::version', 'puppet_5', inventory: agent_version_inventory) + expect(result.count).to eq(1) + expect(result[0]['status']).to eq('success') + expect(result[0]['result']['version']).to match(/^5/) + + # install puppet6 + result = run_task('puppet_agent::install', 'puppet_6', { 'collection' => 'puppet6' }, + config: root_config, inventory: agent_version_inventory) + expect(result.count).to eq(1) + expect(result[0]['status']).to eq('success') + + result = run_task('puppet_agent::version', 'puppet_6', inventory: agent_version_inventory) + expect(result.count).to eq(1) + expect(result[0]['status']).to eq('success') + expect(result[0]['result']['version']).to match(/^6/) + end + + it 'runs an apply plan' do + with_tempfile_containing('inventory', YAML.dump(device_inventory), '.yaml') do |inv| + results = run_cli_json(%W[plan run device_test::facts --nodes device_targets + --modulepath #{modulepath} --inventoryfile #{inv.path}]) + require 'pry'; binding.pry + results.each do |result| + expect(result['status']).to eq('success') + report = result['result']['report'] + expect(report['resource_statuses']).to include("Notify[Apply: Hi!]") + end + end + end + end + end +end diff --git a/spec/integration/inventory_spec.rb b/spec/integration/inventory_spec.rb index d4459fc32d..0ec27d32a2 100644 --- a/spec/integration/inventory_spec.rb +++ b/spec/integration/inventory_spec.rb @@ -125,7 +125,7 @@ def var_plan(name = 'vars') "foo => bar}, kernel => Linux}" } - def fact_plan(name = 'facts') + def fact_plan(name = 'facts_test') ['plan', 'run', name, "host=#{target}"] + config_flags end @@ -142,7 +142,7 @@ def fact_plan(name = 'facts') let(:inventory) { {} } it 'does not error when facts are retrieved' do - expect(run_cli_json(fact_plan('facts::emit'))).to eq("Facts for localhost: {}") + expect(run_cli_json(fact_plan('facts_test::emit'))).to eq("Facts for localhost: {}") end it 'does not error when facts are added' do