From 6571de4e78a49f226e0fcef7fbb37330386dfb11 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:29:04 +0200 Subject: [PATCH 01/13] Add Azure Storage CLI connection config to specs and yml.erb files --- jobs/cloud_controller_clock/spec | 9 +++++++++ .../templates/cloud_controller_ng.yml.erb | 12 ++++++++++++ jobs/cloud_controller_ng/spec | 13 +++++++++++++ .../templates/cloud_controller_ng.yml.erb | 12 ++++++++++++ jobs/cloud_controller_worker/spec | 9 +++++++++ .../templates/cloud_controller_ng.yml.erb | 12 ++++++++++++ 6 files changed, 67 insertions(+) diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index 0051332e29..dcf98a109e 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -35,6 +35,7 @@ packages: - jemalloc - mariadb_connector_c - ruby-3.2 + - azure-storage-cli consumes: - name: database @@ -216,6 +217,8 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.resource_pool.connection_config: + description: "Azure Storage Cli connection hash" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -259,6 +262,8 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.packages.connection_config: + description: "Azure Storage Cli connection hash" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -299,6 +304,8 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.droplets.connection_config: + description: "Azure Storage Cli connection hash" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -339,6 +346,8 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.buildpacks.connection_config: + description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index 71bcb24cc4..3d088f8700 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -211,6 +211,9 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> + <% if_p("cc.resource_pool.connection_config") do %> + connection_config: <%= p("cc.resource_pool.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -234,6 +237,9 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> + <% if_p("cc.packages.connection_config") do %> + connection_config: <%= p("cc.packages.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -256,6 +262,9 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> + <% if_p("cc.droplets.connection_config") do %> + connection_config: <%= p("cc.droplets.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -278,6 +287,9 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> + <% if_p("cc.buildpacks.connection_config") do %> + connection_config: <%= p("cc.buildpacks.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index 73eadd2ef9..355cec5168 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -74,6 +74,7 @@ packages: - jemalloc - mariadb_connector_c - ruby-3.2 + - azure-storage-cli provides: - name: cloud_controller @@ -234,6 +235,10 @@ provides: - cc.temporary_enable_deprecated_thin_webserver - cc.custom_root_links - cc.feature_flag_overrides + - cc.resource_pool.connection_config + - cc.packages.connection_config + - cc.droplets.connection_config + - cc.buildpacks.connection_config consumes: - name: database @@ -562,6 +567,8 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.resource_pool.connection_config: + description: "Azure Storage Cli connection hash" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -610,6 +617,8 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.packages.connection_config: + description: "Azure Storage Cli connection hash" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -655,6 +664,8 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.droplets.connection_config: + description: "Azure Storage Cli connection hash" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -697,6 +708,8 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.buildpacks.connection_config: + description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb index 2c826b0999..7c26272350 100644 --- a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb @@ -339,6 +339,9 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> + <% if_p("cc.resource_pool.connection_config") do %> + connection_config: <%= p("cc.resource_pool.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -363,6 +366,9 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> + <% if_p("cc.packages.connection_config") do %> + connection_config: <%= p("cc.packages.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -385,6 +391,9 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> + <% if_p("cc.droplets.connection_config") do %> + connection_config: <%= p("cc.droplets.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> max_staged_droplets_stored: <%= p("cc.droplets.max_staged_droplets_stored") %> @@ -408,6 +417,9 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> + <% if_p("cc.buildpacks.connection_config") do %> + connection_config: <%= p("cc.buildpacks.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index b1a6bcddcf..027bd580f0 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -45,6 +45,7 @@ packages: - jemalloc - mariadb_connector_c - ruby-3.2 + - azure-storage-cli consumes: - name: database @@ -183,6 +184,8 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.resource_pool.connection_config: + description: "Azure Storage Cli connection hash" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -226,6 +229,8 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.packages.connection_config: + description: "Azure Storage Cli connection hash" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -266,6 +271,8 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.droplets.connection_config: + description: "Azure Storage Cli connection hash" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" @@ -306,6 +313,8 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.buildpacks.connection_config: + description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb index 5f6eccc42d..adb07069b8 100644 --- a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb @@ -176,6 +176,9 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> + <% if_p("cc.resource_pool.connection_config") do %> + connection_config: <%= p("cc.resource_pool.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -200,6 +203,9 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> + <% if_p("cc.packages.connection_config") do %> + connection_config: <%= p("cc.packages.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -222,6 +228,9 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> + <% if_p("cc.droplets.connection_config") do %> + connection_config: <%= p("cc.droplets.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -244,6 +253,9 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> + <% if_p("cc.buildpacks.connection_config") do %> + connection_config: <%= p("cc.buildpacks.connection_config", {}).to_json %> + <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> From 6497b4bca5ccca5029f0b5475d19a39d4461cfa7 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:24:29 +0200 Subject: [PATCH 02/13] pass default timeout of 41s if azure storage cli is used --- .../templates/cloud_controller_ng.yml.erb | 25 +++++++++++++------ .../templates/cloud_controller_ng.yml.erb | 25 +++++++++++++------ .../templates/cloud_controller_ng.yml.erb | 25 +++++++++++++------ 3 files changed, 51 insertions(+), 24 deletions(-) diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index 3d088f8700..ef2b4789ba 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -1,6 +1,15 @@ <% require 'cgi' + # Ensure Azure CLI connection_config has a default timeout if none is set + def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' && !cfg.key?('put_timeout_in_seconds') + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + cfg + end + def discover_external_ip networks = spec.networks.marshal_dump network = networks.values.detect(&:default) || networks.values.first @@ -211,8 +220,8 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> - <% if_p("cc.resource_pool.connection_config") do %> - connection_config: <%= p("cc.resource_pool.connection_config", {}).to_json %> + <% if_p("cc.resource_pool.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.resource_pool.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -237,8 +246,8 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> - <% if_p("cc.packages.connection_config") do %> - connection_config: <%= p("cc.packages.connection_config", {}).to_json %> + <% if_p("cc.packages.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.packages.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -262,8 +271,8 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> - <% if_p("cc.droplets.connection_config") do %> - connection_config: <%= p("cc.droplets.connection_config", {}).to_json %> + <% if_p("cc.droplets.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.droplets.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -287,8 +296,8 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> - <% if_p("cc.buildpacks.connection_config") do %> - connection_config: <%= p("cc.buildpacks.connection_config", {}).to_json %> + <% if_p("cc.buildpacks.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.buildpacks.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb index 7c26272350..71b0d045f8 100644 --- a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb @@ -1,6 +1,15 @@ <% require 'cgi' + # Ensure Azure CLI connection_config has a default timeout if none is set + def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' && !cfg.key?('put_timeout_in_seconds') + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + cfg + end + def discover_external_ip networks = spec.networks.marshal_dump network = networks.values.detect(&:default) || networks.values.first @@ -339,8 +348,8 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> - <% if_p("cc.resource_pool.connection_config") do %> - connection_config: <%= p("cc.resource_pool.connection_config", {}).to_json %> + <% if_p("cc.resource_pool.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.resource_pool.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -366,8 +375,8 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> - <% if_p("cc.packages.connection_config") do %> - connection_config: <%= p("cc.packages.connection_config", {}).to_json %> + <% if_p("cc.packages.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.packages.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -391,8 +400,8 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> - <% if_p("cc.droplets.connection_config") do %> - connection_config: <%= p("cc.droplets.connection_config", {}).to_json %> + <% if_p("cc.droplets.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.droplets.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -417,8 +426,8 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> - <% if_p("cc.buildpacks.connection_config") do %> - connection_config: <%= p("cc.buildpacks.connection_config", {}).to_json %> + <% if_p("cc.buildpacks.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.buildpacks.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb index adb07069b8..aac8cb0ab6 100644 --- a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb @@ -1,6 +1,15 @@ <% require 'cgi' + # Ensure Azure CLI connection_config has a default timeout if none is set + def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' && !cfg.key?('put_timeout_in_seconds') + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + cfg + end + def discover_external_ip networks = spec.networks.marshal_dump network = networks.values.detect(&:default) || networks.values.first @@ -176,8 +185,8 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> - <% if_p("cc.resource_pool.connection_config") do %> - connection_config: <%= p("cc.resource_pool.connection_config", {}).to_json %> + <% if_p("cc.resource_pool.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.resource_pool.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -203,8 +212,8 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> - <% if_p("cc.packages.connection_config") do %> - connection_config: <%= p("cc.packages.connection_config", {}).to_json %> + <% if_p("cc.packages.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.packages.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -228,8 +237,8 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> - <% if_p("cc.droplets.connection_config") do %> - connection_config: <%= p("cc.droplets.connection_config", {}).to_json %> + <% if_p("cc.droplets.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.droplets.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -253,8 +262,8 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> - <% if_p("cc.buildpacks.connection_config") do %> - connection_config: <%= p("cc.buildpacks.connection_config", {}).to_json %> + <% if_p("cc.buildpacks.connection_config") do |cfg| %> + connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.buildpacks.blobstore_type")).to_json %> <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> From ac99f38b85828ec6650fd32ad2f928fe4391dec3 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Fri, 26 Sep 2025 15:18:53 +0200 Subject: [PATCH 03/13] render per-resource storage-cli configs - Add four job templates to api, worker, and clock to emit storage-cli JSON: storage_cli_config_droplets.json.erb storage_cli_config_packages.json.erb storage_cli_config_buildpacks.json.erb storage_cli_config_resource_pool.json.erb Each template builds "connection_config" for AzureRM only, applies a default put_timeout_in_seconds: "41" if missing, and passes through any custom flags. - Expose new config paths in cloud_controller_*/*.yml.erb - Register the new files in job specs so they render under config/ --- jobs/cloud_controller_clock/spec | 4 ++ .../templates/cloud_controller_ng.yml.erb | 14 ++++-- .../storage_cli_config_buildpacks.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_droplets.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_packages.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_resource_pool.json.erb | 47 +++++++++++++++++++ jobs/cloud_controller_ng/spec | 4 ++ .../templates/cloud_controller_ng.yml.erb | 20 ++++++-- .../storage_cli_config_buildpacks.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_droplets.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_packages.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_resource_pool.json.erb | 47 +++++++++++++++++++ jobs/cloud_controller_worker/spec | 4 ++ .../templates/cloud_controller_ng.yml.erb | 31 +++++++----- .../storage_cli_config_buildpacks.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_droplets.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_packages.json.erb | 47 +++++++++++++++++++ .../storage_cli_config_resource_pool.json.erb | 47 +++++++++++++++++++ 18 files changed, 620 insertions(+), 21 deletions(-) create mode 100644 jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb create mode 100644 jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb create mode 100644 jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb create mode 100644 jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb create mode 100644 jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb create mode 100644 jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb create mode 100644 jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb create mode 100644 jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb create mode 100644 jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb create mode 100644 jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb create mode 100644 jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb create mode 100644 jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index dcf98a109e..dd923e8c44 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -14,6 +14,10 @@ templates: cloud_controller_ng.yml.erb: config/cloud_controller_ng.yml newrelic.yml.erb: config/newrelic.yml stacks.yml.erb: config/stacks.yml + storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json + storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json + storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json + storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json drain.sh.erb: bin/drain ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index ef2b4789ba..7de03d7ecd 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -47,6 +47,12 @@ readiness_port: clock: <%= p("cc.readiness_port.clock") %> pid_filename: /var/vcap/sys/run/cloud_controller_clock/cloud_controller_clock.pid + +storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_droplets.json +storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_buildpacks.json +storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_packages.json +storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_resource_pool.json + newrelic_enabled: false development_mode: false @@ -221,7 +227,7 @@ resource_pool: <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> <% if_p("cc.resource_pool.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.resource_pool.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -247,7 +253,7 @@ packages: <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> <% if_p("cc.packages.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.packages.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -272,7 +278,7 @@ droplets: <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> <% if_p("cc.droplets.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.droplets.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -297,7 +303,7 @@ buildpacks: <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> <% if_p("cc.buildpacks.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.buildpacks.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb new file mode 100644 index 0000000000..50fc19b493 --- /dev/null +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.buildpacks.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb new file mode 100644 index 0000000000..f891d99d15 --- /dev/null +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.droplets.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("cc.droplets.connection_config.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb new file mode 100644 index 0000000000..aa4d0bd7fe --- /dev/null +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.packages.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb new file mode 100644 index 0000000000..06cde9ab93 --- /dev/null +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.resource_pool.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index 355cec5168..03c7c2a8d8 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -53,6 +53,10 @@ templates: restart_drain.sh.erb: bin/restart_drain resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem shutdown_drain.rb.erb: bin/shutdown_drain + storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json + storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json + storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json + storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json ruby_version.sh.erb: bin/ruby_version.sh seed_db.sh.erb: bin/seed_db stack_check.sh.erb: bin/stack_check diff --git a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb index 71b0d045f8..3a22a4f9ba 100644 --- a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb @@ -1,5 +1,10 @@ <% require 'cgi' + require "yaml" + + def connection_config_with_context(cfg, context) + (cfg || {}).merge('context' => context) + end # Ensure Azure CLI connection_config has a default timeout if none is set def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) @@ -321,6 +326,11 @@ default_health_check_timeout: <%= p("cc.default_health_check_timeout") %> maximum_health_check_timeout: <%= p("cc.maximum_health_check_timeout") %> stacks_file: /var/vcap/jobs/cloud_controller_ng/config/stacks.yml +storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_droplets.json +storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_buildpacks.json +storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_packages.json +storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_resource_pool.json + shared_isolation_segment_name: <%= p("cc.shared_isolation_segment_name") %> @@ -349,7 +359,7 @@ resource_pool: <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> <% if_p("cc.resource_pool.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.resource_pool.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -375,8 +385,8 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> - <% if_p("cc.packages.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.packages.blobstore_type")).to_json %> + <% if_p("cc.packages.connection_config") do |cfg| %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -401,7 +411,7 @@ droplets: <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> <% if_p("cc.droplets.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.droplets.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -427,7 +437,7 @@ buildpacks: <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> <% if_p("cc.buildpacks.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.buildpacks.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb new file mode 100644 index 0000000000..50fc19b493 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.buildpacks.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb new file mode 100644 index 0000000000..f891d99d15 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.droplets.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("cc.droplets.connection_config.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb new file mode 100644 index 0000000000..aa4d0bd7fe --- /dev/null +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.packages.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb new file mode 100644 index 0000000000..06cde9ab93 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.resource_pool.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index 027bd580f0..63da9ee2dc 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -16,6 +16,10 @@ templates: newrelic.yml.erb: config/newrelic.yml drain.sh.erb: bin/drain stacks.yml.erb: config/stacks.yml + storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json + storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json + storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json + storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console blobstore_waiter.sh.erb: bin/blobstore_waiter.sh diff --git a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb index aac8cb0ab6..f115042e4e 100644 --- a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb @@ -1,14 +1,17 @@ <% require 'cgi' - # Ensure Azure CLI connection_config has a default timeout if none is set - def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' && !cfg.key?('put_timeout_in_seconds') - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - cfg - end + # # Ensure Azure CLI connection_config has a default timeout if none is set + # def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + # cfg = (connection_cfg || {}).dup + # if blobstore_type == 'storage_cli' && !cfg.key?('put_timeout_in_seconds') + # cfg['put_timeout_in_seconds'] = default_seconds.to_s + # end + # cfg + # end + # def connection_config_with_context(cfg, context) + # (cfg || {}).merge('context' => context) + # end def discover_external_ip networks = spec.networks.marshal_dump @@ -163,6 +166,10 @@ default_health_check_timeout: <%= p("cc.default_health_check_timeout") %> maximum_health_check_timeout: <%= p("cc.maximum_health_check_timeout") %> stacks_file: /var/vcap/jobs/cloud_controller_worker/config/stacks.yml +storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_droplets.json +storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_buildpacks.json +storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_packages.json +storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_resource_pool.json resource_pool: blobstore_type: <%= p("cc.resource_pool.blobstore_type") %> @@ -186,7 +193,7 @@ resource_pool: <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> <% if_p("cc.resource_pool.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.resource_pool.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -213,7 +220,7 @@ packages: <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> <% if_p("cc.packages.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.packages.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -238,7 +245,7 @@ droplets: <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> <% if_p("cc.droplets.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.droplets.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -263,7 +270,7 @@ buildpacks: <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> <% if_p("cc.buildpacks.connection_config") do |cfg| %> - connection_config: <%= cli_cfg_with_default_timeout(cfg, p("cc.buildpacks.blobstore_type")).to_json %> + connection_config: <%= cfg.to_json %> <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb new file mode 100644 index 0000000000..50fc19b493 --- /dev/null +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.buildpacks.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb new file mode 100644 index 0000000000..f891d99d15 --- /dev/null +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.droplets.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("cc.droplets.connection_config.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb new file mode 100644 index 0000000000..aa4d0bd7fe --- /dev/null +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.packages.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb new file mode 100644 index 0000000000..06cde9ab93 --- /dev/null +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb @@ -0,0 +1,47 @@ +<% +require "json" + + # Ensure Azure CLI connection_config has a default timeout if none is set +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +# helper: add key only when value is present +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +scope = "cc.resource_pool.connection_config" +provider = p("#{scope}.provider") + +if provider != "AzureRM" + options = {} # for now: all non-azure providers output an empty JSON object +else + options = {} + options["provider"] = provider + options["account_name"] = p("#{scope}.azure_storage_account_name") + options["container_name"] = p("#{scope}.container_name") + add(options, "account_key", p("#{scope}.azure_storage_access_key")) + add(options, "environment", p("#{scope}.environment", "AzureCloud")) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(options, k.to_s, v) } + end + rescue + # ignore if property not defined + end + options = cli_cfg_with_default_timeout(options, 'storage_cli') +end +-%> +<%= JSON.pretty_generate(options) %> \ No newline at end of file From 5f7266803f608c66a38ab5a8f0f9fc8366e340e7 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Tue, 30 Sep 2025 18:15:50 +0200 Subject: [PATCH 04/13] add put_timeout_in_seconds into connection_config wen present, add tests --- .../templates/cloud_controller_ng.yml.erb | 9 --- .../storage_cli_config_buildpacks.json.erb | 1 + .../storage_cli_config_droplets.json.erb | 1 + .../storage_cli_config_packages.json.erb | 1 + .../storage_cli_config_resource_pool.json.erb | 1 + .../templates/cloud_controller_ng.yml.erb | 5 -- .../storage_cli_config_buildpacks.json.erb | 1 + .../storage_cli_config_droplets.json.erb | 1 + .../storage_cli_config_packages.json.erb | 1 + .../storage_cli_config_resource_pool.json.erb | 1 + .../templates/cloud_controller_ng.yml.erb | 12 --- .../storage_cli_config_buildpacks.json.erb | 1 + .../storage_cli_config_droplets.json.erb | 1 + .../storage_cli_config_packages.json.erb | 1 + .../storage_cli_config_resource_pool.json.erb | 1 + .../cloud_controller_ng_spec.rb | 20 ++++- .../storage_cli_config_jsons_spec.rb | 80 +++++++++++++++++++ 17 files changed, 108 insertions(+), 30 deletions(-) create mode 100644 spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index 7de03d7ecd..21f5d1e90e 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -1,15 +1,6 @@ <% require 'cgi' - # Ensure Azure CLI connection_config has a default timeout if none is set - def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' && !cfg.key?('put_timeout_in_seconds') - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - cfg - end - def discover_external_ip networks = spec.networks.marshal_dump network = networks.values.detect(&:default) || networks.values.first diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb index 50fc19b493..5c1853899d 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb index f891d99d15..3819fa080c 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb index aa4d0bd7fe..90dd877e17 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb index 06cde9ab93..ca6efc9ce8 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb index 3a22a4f9ba..808edd091b 100644 --- a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb @@ -1,11 +1,6 @@ <% require 'cgi' require "yaml" - - def connection_config_with_context(cfg, context) - (cfg || {}).merge('context' => context) - end - # Ensure Azure CLI connection_config has a default timeout if none is set def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) cfg = (connection_cfg || {}).dup diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb index 50fc19b493..5c1853899d 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb index f891d99d15..3819fa080c 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb index aa4d0bd7fe..90dd877e17 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb index 06cde9ab93..ca6efc9ce8 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb index f115042e4e..a4e6134cae 100644 --- a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb @@ -1,18 +1,6 @@ <% require 'cgi' - # # Ensure Azure CLI connection_config has a default timeout if none is set - # def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - # cfg = (connection_cfg || {}).dup - # if blobstore_type == 'storage_cli' && !cfg.key?('put_timeout_in_seconds') - # cfg['put_timeout_in_seconds'] = default_seconds.to_s - # end - # cfg - # end - # def connection_config_with_context(cfg, context) - # (cfg || {}).merge('context' => context) - # end - def discover_external_ip networks = spec.networks.marshal_dump network = networks.values.detect(&:default) || networks.values.first diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb index 50fc19b493..5c1853899d 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb index f891d99d15..3819fa080c 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb index aa4d0bd7fe..90dd877e17 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb index 06cde9ab93..ca6efc9ce8 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb @@ -31,6 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) # optional passthrough for extra storage-cli flags begin diff --git a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb index 6f4855f6ac..8dd26da953 100644 --- a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb +++ b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb @@ -971,14 +971,14 @@ module Test end end - context 'when it is set to buidpack' do + context 'when it is set to buildpack' do before do - merged_manifest_properties['cc']['default_app_lifecycle'] = 'buidpack' + merged_manifest_properties['cc']['default_app_lifecycle'] = 'buildpack' end - it 'renders it as buidpack' do + it 'renders it as buildpack' do template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) - expect(template_hash['default_app_lifecycle']).to eq('buidpack') + expect(template_hash['default_app_lifecycle']).to eq('buildpack') end end @@ -993,6 +993,18 @@ module Test end end end + + describe 'storage_cli_config_file_* paths' do + let(:template) { job.template('config/cloud_controller_ng.yml') } + + it 'renders absolute paths for all four scope files' do + yaml = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) + expect(yaml['storage_cli_config_file_droplets']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_droplets.json') + expect(yaml['storage_cli_config_file_packages']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_packages.json') + expect(yaml['storage_cli_config_file_buildpacks']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_buildpacks.json') + expect(yaml['storage_cli_config_file_resource_pool']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_resource_pool.json') + end + end end end end diff --git a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb new file mode 100644 index 0000000000..c5675662f7 --- /dev/null +++ b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb @@ -0,0 +1,80 @@ +# spec/templates/storage_cli_config_jsons_spec.rb +require 'rspec' +require 'json' +require 'bosh/template/test' + +module Bosh::Template::Test + RSpec.describe 'storage-cli JSON templates' do + let(:release_path) { File.join(File.dirname(__FILE__), '../..') } + let(:release) { ReleaseDir.new(release_path) } + let(:job) { release.job('cloud_controller_ng') } + let(:links) { {} } + let(:props) do + { + 'cc' => { + 'droplets' => { 'connection_config' => {} }, + 'buildpacks' => { 'connection_config' => {} }, + 'packages' => { 'connection_config' => {} }, + 'resource_pool'=> { 'connection_config' => {} }, + } + } + end + + TEMPLATES = { + droplets: ['config/storage_cli_config_droplets.json', %w[cc droplets connection_config]], + buildpacks: ['config/storage_cli_config_buildpacks.json', %w[cc buildpacks connection_config]], + packages: ['config/storage_cli_config_packages.json', %w[cc packages connection_config]], + resource_pool: ['config/storage_cli_config_resource_pool.json', %w[cc resource_pool connection_config]], + }.freeze + + TEMPLATES.each do |scope, (template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } + + def set(props, keypath, value) + h = props + keypath[0..-2].each { |k| h = (h[k] ||= {}) } + h[keypath.last] = value + end + + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) + + json = JSON.parse(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end + + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) + + json = JSON.parse(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end + + it 'renders {} for non-Azure providers' do + set(props, keypath, { 'provider' => 'S3' }) + json = JSON.parse(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end + end + end +end From 0c12f663825ebdd94e3896aa50cb41717c46cd69 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Wed, 1 Oct 2025 09:27:52 +0200 Subject: [PATCH 05/13] fix constant --- .../storage_cli_config_jsons_spec.rb | 134 +++++++++--------- 1 file changed, 70 insertions(+), 64 deletions(-) diff --git a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb index c5675662f7..752d910bc9 100644 --- a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb @@ -1,78 +1,84 @@ -# spec/templates/storage_cli_config_jsons_spec.rb +# frozen_string_literal: true + require 'rspec' -require 'json' +require 'yaml' require 'bosh/template/test' -module Bosh::Template::Test - RSpec.describe 'storage-cli JSON templates' do - let(:release_path) { File.join(File.dirname(__FILE__), '../..') } - let(:release) { ReleaseDir.new(release_path) } - let(:job) { release.job('cloud_controller_ng') } - let(:links) { {} } - let(:props) do - { - 'cc' => { - 'droplets' => { 'connection_config' => {} }, - 'buildpacks' => { 'connection_config' => {} }, - 'packages' => { 'connection_config' => {} }, - 'resource_pool'=> { 'connection_config' => {} }, - } - } - end +TEMPLATES = { + droplets: ['config/storage_cli_config_droplets.json', %w[cc droplets connection_config]], + buildpacks: ['config/storage_cli_config_buildpacks.json', %w[cc buildpacks connection_config]], + packages: ['config/storage_cli_config_packages.json', %w[cc packages connection_config]], + resource_pool: ['config/storage_cli_config_resource_pool.json', %w[cc resource_pool connection_config]] +}.freeze - TEMPLATES = { - droplets: ['config/storage_cli_config_droplets.json', %w[cc droplets connection_config]], - buildpacks: ['config/storage_cli_config_buildpacks.json', %w[cc buildpacks connection_config]], - packages: ['config/storage_cli_config_packages.json', %w[cc packages connection_config]], - resource_pool: ['config/storage_cli_config_resource_pool.json', %w[cc resource_pool connection_config]], - }.freeze +module Bosh + module Template + module Test + RSpec.describe 'storage-cli JSON templates' do + let(:release_path) { File.join(File.dirname(__FILE__), '../..') } + let(:release) { ReleaseDir.new(release_path) } + let(:job) { release.job('cloud_controller_ng') } + let(:links) { {} } + let(:props) do + { + 'cc' => { + 'droplets' => { 'connection_config' => {} }, + 'buildpacks' => { 'connection_config' => {} }, + 'packages' => { 'connection_config' => {} }, + 'resource_pool' => { 'connection_config' => {} } + } + } + end - TEMPLATES.each do |scope, (template_path, keypath)| - describe template_path do - let(:template) { job.template(template_path) } + TEMPLATES.each_value do |(template_path, keypath)| + describe template_path do + let(:template) { job.template(template_path) } - def set(props, keypath, value) - h = props - keypath[0..-2].each { |k| h = (h[k] ||= {}) } - h[keypath.last] = value - end + def set(hash, path, value) + cursor = hash + path[0..-2].each { |key| cursor = (cursor[key] ||= {}) } + cursor[path.last] = value + end - it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '' - }) + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) - json = JSON.parse(template.render(props, consumes: links)) - expect(json).to include( - 'provider' => 'AzureRM', - 'account_name' => 'acc', - 'account_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '41' - ) - end + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end - it 'keeps existing put_timeout_in_seconds when provided' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '7' - }) + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) - json = JSON.parse(template.render(props, consumes: links)) - expect(json['put_timeout_in_seconds']).to eq('7') - end + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json['put_timeout_in_seconds']).to eq('7') + end + + it 'renders {} for non-Azure providers' do + set(props, keypath, { 'provider' => 'S3' }) - it 'renders {} for non-Azure providers' do - set(props, keypath, { 'provider' => 'S3' }) - json = JSON.parse(template.render(props, consumes: links)) - expect(json).to eq({}) + json = YAML.safe_load(template.render(props, consumes: links)) + expect(json).to eq({}) + end + end end end end From 52c9d06928e87557497d947d245c2589530efaf0 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:13:50 +0200 Subject: [PATCH 06/13] If the property cc.buildpacks.connection_config.put_timeout_in_seconds is not set, return nil --- .../templates/storage_cli_config_buildpacks.json.erb | 2 +- .../templates/storage_cli_config_droplets.json.erb | 2 +- .../templates/storage_cli_config_packages.json.erb | 2 +- .../templates/storage_cli_config_resource_pool.json.erb | 2 +- .../templates/storage_cli_config_buildpacks.json.erb | 2 +- .../templates/storage_cli_config_droplets.json.erb | 2 +- .../templates/storage_cli_config_packages.json.erb | 2 +- .../templates/storage_cli_config_resource_pool.json.erb | 2 +- .../templates/storage_cli_config_buildpacks.json.erb | 2 +- .../templates/storage_cli_config_droplets.json.erb | 2 +- .../templates/storage_cli_config_packages.json.erb | 2 +- .../templates/storage_cli_config_resource_pool.json.erb | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb index 5c1853899d..01f6eb17a7 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb index 3819fa080c..fac0b31f00 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb index 90dd877e17..0b1785b034 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb index ca6efc9ce8..2a6e6a464c 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb index 5c1853899d..01f6eb17a7 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb index 3819fa080c..fac0b31f00 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb index 90dd877e17..0b1785b034 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb index ca6efc9ce8..2a6e6a464c 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb index 5c1853899d..01f6eb17a7 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb index 3819fa080c..fac0b31f00 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb index 90dd877e17..0b1785b034 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb index ca6efc9ce8..2a6e6a464c 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb @@ -31,7 +31,7 @@ else options["container_name"] = p("#{scope}.container_name") add(options, "account_key", p("#{scope}.azure_storage_access_key")) add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds")) + add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) # optional passthrough for extra storage-cli flags begin From 07f5aefd3d6bcd67a0b7671842fc7601f3efe79f Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:23:53 +0200 Subject: [PATCH 07/13] Add bosh-azure-storage-cli binary Co-Authored-By: johha --- config/blobs.yml | 4 ++++ jobs/cloud_controller_clock/spec | 1 + jobs/cloud_controller_ng/spec | 1 + jobs/cloud_controller_worker/spec | 1 + packages/azure-storage-cli/README.md | 9 +++++++++ packages/azure-storage-cli/packaging | 5 +++++ packages/azure-storage-cli/spec | 4 ++++ 7 files changed, 25 insertions(+) create mode 100644 packages/azure-storage-cli/README.md create mode 100644 packages/azure-storage-cli/packaging create mode 100644 packages/azure-storage-cli/spec diff --git a/config/blobs.yml b/config/blobs.yml index 40c0ec7d2a..84b904434d 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -1,3 +1,7 @@ +azure-storage-cli/azure-storage-cli-linux-amd64: + size: 7143608 + object_id: ceb2994f-cbe8-4695-413c-33785a0b6322 + sha: sha256:246944046f7e2f919965466055e957c30aeecf13012059c624d12ab04f48f822 expat/expat-2.5.0.tar.bz2: size: 569205 object_id: 970ccd16-75ac-4c55-5280-c00c4aa8f6cc diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index dd923e8c44..6b48a5deae 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -31,6 +31,7 @@ templates: credhub_ca.crt.erb: config/certs/credhub_ca.crt packages: + - azure-storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index 03c7c2a8d8..57bb102207 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -70,6 +70,7 @@ templates: cloud_controller_local_worker_override.yml.erb: config/cloud_controller_local_worker_override.yml packages: + - azure-storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index 63da9ee2dc..f00ff675b1 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -41,6 +41,7 @@ templates: scrape_ca.crt.erb: config/certs/scrape_ca.crt packages: + - azure-storage-cli - capi_utils - cloud_controller_ng - nginx diff --git a/packages/azure-storage-cli/README.md b/packages/azure-storage-cli/README.md new file mode 100644 index 0000000000..7b55338945 --- /dev/null +++ b/packages/azure-storage-cli/README.md @@ -0,0 +1,9 @@ +azure-storage-cli-package +============ +bosh-azure-storage-cli, a command line interface for Azure Storage, is used for BOSH deployments instead of outdated fog-azure tools. + +The file can be downloaded from the following location: + +| Filename | Download URL | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------| +| bosh-azure-storage-cli-0.0.186.tar.gz | [https://github.com/cloudfoundry/bosh-azure-storage-cli](https://github.com/cloudfoundry/bosh-azure-storage-cli/archive/refs/tags/v0.0.186.tar.gz) | diff --git a/packages/azure-storage-cli/packaging b/packages/azure-storage-cli/packaging new file mode 100644 index 0000000000..f996419de3 --- /dev/null +++ b/packages/azure-storage-cli/packaging @@ -0,0 +1,5 @@ +set -e + +mkdir -p ${BOSH_INSTALL_TARGET}/bin +mv azure-storage-cli/azure-storage-cli-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli +chmod +x ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli diff --git a/packages/azure-storage-cli/spec b/packages/azure-storage-cli/spec new file mode 100644 index 0000000000..ec87110de6 --- /dev/null +++ b/packages/azure-storage-cli/spec @@ -0,0 +1,4 @@ +--- +name: azure-storage-cli +files: + - azure-storage-cli/azure-storage-cli-linux-amd64 \ No newline at end of file From d7b2b3925566b817d19005cee8944e8ff81ed0cb Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Mon, 13 Oct 2025 11:08:39 +0200 Subject: [PATCH 08/13] add to manifest: blobstore_provider, jsons read provider from manifest, CCNG reads provider from JSON --- jobs/cloud_controller_clock/spec | 9 +++++++- .../templates/cloud_controller_ng.yml.erb | 12 ----------- .../storage_cli_config_buildpacks.json.erb | 2 +- .../storage_cli_config_droplets.json.erb | 2 +- .../storage_cli_config_packages.json.erb | 2 +- .../storage_cli_config_resource_pool.json.erb | 2 +- jobs/cloud_controller_ng/spec | 13 +++++++++++- .../templates/cloud_controller_ng.yml.erb | 21 ------------------- .../storage_cli_config_buildpacks.json.erb | 2 +- .../storage_cli_config_droplets.json.erb | 2 +- .../storage_cli_config_packages.json.erb | 2 +- .../storage_cli_config_resource_pool.json.erb | 2 +- jobs/cloud_controller_worker/spec | 9 +++++++- .../templates/cloud_controller_ng.yml.erb | 12 ----------- .../storage_cli_config_buildpacks.json.erb | 2 +- .../storage_cli_config_droplets.json.erb | 2 +- .../storage_cli_config_packages.json.erb | 2 +- .../storage_cli_config_resource_pool.json.erb | 2 +- .../storage_cli_config_jsons_spec.rb | 10 ++++----- 19 files changed, 45 insertions(+), 65 deletions(-) diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index 6b48a5deae..bbcb68872a 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -40,7 +40,6 @@ packages: - jemalloc - mariadb_connector_c - ruby-3.2 - - azure-storage-cli consumes: - name: database @@ -177,6 +176,8 @@ properties: cc.staging_upload_password: description: "User's password used to access internal endpoints of Cloud Controller to upload files when staging" + cc.resource_pool.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -225,6 +226,8 @@ properties: cc.resource_pool.connection_config: description: "Azure Storage Cli connection hash" + cc.packages.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -270,6 +273,8 @@ properties: cc.packages.connection_config: description: "Azure Storage Cli connection hash" + cc.droplets.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -312,6 +317,8 @@ properties: cc.droplets.connection_config: description: "Azure Storage Cli connection hash" + cc.buildpacks.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index 21f5d1e90e..ad3e416281 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -217,9 +217,6 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> - <% if_p("cc.resource_pool.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -243,9 +240,6 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> - <% if_p("cc.packages.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -268,9 +262,6 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> - <% if_p("cc.droplets.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -293,9 +284,6 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> - <% if_p("cc.buildpacks.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb index 01f6eb17a7..64d984b7d7 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.buildpacks.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.buildpacks.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb index fac0b31f00..883ab5e685 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.droplets.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.droplets.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb index 0b1785b034..4cf8494316 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.packages.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.packages.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb index 2a6e6a464c..5b7d7ef63c 100644 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.resource_pool.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.resource_pool.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index 57bb102207..3bb7ef1ea7 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -79,7 +79,6 @@ packages: - jemalloc - mariadb_connector_c - ruby-3.2 - - azure-storage-cli provides: - name: cloud_controller @@ -124,6 +123,7 @@ provides: - name: cloud_controller_internal type: cloud_controller_internal properties: + - cc.buildpacks.blobstore_provider - cc.buildpacks.blobstore_type - cc.buildpacks.buildpack_directory_key - cc.buildpacks.cdn.key_pair_id @@ -152,6 +152,7 @@ provides: - cc.default_stack - cc.default_app_lifecycle - cc.disable_private_domain_cross_space_context_path_route_sharing + - cc.droplets.blobstore_provider - cc.droplets.blobstore_type - cc.droplets.cdn.key_pair_id - cc.droplets.cdn.private_key @@ -180,6 +181,7 @@ provides: - cc.max_annotations_per_resource - cc.maximum_health_check_timeout - cc.packages.app_package_directory_key + - cc.packages.blobstore_provider - cc.packages.blobstore_type - cc.packages.cdn.key_pair_id - cc.packages.cdn.private_key @@ -195,6 +197,7 @@ provides: - cc.packages.webdav_config.private_endpoint - cc.packages.webdav_config.public_endpoint - cc.packages.webdav_config.username + - cc.resource_pool.blobstore_provider - cc.resource_pool.blobstore_type - cc.resource_pool.cdn.key_pair_id - cc.resource_pool.cdn.private_key @@ -525,6 +528,8 @@ properties: default: default description: "The name of the quota definition CC will fallback on for org and space limits from the list of quota definitions." + cc.resource_pool.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -575,6 +580,8 @@ properties: cc.resource_pool.connection_config: description: "Azure Storage Cli connection hash" + cc.packages.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -625,6 +632,8 @@ properties: cc.packages.connection_config: description: "Azure Storage Cli connection hash" + cc.droplets.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -672,6 +681,8 @@ properties: cc.droplets.connection_config: description: "Azure Storage Cli connection hash" + cc.buildpacks.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" diff --git a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb index 808edd091b..643f136db9 100644 --- a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb @@ -1,14 +1,5 @@ <% require 'cgi' - require "yaml" - # Ensure Azure CLI connection_config has a default timeout if none is set - def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' && !cfg.key?('put_timeout_in_seconds') - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - cfg - end def discover_external_ip networks = spec.networks.marshal_dump @@ -353,9 +344,6 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> - <% if_p("cc.resource_pool.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -380,9 +368,6 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> - <% if_p("cc.packages.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -405,9 +390,6 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> - <% if_p("cc.droplets.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> max_staged_droplets_stored: <%= p("cc.droplets.max_staged_droplets_stored") %> @@ -431,9 +413,6 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> - <% if_p("cc.buildpacks.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb index 01f6eb17a7..64d984b7d7 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.buildpacks.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.buildpacks.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb index fac0b31f00..a29fa5cc34 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.droplets.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.droplets.blobstore_provider", "AzureRM") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb index 0b1785b034..4cf8494316 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.packages.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.packages.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb index 2a6e6a464c..5b7d7ef63c 100644 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.resource_pool.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.resource_pool.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index f00ff675b1..8db1bbbb4b 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -50,7 +50,6 @@ packages: - jemalloc - mariadb_connector_c - ruby-3.2 - - azure-storage-cli consumes: - name: database @@ -144,6 +143,8 @@ properties: cc.staging_upload_password: description: "User's password used to access internal endpoints of Cloud Controller to upload files when staging" + cc.resource_pool.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -192,6 +193,8 @@ properties: cc.resource_pool.connection_config: description: "Azure Storage Cli connection hash" + cc.packages.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -237,6 +240,8 @@ properties: cc.packages.connection_config: description: "Azure Storage Cli connection hash" + cc.droplets.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -279,6 +284,8 @@ properties: cc.droplets.connection_config: description: "Azure Storage Cli connection hash" + cc.buildpacks.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" diff --git a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb index a4e6134cae..1d9a5184b8 100644 --- a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb @@ -180,9 +180,6 @@ resource_pool: private_key: <%= p("cc.resource_pool.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.resource_pool.fog_connection", {}).to_json %> - <% if_p("cc.resource_pool.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.resource_pool.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.resource_pool.fog_gcp_storage_options", {}).to_json %> @@ -207,9 +204,6 @@ packages: private_key: <%= p("cc.packages.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.packages.fog_connection", {}).to_json %> - <% if_p("cc.packages.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.packages.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.packages.fog_gcp_storage_options", {}).to_json %> @@ -232,9 +226,6 @@ droplets: private_key: <%= p("cc.droplets.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.droplets.fog_connection", {}).to_json %> - <% if_p("cc.droplets.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.droplets.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.droplets.fog_gcp_storage_options", {}).to_json %> @@ -257,9 +248,6 @@ buildpacks: private_key: <%= p("cc.buildpacks.cdn.private_key").inspect %> <% end %> fog_connection: <%= p("cc.buildpacks.fog_connection", {}).to_json %> - <% if_p("cc.buildpacks.connection_config") do |cfg| %> - connection_config: <%= cfg.to_json %> - <% end %> fog_aws_storage_options: <%= p("cc.buildpacks.fog_aws_storage_options", {}).to_json %> fog_gcp_storage_options: <%= p("cc.buildpacks.fog_gcp_storage_options", {}).to_json %> diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb index 01f6eb17a7..64d984b7d7 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.buildpacks.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.buildpacks.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb index fac0b31f00..883ab5e685 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.droplets.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.droplets.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb index 0b1785b034..4cf8494316 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.packages.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.packages.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb index 2a6e6a464c..5b7d7ef63c 100644 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb +++ b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb @@ -20,7 +20,7 @@ def add(h, key, val) end scope = "cc.resource_pool.connection_config" -provider = p("#{scope}.provider") +provider = p("cc.resource_pool.blobstore_provider") if provider != "AzureRM" options = {} # for now: all non-azure providers output an empty JSON object diff --git a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb index 752d910bc9..c1fdd84820 100644 --- a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb @@ -22,10 +22,10 @@ module Test let(:props) do { 'cc' => { - 'droplets' => { 'connection_config' => {} }, - 'buildpacks' => { 'connection_config' => {} }, - 'packages' => { 'connection_config' => {} }, - 'resource_pool' => { 'connection_config' => {} } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' } } } end @@ -73,7 +73,7 @@ def set(hash, path, value) end it 'renders {} for non-Azure providers' do - set(props, keypath, { 'provider' => 'S3' }) + keypath[0..-2].reduce(props) { |acc, elem| acc[elem] ||= {} }['blobstore_provider'] = 'S3' json = YAML.safe_load(template.render(props, consumes: links)) expect(json).to eq({}) From e6173cae601877837369bee5467bcf0be7fba16f Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Tue, 21 Oct 2025 08:46:06 +0200 Subject: [PATCH 09/13] refactor storage_cli_jsons into 1 big json and let pre-start write the dfferent configs --- jobs/cloud_controller_clock/spec | 5 +- .../templates/blobstore_configs.json.erb | 52 +++++++++++++++++++ .../templates/pre-start.sh.erb | 22 ++++++++ .../storage_cli_config_buildpacks.json.erb | 48 ----------------- .../storage_cli_config_droplets.json.erb | 48 ----------------- .../storage_cli_config_packages.json.erb | 48 ----------------- .../storage_cli_config_resource_pool.json.erb | 48 ----------------- jobs/cloud_controller_ng/spec | 5 +- .../templates/blobstore_configs.json.erb | 52 +++++++++++++++++++ .../templates/pre-start.sh.erb | 23 ++++++++ .../storage_cli_config_buildpacks.json.erb | 48 ----------------- .../storage_cli_config_droplets.json.erb | 48 ----------------- .../storage_cli_config_packages.json.erb | 48 ----------------- .../storage_cli_config_resource_pool.json.erb | 48 ----------------- jobs/cloud_controller_worker/spec | 5 +- .../templates/blobstore_configs.json.erb | 52 +++++++++++++++++++ .../templates/pre-start.sh.erb | 22 ++++++++ .../storage_cli_config_buildpacks.json.erb | 48 ----------------- .../storage_cli_config_droplets.json.erb | 48 ----------------- .../storage_cli_config_packages.json.erb | 48 ----------------- .../storage_cli_config_resource_pool.json.erb | 48 ----------------- 21 files changed, 226 insertions(+), 588 deletions(-) create mode 100644 jobs/cloud_controller_clock/templates/blobstore_configs.json.erb delete mode 100644 jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb delete mode 100644 jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb delete mode 100644 jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb delete mode 100644 jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb create mode 100644 jobs/cloud_controller_ng/templates/blobstore_configs.json.erb delete mode 100644 jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb delete mode 100644 jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb delete mode 100644 jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb delete mode 100644 jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb create mode 100644 jobs/cloud_controller_worker/templates/blobstore_configs.json.erb delete mode 100644 jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb delete mode 100644 jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb delete mode 100644 jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb delete mode 100644 jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index bbcb68872a..d476c06397 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -14,10 +14,7 @@ templates: cloud_controller_ng.yml.erb: config/cloud_controller_ng.yml newrelic.yml.erb: config/newrelic.yml stacks.yml.erb: config/stacks.yml - storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json - storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json - storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json - storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json + blobstore_configs.json.erb: config/blobstore_configs.json drain.sh.erb: bin/drain ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console diff --git a/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb b/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb new file mode 100644 index 0000000000..f4e3a36370 --- /dev/null +++ b/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb @@ -0,0 +1,52 @@ +<% +require "json" + +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +def options_for(scope_prefix, provider_prop) + provider = p(provider_prop, nil) + return {} unless provider == "AzureRM" + + h = {} + h["provider"] = provider + h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") + h["container_name"] = p("#{scope_prefix}.container_name") + add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) + add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) + add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope_prefix}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(h, k.to_s, v) } + end + rescue + # property might not exist; ignore + end + + cli_cfg_with_default_timeout(h, 'storage_cli') +end + +all = { + "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), + "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), + "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), + "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), +} +-%> +<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_clock/templates/pre-start.sh.erb b/jobs/cloud_controller_clock/templates/pre-start.sh.erb index ff5912e8a8..a1287a685e 100644 --- a/jobs/cloud_controller_clock/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_clock/templates/pre-start.sh.erb @@ -23,8 +23,30 @@ function setup_directories { chown -R vcap:vcap "$LOG_DIR" } +write_blobstore_scope_configs() { + local cfg_json="${CONFIG_DIR}/blobstore_configs.json" + if [[ ! -f "$cfg_json" ]]; then + echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" + return 0 + fi + + # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. + CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' + require "json" + cfg_path = ENV.fetch("CFG") + dest = ENV.fetch("DEST") + data = JSON.parse(File.read(cfg_path)) + + %w[buildpacks droplets packages resource_pool].each do |k| + out = File.join(dest, "storage_cli_config_#{k}.json") + File.write(out, JSON.pretty_generate(data[k] || {})) + end +RUBY +} + function main { setup_directories + write_blobstore_scope_configs } main diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb deleted file mode 100644 index 64d984b7d7..0000000000 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_buildpacks.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.buildpacks.connection_config" -provider = p("cc.buildpacks.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb deleted file mode 100644 index 883ab5e685..0000000000 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_droplets.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.droplets.connection_config" -provider = p("cc.droplets.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("cc.droplets.connection_config.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb deleted file mode 100644 index 4cf8494316..0000000000 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_packages.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.packages.connection_config" -provider = p("cc.packages.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb deleted file mode 100644 index 5b7d7ef63c..0000000000 --- a/jobs/cloud_controller_clock/templates/storage_cli_config_resource_pool.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.resource_pool.connection_config" -provider = p("cc.resource_pool.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index 3bb7ef1ea7..3155090f3b 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -53,10 +53,7 @@ templates: restart_drain.sh.erb: bin/restart_drain resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem shutdown_drain.rb.erb: bin/shutdown_drain - storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json - storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json - storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json - storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json + blobstore_configs.json.erb: config/blobstore_configs.json ruby_version.sh.erb: bin/ruby_version.sh seed_db.sh.erb: bin/seed_db stack_check.sh.erb: bin/stack_check diff --git a/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb b/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb new file mode 100644 index 0000000000..f4e3a36370 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb @@ -0,0 +1,52 @@ +<% +require "json" + +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +def options_for(scope_prefix, provider_prop) + provider = p(provider_prop, nil) + return {} unless provider == "AzureRM" + + h = {} + h["provider"] = provider + h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") + h["container_name"] = p("#{scope_prefix}.container_name") + add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) + add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) + add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope_prefix}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(h, k.to_s, v) } + end + rescue + # property might not exist; ignore + end + + cli_cfg_with_default_timeout(h, 'storage_cli') +end + +all = { + "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), + "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), + "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), + "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), +} +-%> +<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_ng/templates/pre-start.sh.erb b/jobs/cloud_controller_ng/templates/pre-start.sh.erb index 9e1c91a2a7..d0c453df70 100644 --- a/jobs/cloud_controller_ng/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_ng/templates/pre-start.sh.erb @@ -18,6 +18,28 @@ source ${CC_JOB_DIR}/bin/setup_local_blobstore.sh source "${SCRIPT_DIR}/ruby_version.sh" +write_blobstore_scope_configs() { + local cfg_json="${CONFIG_DIR}/blobstore_configs.json" + if [[ ! -f "$cfg_json" ]]; then + echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" + return 0 + fi + + # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. + CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' + require "json" + cfg_path = ENV.fetch("CFG") + dest = ENV.fetch("DEST") + data = JSON.parse(File.read(cfg_path)) + + %w[buildpacks droplets packages resource_pool].each do |k| + out = File.join(dest, "storage_cli_config_#{k}.json") + File.write(out, JSON.pretty_generate(data[k] || {})) + end +RUBY +} + + function setup_nginx_directories { mkdir -p "/var/vcap/sys/run/nginx_cc" chown -R vcap:vcap "/var/vcap/sys/run/nginx_cc" @@ -141,6 +163,7 @@ function start_consul_agent { function main { start_bosh_dns_or_consul setup_directories + write_blobstore_scope_configs <% if spec.bootstrap && p('cc.run_prestart_migrations') %> stack_check perform_migration diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb deleted file mode 100644 index 64d984b7d7..0000000000 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_buildpacks.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.buildpacks.connection_config" -provider = p("cc.buildpacks.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb deleted file mode 100644 index a29fa5cc34..0000000000 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_droplets.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.droplets.connection_config" -provider = p("cc.droplets.blobstore_provider", "AzureRM") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("cc.droplets.connection_config.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb deleted file mode 100644 index 4cf8494316..0000000000 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_packages.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.packages.connection_config" -provider = p("cc.packages.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb deleted file mode 100644 index 5b7d7ef63c..0000000000 --- a/jobs/cloud_controller_ng/templates/storage_cli_config_resource_pool.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.resource_pool.connection_config" -provider = p("cc.resource_pool.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index 8db1bbbb4b..069889b121 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -16,10 +16,7 @@ templates: newrelic.yml.erb: config/newrelic.yml drain.sh.erb: bin/drain stacks.yml.erb: config/stacks.yml - storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json - storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json - storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json - storage_cli_config_resource_pool.json.erb: config/storage_cli_config_resource_pool.json + blobstore_configs.json.erb: config/blobstore_configs.json ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console blobstore_waiter.sh.erb: bin/blobstore_waiter.sh diff --git a/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb b/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb new file mode 100644 index 0000000000..f4e3a36370 --- /dev/null +++ b/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb @@ -0,0 +1,52 @@ +<% +require "json" + +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +def options_for(scope_prefix, provider_prop) + provider = p(provider_prop, nil) + return {} unless provider == "AzureRM" + + h = {} + h["provider"] = provider + h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") + h["container_name"] = p("#{scope_prefix}.container_name") + add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) + add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) + add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope_prefix}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(h, k.to_s, v) } + end + rescue + # property might not exist; ignore + end + + cli_cfg_with_default_timeout(h, 'storage_cli') +end + +all = { + "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), + "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), + "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), + "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), +} +-%> +<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_worker/templates/pre-start.sh.erb b/jobs/cloud_controller_worker/templates/pre-start.sh.erb index dca9855e2d..508aef279e 100644 --- a/jobs/cloud_controller_worker/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_worker/templates/pre-start.sh.erb @@ -26,8 +26,30 @@ function setup_directories { chpst -u vcap:vcap chmod -R go-w $BUNDLER_DIR } +write_blobstore_scope_configs() { + local cfg_json="${CONFIG_DIR}/blobstore_configs.json" + if [[ ! -f "$cfg_json" ]]; then + echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" + return 0 + fi + + # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. + CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' + require "json" + cfg_path = ENV.fetch("CFG") + dest = ENV.fetch("DEST") + data = JSON.parse(File.read(cfg_path)) + + %w[buildpacks droplets packages resource_pool].each do |k| + out = File.join(dest, "storage_cli_config_#{k}.json") + File.write(out, JSON.pretty_generate(data[k] || {})) + end +RUBY +} + function main { setup_directories + write_blobstore_scope_configs } main diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb deleted file mode 100644 index 64d984b7d7..0000000000 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_buildpacks.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.buildpacks.connection_config" -provider = p("cc.buildpacks.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb deleted file mode 100644 index 883ab5e685..0000000000 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_droplets.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.droplets.connection_config" -provider = p("cc.droplets.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("cc.droplets.connection_config.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb deleted file mode 100644 index 4cf8494316..0000000000 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_packages.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.packages.connection_config" -provider = p("cc.packages.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb b/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb deleted file mode 100644 index 5b7d7ef63c..0000000000 --- a/jobs/cloud_controller_worker/templates/storage_cli_config_resource_pool.json.erb +++ /dev/null @@ -1,48 +0,0 @@ -<% -require "json" - - # Ensure Azure CLI connection_config has a default timeout if none is set -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -# helper: add key only when value is present -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -scope = "cc.resource_pool.connection_config" -provider = p("cc.resource_pool.blobstore_provider") - -if provider != "AzureRM" - options = {} # for now: all non-azure providers output an empty JSON object -else - options = {} - options["provider"] = provider - options["account_name"] = p("#{scope}.azure_storage_account_name") - options["container_name"] = p("#{scope}.container_name") - add(options, "account_key", p("#{scope}.azure_storage_access_key")) - add(options, "environment", p("#{scope}.environment", "AzureCloud")) - add(options, "put_timeout_in_seconds", p("#{scope}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(options, k.to_s, v) } - end - rescue - # ignore if property not defined - end - options = cli_cfg_with_default_timeout(options, 'storage_cli') -end --%> -<%= JSON.pretty_generate(options) %> \ No newline at end of file From bf0ce4621a1aaf269fa329e6721825b52d08bbda Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Tue, 21 Oct 2025 09:37:12 +0200 Subject: [PATCH 10/13] readd test --- .../cloud_controller_ng_spec.rb | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb index 4fd5ae9681..42f431d4b2 100644 --- a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb +++ b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb @@ -963,23 +963,35 @@ module Test expect(template_hash['default_app_lifecycle']).to eq('cnb') end end - end - context 'with max_service_credential_bindings_per_app_service_instance parameter' do - it 'defaults to 1' do - template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) - expect(template_hash['max_service_credential_bindings_per_app_service_instance']).to eq(1) - end + context 'with max_service_credential_bindings_per_app_service_instance parameter' do + it 'defaults to 1' do + template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) + expect(template_hash['max_service_credential_bindings_per_app_service_instance']).to eq(1) + end - context 'when set in the manifest' do - before { merged_manifest_properties['cc']['max_service_credential_bindings_per_app_service_instance'] = 5 } + context 'when set in the manifest' do + before { merged_manifest_properties['cc']['max_service_credential_bindings_per_app_service_instance'] = 5 } - it 'renders the value from the manifest' do - template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) - expect(template_hash['max_service_credential_bindings_per_app_service_instance']).to eq(5) + it 'renders the value from the manifest' do + template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) + expect(template_hash['max_service_credential_bindings_per_app_service_instance']).to eq(5) + end end end end + + describe 'storage_cli_config_file_* paths' do + let(:template) { job.template('config/cloud_controller_ng.yml') } + + it 'renders absolute paths for all four scope files' do + yaml = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) + expect(yaml['storage_cli_config_file_droplets']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_droplets.json') + expect(yaml['storage_cli_config_file_packages']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_packages.json') + expect(yaml['storage_cli_config_file_buildpacks']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_buildpacks.json') + expect(yaml['storage_cli_config_file_resource_pool']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_resource_pool.json') + end + end end end end From a3c55edb90e13dc45cd582e1295cd28f4e8ea87f Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Tue, 21 Oct 2025 10:55:40 +0200 Subject: [PATCH 11/13] adapt test for new json.erb --- .../storage_cli_config_jsons_spec.rb | 100 ++++++++++-------- 1 file changed, 56 insertions(+), 44 deletions(-) diff --git a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb index c1fdd84820..f16a292b0a 100644 --- a/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb +++ b/spec/cloud_controller_ng/storage_cli_config_jsons_spec.rb @@ -5,10 +5,10 @@ require 'bosh/template/test' TEMPLATES = { - droplets: ['config/storage_cli_config_droplets.json', %w[cc droplets connection_config]], - buildpacks: ['config/storage_cli_config_buildpacks.json', %w[cc buildpacks connection_config]], - packages: ['config/storage_cli_config_packages.json', %w[cc packages connection_config]], - resource_pool: ['config/storage_cli_config_resource_pool.json', %w[cc resource_pool connection_config]] + droplets: ['config/blobstore_configs.json', %w[cc droplets connection_config]], + buildpacks: ['config/blobstore_configs.json', %w[cc buildpacks connection_config]], + packages: ['config/blobstore_configs.json', %w[cc packages connection_config]], + resource_pool: ['config/blobstore_configs.json', %w[cc resource_pool connection_config]] }.freeze module Bosh @@ -16,21 +16,23 @@ module Template module Test RSpec.describe 'storage-cli JSON templates' do let(:release_path) { File.join(File.dirname(__FILE__), '../..') } - let(:release) { ReleaseDir.new(release_path) } - let(:job) { release.job('cloud_controller_ng') } - let(:links) { {} } + let(:release) { ReleaseDir.new(release_path) } + let(:job) { release.job('cloud_controller_ng') } + let(:links) { {} } + + # Default all scopes to non-Azure so only the scope under test must be populated let(:props) do { 'cc' => { - 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' }, - 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'AzureRM' } + 'droplets' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, + 'buildpacks' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, + 'packages' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' }, + 'resource_pool' => { 'connection_config' => {}, 'blobstore_provider' => 'S3' } } } end - TEMPLATES.each_value do |(template_path, keypath)| + TEMPLATES.each do |scope, (template_path, keypath)| describe template_path do let(:template) { job.template(template_path) } @@ -40,43 +42,53 @@ def set(hash, path, value) cursor[path.last] = value end - it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '' - }) + context "when provider is AzureRM for #{scope}" do + before do + props['cc'][scope.to_s]['blobstore_provider'] = 'AzureRM' + end - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to include( - 'provider' => 'AzureRM', - 'account_name' => 'acc', - 'account_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '41' - ) - end + it 'renders and normalizes put_timeout_in_seconds to "41" when blank' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '' + }) - it 'keeps existing put_timeout_in_seconds when provided' do - set(props, keypath, { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => 'acc', - 'azure_storage_access_key' => 'key', - 'container_name' => 'cont', - 'put_timeout_in_seconds' => '7' - }) + full = YAML.safe_load(template.render(props, consumes: links)) + json = full.fetch(scope.to_s) + expect(json).to include( + 'provider' => 'AzureRM', + 'account_name' => 'acc', + 'account_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '41' + ) + end - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json['put_timeout_in_seconds']).to eq('7') - end + it 'keeps existing put_timeout_in_seconds when provided' do + set(props, keypath, { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => 'acc', + 'azure_storage_access_key' => 'key', + 'container_name' => 'cont', + 'put_timeout_in_seconds' => '7' + }) - it 'renders {} for non-Azure providers' do - keypath[0..-2].reduce(props) { |acc, elem| acc[elem] ||= {} }['blobstore_provider'] = 'S3' + full = YAML.safe_load(template.render(props, consumes: links)) + json = full.fetch(scope.to_s) + expect(json['put_timeout_in_seconds']).to eq('7') + end + end - json = YAML.safe_load(template.render(props, consumes: links)) - expect(json).to eq({}) + context "when provider is non-Azure for #{scope}" do + it 'renders {}' do + # leave default 'S3' for this scope + full = YAML.safe_load(template.render(props, consumes: links)) + json = full.fetch(scope.to_s) + expect(json).to eq({}) + end end end end From 0132cad5f79cbaa96841c8a98b8952a2d20a9218 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Tue, 21 Oct 2025 16:54:52 +0200 Subject: [PATCH 12/13] Revert "Merge branch 'develop' into azure-storage-cli-connection-config" This reverts commit 815fb2df2cf459ceb204b5f1c68c290b31ee6987, reversing changes made to 01c1bb63aba9b8ac24ac51c430df0733ef4ad30b. --- jobs/cloud_controller_clock/spec | 17 ------ .../templates/blobstore_configs.json.erb | 52 ------------------- .../templates/cloud_controller_ng.yml.erb | 6 --- .../templates/pre-start.sh.erb | 22 -------- jobs/cloud_controller_ng/spec | 25 --------- .../templates/blobstore_configs.json.erb | 52 ------------------- .../templates/cloud_controller_ng.yml.erb | 5 -- .../templates/pre-start.sh.erb | 23 -------- jobs/cloud_controller_worker/spec | 17 ------ .../templates/blobstore_configs.json.erb | 52 ------------------- .../templates/cloud_controller_ng.yml.erb | 4 -- .../templates/pre-start.sh.erb | 22 -------- .../cloud_controller_ng_spec.rb | 8 +-- 13 files changed, 4 insertions(+), 301 deletions(-) delete mode 100644 jobs/cloud_controller_clock/templates/blobstore_configs.json.erb delete mode 100644 jobs/cloud_controller_ng/templates/blobstore_configs.json.erb delete mode 100644 jobs/cloud_controller_worker/templates/blobstore_configs.json.erb diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index d476c06397..1f2f88ed63 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -14,7 +14,6 @@ templates: cloud_controller_ng.yml.erb: config/cloud_controller_ng.yml newrelic.yml.erb: config/newrelic.yml stacks.yml.erb: config/stacks.yml - blobstore_configs.json.erb: config/blobstore_configs.json drain.sh.erb: bin/drain ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console @@ -173,8 +172,6 @@ properties: cc.staging_upload_password: description: "User's password used to access internal endpoints of Cloud Controller to upload files when staging" - cc.resource_pool.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -220,11 +217,7 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.resource_pool.connection_config: - description: "Azure Storage Cli connection hash" - cc.packages.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -267,11 +260,7 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.packages.connection_config: - description: "Azure Storage Cli connection hash" - cc.droplets.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -311,11 +300,7 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.droplets.connection_config: - description: "Azure Storage Cli connection hash" - cc.buildpacks.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -355,8 +340,6 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.buildpacks.connection_config: - description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb b/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb deleted file mode 100644 index f4e3a36370..0000000000 --- a/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb +++ /dev/null @@ -1,52 +0,0 @@ -<% -require "json" - -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -def options_for(scope_prefix, provider_prop) - provider = p(provider_prop, nil) - return {} unless provider == "AzureRM" - - h = {} - h["provider"] = provider - h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") - h["container_name"] = p("#{scope_prefix}.container_name") - add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) - add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) - add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope_prefix}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(h, k.to_s, v) } - end - rescue - # property might not exist; ignore - end - - cli_cfg_with_default_timeout(h, 'storage_cli') -end - -all = { - "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), - "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), - "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), - "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), -} --%> -<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index ad3e416281..71bcb24cc4 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -38,12 +38,6 @@ readiness_port: clock: <%= p("cc.readiness_port.clock") %> pid_filename: /var/vcap/sys/run/cloud_controller_clock/cloud_controller_clock.pid - -storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_droplets.json -storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_buildpacks.json -storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_packages.json -storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_resource_pool.json - newrelic_enabled: false development_mode: false diff --git a/jobs/cloud_controller_clock/templates/pre-start.sh.erb b/jobs/cloud_controller_clock/templates/pre-start.sh.erb index a1287a685e..ff5912e8a8 100644 --- a/jobs/cloud_controller_clock/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_clock/templates/pre-start.sh.erb @@ -23,30 +23,8 @@ function setup_directories { chown -R vcap:vcap "$LOG_DIR" } -write_blobstore_scope_configs() { - local cfg_json="${CONFIG_DIR}/blobstore_configs.json" - if [[ ! -f "$cfg_json" ]]; then - echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" - return 0 - fi - - # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. - CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' - require "json" - cfg_path = ENV.fetch("CFG") - dest = ENV.fetch("DEST") - data = JSON.parse(File.read(cfg_path)) - - %w[buildpacks droplets packages resource_pool].each do |k| - out = File.join(dest, "storage_cli_config_#{k}.json") - File.write(out, JSON.pretty_generate(data[k] || {})) - end -RUBY -} - function main { setup_directories - write_blobstore_scope_configs } main diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index 3a1c4d9131..de1ec2bbbd 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -53,7 +53,6 @@ templates: restart_drain.sh.erb: bin/restart_drain resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem shutdown_drain.rb.erb: bin/shutdown_drain - blobstore_configs.json.erb: config/blobstore_configs.json ruby_version.sh.erb: bin/ruby_version.sh seed_db.sh.erb: bin/seed_db stack_check.sh.erb: bin/stack_check @@ -116,7 +115,6 @@ provides: - name: cloud_controller_internal type: cloud_controller_internal properties: - - cc.buildpacks.blobstore_provider - cc.buildpacks.blobstore_type - cc.buildpacks.buildpack_directory_key - cc.buildpacks.cdn.key_pair_id @@ -145,7 +143,6 @@ provides: - cc.default_stack - cc.default_app_lifecycle - cc.disable_private_domain_cross_space_context_path_route_sharing - - cc.droplets.blobstore_provider - cc.droplets.blobstore_type - cc.droplets.cdn.key_pair_id - cc.droplets.cdn.private_key @@ -175,7 +172,6 @@ provides: - cc.max_annotations_per_resource - cc.maximum_health_check_timeout - cc.packages.app_package_directory_key - - cc.packages.blobstore_provider - cc.packages.blobstore_type - cc.packages.cdn.key_pair_id - cc.packages.cdn.private_key @@ -191,7 +187,6 @@ provides: - cc.packages.webdav_config.private_endpoint - cc.packages.webdav_config.public_endpoint - cc.packages.webdav_config.username - - cc.resource_pool.blobstore_provider - cc.resource_pool.blobstore_type - cc.resource_pool.cdn.key_pair_id - cc.resource_pool.cdn.private_key @@ -236,10 +231,6 @@ provides: - cc.temporary_enable_deprecated_thin_webserver - cc.custom_root_links - cc.feature_flag_overrides - - cc.resource_pool.connection_config - - cc.packages.connection_config - - cc.droplets.connection_config - - cc.buildpacks.connection_config consumes: - name: database @@ -521,8 +512,6 @@ properties: default: default description: "The name of the quota definition CC will fallback on for org and space limits from the list of quota definitions." - cc.resource_pool.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -570,11 +559,7 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.resource_pool.connection_config: - description: "Azure Storage Cli connection hash" - cc.packages.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -622,11 +607,7 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.packages.connection_config: - description: "Azure Storage Cli connection hash" - cc.droplets.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -671,11 +652,7 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.droplets.connection_config: - description: "Azure Storage Cli connection hash" - cc.buildpacks.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -717,8 +694,6 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.buildpacks.connection_config: - description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb b/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb deleted file mode 100644 index f4e3a36370..0000000000 --- a/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb +++ /dev/null @@ -1,52 +0,0 @@ -<% -require "json" - -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -def options_for(scope_prefix, provider_prop) - provider = p(provider_prop, nil) - return {} unless provider == "AzureRM" - - h = {} - h["provider"] = provider - h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") - h["container_name"] = p("#{scope_prefix}.container_name") - add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) - add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) - add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope_prefix}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(h, k.to_s, v) } - end - rescue - # property might not exist; ignore - end - - cli_cfg_with_default_timeout(h, 'storage_cli') -end - -all = { - "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), - "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), - "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), - "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), -} --%> -<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb index ef4864eeb9..ebd6ac7719 100644 --- a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb @@ -312,11 +312,6 @@ default_health_check_timeout: <%= p("cc.default_health_check_timeout") %> maximum_health_check_timeout: <%= p("cc.maximum_health_check_timeout") %> stacks_file: /var/vcap/jobs/cloud_controller_ng/config/stacks.yml -storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_droplets.json -storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_buildpacks.json -storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_packages.json -storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_resource_pool.json - shared_isolation_segment_name: <%= p("cc.shared_isolation_segment_name") %> diff --git a/jobs/cloud_controller_ng/templates/pre-start.sh.erb b/jobs/cloud_controller_ng/templates/pre-start.sh.erb index d0c453df70..9e1c91a2a7 100644 --- a/jobs/cloud_controller_ng/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_ng/templates/pre-start.sh.erb @@ -18,28 +18,6 @@ source ${CC_JOB_DIR}/bin/setup_local_blobstore.sh source "${SCRIPT_DIR}/ruby_version.sh" -write_blobstore_scope_configs() { - local cfg_json="${CONFIG_DIR}/blobstore_configs.json" - if [[ ! -f "$cfg_json" ]]; then - echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" - return 0 - fi - - # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. - CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' - require "json" - cfg_path = ENV.fetch("CFG") - dest = ENV.fetch("DEST") - data = JSON.parse(File.read(cfg_path)) - - %w[buildpacks droplets packages resource_pool].each do |k| - out = File.join(dest, "storage_cli_config_#{k}.json") - File.write(out, JSON.pretty_generate(data[k] || {})) - end -RUBY -} - - function setup_nginx_directories { mkdir -p "/var/vcap/sys/run/nginx_cc" chown -R vcap:vcap "/var/vcap/sys/run/nginx_cc" @@ -163,7 +141,6 @@ function start_consul_agent { function main { start_bosh_dns_or_consul setup_directories - write_blobstore_scope_configs <% if spec.bootstrap && p('cc.run_prestart_migrations') %> stack_check perform_migration diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index 63a3038aa6..e6997d43b7 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -16,7 +16,6 @@ templates: newrelic.yml.erb: config/newrelic.yml drain.sh.erb: bin/drain stacks.yml.erb: config/stacks.yml - blobstore_configs.json.erb: config/blobstore_configs.json ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console blobstore_waiter.sh.erb: bin/blobstore_waiter.sh @@ -140,8 +139,6 @@ properties: cc.staging_upload_password: description: "User's password used to access internal endpoints of Cloud Controller to upload files when staging" - cc.resource_pool.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -187,11 +184,7 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.resource_pool.connection_config: - description: "Azure Storage Cli connection hash" - cc.packages.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -234,11 +227,7 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.packages.connection_config: - description: "Azure Storage Cli connection hash" - cc.droplets.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -278,11 +267,7 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.droplets.connection_config: - description: "Azure Storage Cli connection hash" - cc.buildpacks.blobstore_provider: - description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -322,8 +307,6 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" - cc.buildpacks.connection_config: - description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb b/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb deleted file mode 100644 index f4e3a36370..0000000000 --- a/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb +++ /dev/null @@ -1,52 +0,0 @@ -<% -require "json" - -def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) - cfg = (connection_cfg || {}).dup - if blobstore_type == 'storage_cli' - if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? - cfg['put_timeout_in_seconds'] = default_seconds.to_s - end - end - cfg -end - -def add(h, key, val) - return if val.nil? - return if val.respond_to?(:empty?) && val.empty? - h[key] = val -end - -def options_for(scope_prefix, provider_prop) - provider = p(provider_prop, nil) - return {} unless provider == "AzureRM" - - h = {} - h["provider"] = provider - h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") - h["container_name"] = p("#{scope_prefix}.container_name") - add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) - add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) - add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) - - # optional passthrough for extra storage-cli flags - begin - custom = p("#{scope_prefix}.custom", {}) - if custom.respond_to?(:each) - custom.each { |k, v| add(h, k.to_s, v) } - end - rescue - # property might not exist; ignore - end - - cli_cfg_with_default_timeout(h, 'storage_cli') -end - -all = { - "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), - "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), - "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), - "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), -} --%> -<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb index 7f375dd652..6ff07aae79 100644 --- a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb @@ -154,10 +154,6 @@ default_health_check_timeout: <%= p("cc.default_health_check_timeout") %> maximum_health_check_timeout: <%= p("cc.maximum_health_check_timeout") %> stacks_file: /var/vcap/jobs/cloud_controller_worker/config/stacks.yml -storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_droplets.json -storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_buildpacks.json -storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_packages.json -storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_resource_pool.json resource_pool: blobstore_type: <%= p("cc.resource_pool.blobstore_type") %> diff --git a/jobs/cloud_controller_worker/templates/pre-start.sh.erb b/jobs/cloud_controller_worker/templates/pre-start.sh.erb index 508aef279e..dca9855e2d 100644 --- a/jobs/cloud_controller_worker/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_worker/templates/pre-start.sh.erb @@ -26,30 +26,8 @@ function setup_directories { chpst -u vcap:vcap chmod -R go-w $BUNDLER_DIR } -write_blobstore_scope_configs() { - local cfg_json="${CONFIG_DIR}/blobstore_configs.json" - if [[ ! -f "$cfg_json" ]]; then - echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" - return 0 - fi - - # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. - CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' - require "json" - cfg_path = ENV.fetch("CFG") - dest = ENV.fetch("DEST") - data = JSON.parse(File.read(cfg_path)) - - %w[buildpacks droplets packages resource_pool].each do |k| - out = File.join(dest, "storage_cli_config_#{k}.json") - File.write(out, JSON.pretty_generate(data[k] || {})) - end -RUBY -} - function main { setup_directories - write_blobstore_scope_configs } main diff --git a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb index 42f431d4b2..5e5f730c2f 100644 --- a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb +++ b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb @@ -942,14 +942,14 @@ module Test end end - context 'when it is set to buildpack' do + context 'when it is set to buidpack' do before do - merged_manifest_properties['cc']['default_app_lifecycle'] = 'buildpack' + merged_manifest_properties['cc']['default_app_lifecycle'] = 'buidpack' end - it 'renders it as buildpack' do + it 'renders it as buidpack' do template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) - expect(template_hash['default_app_lifecycle']).to eq('buildpack') + expect(template_hash['default_app_lifecycle']).to eq('buidpack') end end From fd72857bce8c4294971498bb6735d8cadf70e705 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Tue, 21 Oct 2025 16:56:34 +0200 Subject: [PATCH 13/13] Reapply "Merge branch 'develop' into azure-storage-cli-connection-config" This reverts commit 0132cad5f79cbaa96841c8a98b8952a2d20a9218. --- jobs/cloud_controller_clock/spec | 17 ++++++ .../templates/blobstore_configs.json.erb | 52 +++++++++++++++++++ .../templates/cloud_controller_ng.yml.erb | 6 +++ .../templates/pre-start.sh.erb | 22 ++++++++ jobs/cloud_controller_ng/spec | 25 +++++++++ .../templates/blobstore_configs.json.erb | 52 +++++++++++++++++++ .../templates/cloud_controller_ng.yml.erb | 5 ++ .../templates/pre-start.sh.erb | 23 ++++++++ jobs/cloud_controller_worker/spec | 17 ++++++ .../templates/blobstore_configs.json.erb | 52 +++++++++++++++++++ .../templates/cloud_controller_ng.yml.erb | 4 ++ .../templates/pre-start.sh.erb | 22 ++++++++ .../cloud_controller_ng_spec.rb | 8 +-- 13 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 jobs/cloud_controller_clock/templates/blobstore_configs.json.erb create mode 100644 jobs/cloud_controller_ng/templates/blobstore_configs.json.erb create mode 100644 jobs/cloud_controller_worker/templates/blobstore_configs.json.erb diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index 1f2f88ed63..d476c06397 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -14,6 +14,7 @@ templates: cloud_controller_ng.yml.erb: config/cloud_controller_ng.yml newrelic.yml.erb: config/newrelic.yml stacks.yml.erb: config/stacks.yml + blobstore_configs.json.erb: config/blobstore_configs.json drain.sh.erb: bin/drain ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console @@ -172,6 +173,8 @@ properties: cc.staging_upload_password: description: "User's password used to access internal endpoints of Cloud Controller to upload files when staging" + cc.resource_pool.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -217,7 +220,11 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.resource_pool.connection_config: + description: "Azure Storage Cli connection hash" + cc.packages.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -260,7 +267,11 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.packages.connection_config: + description: "Azure Storage Cli connection hash" + cc.droplets.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -300,7 +311,11 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.droplets.connection_config: + description: "Azure Storage Cli connection hash" + cc.buildpacks.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -340,6 +355,8 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.buildpacks.connection_config: + description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb b/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb new file mode 100644 index 0000000000..f4e3a36370 --- /dev/null +++ b/jobs/cloud_controller_clock/templates/blobstore_configs.json.erb @@ -0,0 +1,52 @@ +<% +require "json" + +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +def options_for(scope_prefix, provider_prop) + provider = p(provider_prop, nil) + return {} unless provider == "AzureRM" + + h = {} + h["provider"] = provider + h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") + h["container_name"] = p("#{scope_prefix}.container_name") + add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) + add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) + add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope_prefix}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(h, k.to_s, v) } + end + rescue + # property might not exist; ignore + end + + cli_cfg_with_default_timeout(h, 'storage_cli') +end + +all = { + "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), + "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), + "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), + "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), +} +-%> +<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index 71bcb24cc4..ad3e416281 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -38,6 +38,12 @@ readiness_port: clock: <%= p("cc.readiness_port.clock") %> pid_filename: /var/vcap/sys/run/cloud_controller_clock/cloud_controller_clock.pid + +storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_droplets.json +storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_buildpacks.json +storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_packages.json +storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_clock/config/storage_cli_config_resource_pool.json + newrelic_enabled: false development_mode: false diff --git a/jobs/cloud_controller_clock/templates/pre-start.sh.erb b/jobs/cloud_controller_clock/templates/pre-start.sh.erb index ff5912e8a8..a1287a685e 100644 --- a/jobs/cloud_controller_clock/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_clock/templates/pre-start.sh.erb @@ -23,8 +23,30 @@ function setup_directories { chown -R vcap:vcap "$LOG_DIR" } +write_blobstore_scope_configs() { + local cfg_json="${CONFIG_DIR}/blobstore_configs.json" + if [[ ! -f "$cfg_json" ]]; then + echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" + return 0 + fi + + # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. + CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' + require "json" + cfg_path = ENV.fetch("CFG") + dest = ENV.fetch("DEST") + data = JSON.parse(File.read(cfg_path)) + + %w[buildpacks droplets packages resource_pool].each do |k| + out = File.join(dest, "storage_cli_config_#{k}.json") + File.write(out, JSON.pretty_generate(data[k] || {})) + end +RUBY +} + function main { setup_directories + write_blobstore_scope_configs } main diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index de1ec2bbbd..3a1c4d9131 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -53,6 +53,7 @@ templates: restart_drain.sh.erb: bin/restart_drain resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem shutdown_drain.rb.erb: bin/shutdown_drain + blobstore_configs.json.erb: config/blobstore_configs.json ruby_version.sh.erb: bin/ruby_version.sh seed_db.sh.erb: bin/seed_db stack_check.sh.erb: bin/stack_check @@ -115,6 +116,7 @@ provides: - name: cloud_controller_internal type: cloud_controller_internal properties: + - cc.buildpacks.blobstore_provider - cc.buildpacks.blobstore_type - cc.buildpacks.buildpack_directory_key - cc.buildpacks.cdn.key_pair_id @@ -143,6 +145,7 @@ provides: - cc.default_stack - cc.default_app_lifecycle - cc.disable_private_domain_cross_space_context_path_route_sharing + - cc.droplets.blobstore_provider - cc.droplets.blobstore_type - cc.droplets.cdn.key_pair_id - cc.droplets.cdn.private_key @@ -172,6 +175,7 @@ provides: - cc.max_annotations_per_resource - cc.maximum_health_check_timeout - cc.packages.app_package_directory_key + - cc.packages.blobstore_provider - cc.packages.blobstore_type - cc.packages.cdn.key_pair_id - cc.packages.cdn.private_key @@ -187,6 +191,7 @@ provides: - cc.packages.webdav_config.private_endpoint - cc.packages.webdav_config.public_endpoint - cc.packages.webdav_config.username + - cc.resource_pool.blobstore_provider - cc.resource_pool.blobstore_type - cc.resource_pool.cdn.key_pair_id - cc.resource_pool.cdn.private_key @@ -231,6 +236,10 @@ provides: - cc.temporary_enable_deprecated_thin_webserver - cc.custom_root_links - cc.feature_flag_overrides + - cc.resource_pool.connection_config + - cc.packages.connection_config + - cc.droplets.connection_config + - cc.buildpacks.connection_config consumes: - name: database @@ -512,6 +521,8 @@ properties: default: default description: "The name of the quota definition CC will fallback on for org and space limits from the list of quota definitions." + cc.resource_pool.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -559,7 +570,11 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.resource_pool.connection_config: + description: "Azure Storage Cli connection hash" + cc.packages.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -607,7 +622,11 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.packages.connection_config: + description: "Azure Storage Cli connection hash" + cc.droplets.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -652,7 +671,11 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.droplets.connection_config: + description: "Azure Storage Cli connection hash" + cc.buildpacks.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -694,6 +717,8 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.buildpacks.connection_config: + description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb b/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb new file mode 100644 index 0000000000..f4e3a36370 --- /dev/null +++ b/jobs/cloud_controller_ng/templates/blobstore_configs.json.erb @@ -0,0 +1,52 @@ +<% +require "json" + +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +def options_for(scope_prefix, provider_prop) + provider = p(provider_prop, nil) + return {} unless provider == "AzureRM" + + h = {} + h["provider"] = provider + h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") + h["container_name"] = p("#{scope_prefix}.container_name") + add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) + add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) + add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope_prefix}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(h, k.to_s, v) } + end + rescue + # property might not exist; ignore + end + + cli_cfg_with_default_timeout(h, 'storage_cli') +end + +all = { + "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), + "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), + "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), + "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), +} +-%> +<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb index ebd6ac7719..ef4864eeb9 100644 --- a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb @@ -312,6 +312,11 @@ default_health_check_timeout: <%= p("cc.default_health_check_timeout") %> maximum_health_check_timeout: <%= p("cc.maximum_health_check_timeout") %> stacks_file: /var/vcap/jobs/cloud_controller_ng/config/stacks.yml +storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_droplets.json +storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_buildpacks.json +storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_packages.json +storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_resource_pool.json + shared_isolation_segment_name: <%= p("cc.shared_isolation_segment_name") %> diff --git a/jobs/cloud_controller_ng/templates/pre-start.sh.erb b/jobs/cloud_controller_ng/templates/pre-start.sh.erb index 9e1c91a2a7..d0c453df70 100644 --- a/jobs/cloud_controller_ng/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_ng/templates/pre-start.sh.erb @@ -18,6 +18,28 @@ source ${CC_JOB_DIR}/bin/setup_local_blobstore.sh source "${SCRIPT_DIR}/ruby_version.sh" +write_blobstore_scope_configs() { + local cfg_json="${CONFIG_DIR}/blobstore_configs.json" + if [[ ! -f "$cfg_json" ]]; then + echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" + return 0 + fi + + # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. + CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' + require "json" + cfg_path = ENV.fetch("CFG") + dest = ENV.fetch("DEST") + data = JSON.parse(File.read(cfg_path)) + + %w[buildpacks droplets packages resource_pool].each do |k| + out = File.join(dest, "storage_cli_config_#{k}.json") + File.write(out, JSON.pretty_generate(data[k] || {})) + end +RUBY +} + + function setup_nginx_directories { mkdir -p "/var/vcap/sys/run/nginx_cc" chown -R vcap:vcap "/var/vcap/sys/run/nginx_cc" @@ -141,6 +163,7 @@ function start_consul_agent { function main { start_bosh_dns_or_consul setup_directories + write_blobstore_scope_configs <% if spec.bootstrap && p('cc.run_prestart_migrations') %> stack_check perform_migration diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index e6997d43b7..63a3038aa6 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -16,6 +16,7 @@ templates: newrelic.yml.erb: config/newrelic.yml drain.sh.erb: bin/drain stacks.yml.erb: config/stacks.yml + blobstore_configs.json.erb: config/blobstore_configs.json ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console blobstore_waiter.sh.erb: bin/blobstore_waiter.sh @@ -139,6 +140,8 @@ properties: cc.staging_upload_password: description: "User's password used to access internal endpoints of Cloud Controller to upload files when staging" + cc.resource_pool.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.resource_pool.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -184,7 +187,11 @@ properties: cc.resource_pool.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.resource_pool.connection_config: + description: "Azure Storage Cli connection hash" + cc.packages.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.packages.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -227,7 +234,11 @@ properties: cc.packages.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.packages.connection_config: + description: "Azure Storage Cli connection hash" + cc.droplets.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.droplets.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -267,7 +278,11 @@ properties: cc.droplets.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.droplets.connection_config: + description: "Azure Storage Cli connection hash" + cc.buildpacks.blobstore_provider: + description: "The provider of blobstore storage cli to use. Valid values: ['AzureRM']" cc.buildpacks.blobstore_type: description: "The type of blobstore backing to use. Valid values: ['fog', 'webdav']" default: "fog" @@ -307,6 +322,8 @@ properties: cc.buildpacks.cdn.key_pair_id: description: "Key pair name for signed download URIs" default: "" + cc.buildpacks.connection_config: + description: "Azure Storage Cli connection hash" ccdb.databases: description: "Contains the name of the database on the database server" diff --git a/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb b/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb new file mode 100644 index 0000000000..f4e3a36370 --- /dev/null +++ b/jobs/cloud_controller_worker/templates/blobstore_configs.json.erb @@ -0,0 +1,52 @@ +<% +require "json" + +def cli_cfg_with_default_timeout(connection_cfg, blobstore_type, default_seconds: 41) + cfg = (connection_cfg || {}).dup + if blobstore_type == 'storage_cli' + if !cfg.key?('put_timeout_in_seconds') || cfg['put_timeout_in_seconds'].to_s.empty? + cfg['put_timeout_in_seconds'] = default_seconds.to_s + end + end + cfg +end + +def add(h, key, val) + return if val.nil? + return if val.respond_to?(:empty?) && val.empty? + h[key] = val +end + +def options_for(scope_prefix, provider_prop) + provider = p(provider_prop, nil) + return {} unless provider == "AzureRM" + + h = {} + h["provider"] = provider + h["account_name"] = p("#{scope_prefix}.azure_storage_account_name") + h["container_name"] = p("#{scope_prefix}.container_name") + add(h, "account_key", p("#{scope_prefix}.azure_storage_access_key")) + add(h, "environment", p("#{scope_prefix}.environment", "AzureCloud")) + add(h, "put_timeout_in_seconds", p("#{scope_prefix}.put_timeout_in_seconds", nil)) + + # optional passthrough for extra storage-cli flags + begin + custom = p("#{scope_prefix}.custom", {}) + if custom.respond_to?(:each) + custom.each { |k, v| add(h, k.to_s, v) } + end + rescue + # property might not exist; ignore + end + + cli_cfg_with_default_timeout(h, 'storage_cli') +end + +all = { + "buildpacks" => options_for("cc.buildpacks.connection_config", "cc.buildpacks.blobstore_provider"), + "droplets" => options_for("cc.droplets.connection_config", "cc.droplets.blobstore_provider"), + "packages" => options_for("cc.packages.connection_config", "cc.packages.blobstore_provider"), + "resource_pool" => options_for("cc.resource_pool.connection_config", "cc.resource_pool.blobstore_provider"), +} +-%> +<%= JSON.pretty_generate(all) %> diff --git a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb index 6ff07aae79..7f375dd652 100644 --- a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb @@ -154,6 +154,10 @@ default_health_check_timeout: <%= p("cc.default_health_check_timeout") %> maximum_health_check_timeout: <%= p("cc.maximum_health_check_timeout") %> stacks_file: /var/vcap/jobs/cloud_controller_worker/config/stacks.yml +storage_cli_config_file_droplets: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_droplets.json +storage_cli_config_file_buildpacks: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_buildpacks.json +storage_cli_config_file_packages: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_packages.json +storage_cli_config_file_resource_pool: /var/vcap/jobs/cloud_controller_worker/config/storage_cli_config_resource_pool.json resource_pool: blobstore_type: <%= p("cc.resource_pool.blobstore_type") %> diff --git a/jobs/cloud_controller_worker/templates/pre-start.sh.erb b/jobs/cloud_controller_worker/templates/pre-start.sh.erb index dca9855e2d..508aef279e 100644 --- a/jobs/cloud_controller_worker/templates/pre-start.sh.erb +++ b/jobs/cloud_controller_worker/templates/pre-start.sh.erb @@ -26,8 +26,30 @@ function setup_directories { chpst -u vcap:vcap chmod -R go-w $BUNDLER_DIR } +write_blobstore_scope_configs() { + local cfg_json="${CONFIG_DIR}/blobstore_configs.json" + if [[ ! -f "$cfg_json" ]]; then + echo "blobstore_configs.json not found at $cfg_json; skipping fan-out" + return 0 + fi + + # Use the packaged Ruby if you prefer; system Ruby is usually fine on CF VMs. + CFG="$cfg_json" DEST="$BUNDLER_DIR" ruby <<'RUBY' + require "json" + cfg_path = ENV.fetch("CFG") + dest = ENV.fetch("DEST") + data = JSON.parse(File.read(cfg_path)) + + %w[buildpacks droplets packages resource_pool].each do |k| + out = File.join(dest, "storage_cli_config_#{k}.json") + File.write(out, JSON.pretty_generate(data[k] || {})) + end +RUBY +} + function main { setup_directories + write_blobstore_scope_configs } main diff --git a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb index 5e5f730c2f..42f431d4b2 100644 --- a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb +++ b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb @@ -942,14 +942,14 @@ module Test end end - context 'when it is set to buidpack' do + context 'when it is set to buildpack' do before do - merged_manifest_properties['cc']['default_app_lifecycle'] = 'buidpack' + merged_manifest_properties['cc']['default_app_lifecycle'] = 'buildpack' end - it 'renders it as buidpack' do + it 'renders it as buildpack' do template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) - expect(template_hash['default_app_lifecycle']).to eq('buidpack') + expect(template_hash['default_app_lifecycle']).to eq('buildpack') end end