From 06ff14e28a53fc07ee77d701fc5f186c5ae1aad3 Mon Sep 17 00:00:00 2001 From: Johnny Shum Date: Thu, 29 Sep 2022 16:08:40 -0500 Subject: [PATCH 01/23] initial checkin --- .../wlsdeploy/aliases/model_constants.py | 2 + .../tool/deploy/applications_deployer.py | 46 +++++++++++++++---- .../deploy/applications_version_helper.py | 10 +++- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/core/src/main/python/wlsdeploy/aliases/model_constants.py b/core/src/main/python/wlsdeploy/aliases/model_constants.py index 57ebb37b2..e882f596f 100644 --- a/core/src/main/python/wlsdeploy/aliases/model_constants.py +++ b/core/src/main/python/wlsdeploy/aliases/model_constants.py @@ -192,6 +192,7 @@ MESSAGE_LOGGING_PARAMS = 'MessageLoggingParams' MESSAGING_BRIDGE = 'MessagingBridge' METHOD = 'Method' +MODULE_TYPE = 'ModuleType' MULTICAST = 'Multicast' MULTICAST_ADDRESS = 'MulticastAddress' MULTICAST_PORT = 'MulticastPort' @@ -287,6 +288,7 @@ STORE = 'Store' SUB_DEPLOYMENT = 'SubDeployment' SUB_DEPLOYMENT_NAME = 'SubDeploymentName' +SUB_MODULE_TARGETS = 'subModuleTargets' SYSTEM_COMPONENT = 'SystemComponent' SYSTEM_PASSWORD_VALIDATOR = 'SystemPasswordValidator' TARGET = 'Target' diff --git a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py index d0253d04a..91de80e71 100644 --- a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py +++ b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py @@ -28,6 +28,7 @@ from wlsdeploy.aliases.model_constants import APPLICATION from wlsdeploy.aliases.model_constants import DEPLOYMENT_ORDER from wlsdeploy.aliases.model_constants import LIBRARY +from wlsdeploy.aliases.model_constants import MODULE_TYPE from wlsdeploy.aliases.model_constants import PARTITION from wlsdeploy.aliases.model_constants import PLAN_DIR from wlsdeploy.aliases.model_constants import PLAN_PATH @@ -38,6 +39,8 @@ from wlsdeploy.aliases.model_constants import SECURITY_DD_MODEL from wlsdeploy.aliases.model_constants import SOURCE_PATH from wlsdeploy.aliases.model_constants import STAGE_MODE +from wlsdeploy.aliases.model_constants import SUB_DEPLOYMENT +from wlsdeploy.aliases.model_constants import SUB_MODULE_TARGETS from wlsdeploy.aliases.model_constants import TARGET from wlsdeploy.aliases.model_constants import TARGETS from wlsdeploy.aliases.wlst_modes import WlstModes @@ -192,10 +195,11 @@ def __add_applications(self): self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name) raise ex + module_type = dictionary_utils.get_element(application, MODULE_TYPE) application_name = \ - self.version_helper.get_application_versioned_name(app_source_path, application_name) - + self.version_helper.get_application_versioned_name(app_source_path, application_name, + module_type=module_type) quoted_application_name = self.wlst_helper.get_quoted_name_for_wlst(application_name) application_location.add_name_token(application_token, quoted_application_name) @@ -935,7 +939,8 @@ def __deploy_model_libraries(self, model_libs, lib_location): plan_file = dictionary_utils.get_element(lib_dict, PLAN_PATH) targets = dictionary_utils.get_element(lib_dict, TARGET) stage_mode = dictionary_utils.get_element(lib_dict, STAGE_MODE) - options = _get_deploy_options(model_libs, lib_name, library_module='true') + options, sub_module_targets = _get_deploy_options(model_libs, lib_name, library_module='true', + application_version_helper=self.version_helper) for uses_path_tokens_attribute_name in uses_path_tokens_attribute_names: if uses_path_tokens_attribute_name in lib_dict: path = lib_dict[uses_path_tokens_attribute_name] @@ -963,7 +968,8 @@ def __deploy_model_applications(self, model_apps, app_location, deployed_applist plan_file = dictionary_utils.get_element(app_dict, PLAN_PATH) stage_mode = dictionary_utils.get_element(app_dict, STAGE_MODE) targets = dictionary_utils.get_element(app_dict, TARGET) - options = _get_deploy_options(model_apps, app_name, library_module='false') + options, sub_module_targets = _get_deploy_options(model_apps, app_name, library_module='false', + application_version_helper=self.version_helper) # any attribute with 'uses_path_tokens' may be in the archive (such as SourcePath) for uses_path_tokens_attribute_name in uses_path_tokens_attribute_names: @@ -975,10 +981,14 @@ def __deploy_model_applications(self, model_apps, app_location, deployed_applist resource_group_template_name, resource_group_name, partition_name = \ self.__get_mt_names_from_location(location) + module_type = dictionary_utils.get_element(app_dict, MODULE_TYPE) + new_app_name = self.__deploy_app_online(app_name, src_path, targets, plan=plan_file, stage_mode=stage_mode, partition=partition_name, resource_group=resource_group_name, resource_group_template=resource_group_template_name, + module_type=module_type, + sub_module_targets=sub_module_targets, options=options) location.remove_name_token(token_name) deployed_applist.append(new_app_name) @@ -1005,7 +1015,8 @@ def __get_mt_names_from_location(self, app_location): return resource_group_template_name, resource_group_name, partition_name def __deploy_app_online(self, application_name, source_path, targets, stage_mode=None, plan=None, partition=None, - resource_group=None, resource_group_template=None, options=None): + resource_group=None, resource_group_template=None, sub_module_targets = None, + module_type = None, options=None): """ Deploy an application or shared library in online mode. :param application_name: the name of the app or library from the model @@ -1048,7 +1059,8 @@ def __deploy_app_online(self, application_name, source_path, targets, stage_mode if is_library: computed_name = self.version_helper.get_library_versioned_name(source_path, application_name) else: - computed_name = self.version_helper.get_application_versioned_name(source_path, application_name) + computed_name = self.version_helper.get_application_versioned_name(source_path, application_name, + module_type=module_type) application_name = computed_name @@ -1077,6 +1089,9 @@ def __deploy_app_online(self, application_name, source_path, targets, stage_mode kwargs[key] = value kwargs['timeout'] = self.model_context.get_model_config().get_deploy_timeout() + if module_type in ['jms', 'jdbc', 'wldf' ] and sub_module_targets is not None: + kwargs[SUB_MODULE_TARGETS] = sub_module_targets + self.logger.fine('WLSDPLY-09320', type_name, application_name, kwargs, class_name=self._class_name, method_name=_method_name) self.wlst_helper.deploy_application(application_name, *args, **kwargs) @@ -1187,7 +1202,7 @@ def __start_all_apps(self, deployed_app_list, base_location): self.__start_app(app) -def _get_deploy_options(model_apps, app_name, library_module): +def _get_deploy_options(model_apps, app_name, library_module, application_version_helper): """ Get the deploy command options. :param model_apps: the apps dictionary @@ -1197,8 +1212,8 @@ def _get_deploy_options(model_apps, app_name, library_module): """ deploy_options = OrderedDict() # not sure about altDD, altWlsDD + app = dictionary_utils.get_dictionary_element(model_apps, app_name) for option in [DEPLOYMENT_ORDER, SECURITY_DD_MODEL, PLAN_STAGING_MODE, STAGE_MODE]: - app = dictionary_utils.get_dictionary_element(model_apps, app_name) value = dictionary_utils.get_element(app, option) option_name = '' @@ -1219,8 +1234,21 @@ def _get_deploy_options(model_apps, app_name, library_module): if len(deploy_options) == 0: deploy_options = None - return deploy_options + module_type = dictionary_utils.get_element(app, MODULE_TYPE) + sub_module_targets = '' + if application_version_helper.is_module_type_resource_type(module_type): + sub_deployments = dictionary_utils.get_element(app, SUB_DEPLOYMENT) + if sub_deployments is not None: + for sub_deployment in sub_deployments: + if sub_module_targets != '': + sub_module_targets += ',' + name = sub_deployment + target = sub_deployments[name][TARGET] + sub_module_targets += '%s@%s' % (name, target) + + + return deploy_options, sub_module_targets def _find_deployorder_list(apps_dict, ordered_list, order): """ diff --git a/core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py b/core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py index 53ebe1ec7..1c4bf979b 100644 --- a/core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py +++ b/core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py @@ -82,7 +82,7 @@ def get_library_versioned_name(self, source_path, model_name, from_archive=False self.logger.exiting(class_name=self._class_name, method_name=_method_name, result=versioned_name) return versioned_name - def get_application_versioned_name(self, source_path, model_name, from_archive=False): + def get_application_versioned_name(self, source_path, model_name, from_archive=False, module_type=None): """ Get the proper name of the deployable application that WLST requires in the target domain. This method is needed for the case where the application is explicitly versioned in its ear/war manifest. @@ -100,6 +100,8 @@ def get_application_versioned_name(self, source_path, model_name, from_archive=F # if no manifest version is found, leave the original name unchanged versioned_name = model_name + if self.is_module_type_resource_type(module_type): + return model_name try: manifest = self.__get_manifest(source_path, from_archive) @@ -123,6 +125,12 @@ def get_application_versioned_name(self, source_path, model_name, from_archive=F self.logger.exiting(class_name=self._class_name, method_name=_method_name, result=versioned_name) return versioned_name + def is_module_type_resource_type(self, module_type): + if module_type in [ 'jms', 'jdbc', 'wldf']: + return True + else: + return False + def __get_manifest(self, source_path, from_archive): """ Returns the manifest object for the specified path. From edc1af468fe5fbfda0a97781aad245c2e70d2b5b Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Fri, 7 Oct 2022 13:45:03 -0500 Subject: [PATCH 02/23] fix password and user in jdbc standalone xml --- .../weblogic/deploy/util/FileUtils.java | 5 +- .../deploy/util/WLSDeployArchive.java | 9 +++ .../wlsdeploy/aliases/model_constants.py | 1 + .../tool/discover/deployments_discoverer.py | 65 ++++++++++++++++++- .../tool/util/credential_injector.py | 20 +++++- 5 files changed, 96 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java b/core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java index 790558067..c12f57f5b 100644 --- a/core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java +++ b/core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java @@ -618,7 +618,7 @@ public static byte[] readInputStreamToByteArray(InputStream input) throws IOExce } public static File writeInputStreamToFile(InputStream input, String fileName) throws IOException { - File tmpdir = new File(System.getProperty("java.io.tmpdir")); + File tmpdir = getTmpDir(); File file = new File(tmpdir, fileName); try (FileOutputStream fos = new FileOutputStream(file)) { byte[] byteArray = FileUtils.readInputStreamToByteArray(input); @@ -627,6 +627,9 @@ public static File writeInputStreamToFile(InputStream input, String fileName) th return file; } + public static File getTmpDir() { + return new File(System.getProperty("java.io.tmpdir")); + } public static void extractZipFileContent(WLSDeployArchive archiveFile, String zipEntry, String extractPath) { final String METHOD = "extractZipFileContent"; diff --git a/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java b/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java index aca36b397..7fd59cd86 100644 --- a/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java +++ b/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java @@ -615,6 +615,15 @@ public String addApplication(String appPath) throws WLSDeployArchiveIOException return newName; } + public String readdApplication(String appPath, String tempFile) throws WLSDeployArchiveIOException { + final String METHOD = "readdApplication"; + LOGGER.entering(CLASS, METHOD, appPath); + getZipFile().removeZipEntry(appPath); + String newName = addApplication(tempFile); + LOGGER.exiting(CLASS, METHOD, newName); + return newName; + } + public String addApplicationFolder(String appName, String appPath) throws WLSDeployArchiveIOException { final String METHOD = "addApplicationFolder"; diff --git a/core/src/main/python/wlsdeploy/aliases/model_constants.py b/core/src/main/python/wlsdeploy/aliases/model_constants.py index 57ebb37b2..00a3cde90 100644 --- a/core/src/main/python/wlsdeploy/aliases/model_constants.py +++ b/core/src/main/python/wlsdeploy/aliases/model_constants.py @@ -192,6 +192,7 @@ MESSAGE_LOGGING_PARAMS = 'MessageLoggingParams' MESSAGING_BRIDGE = 'MessagingBridge' METHOD = 'Method' +MODULE_TYPE = 'ModuleType' MULTICAST = 'Multicast' MULTICAST_ADDRESS = 'MulticastAddress' MULTICAST_PORT = 'MulticastPort' diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index 65b9ec1c1..90b90f774 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -4,14 +4,20 @@ """ import os +from java.io import BufferedReader +from java.io import BufferedWriter from java.io import File +from java.io import FileReader +from java.io import FileWriter from java.lang import IllegalArgumentException from oracle.weblogic.deploy.util import PyOrderedDict as OrderedDict +from oracle.weblogic.deploy.util import FileUtils from oracle.weblogic.deploy.util import StringUtils from oracle.weblogic.deploy.util import WLSDeployArchiveIOException from oracle.weblogic.deploy.util import WLSDeployArchive +from wlsdeploy.aliases.alias_constants import PASSWORD_TOKEN from wlsdeploy.aliases import model_constants from wlsdeploy.aliases.location_context import LocationContext from wlsdeploy.aliases.wlst_modes import WlstModes @@ -19,6 +25,7 @@ from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.tool.discover import discoverer from wlsdeploy.tool.discover.discoverer import Discoverer +from wlsdeploy.util import dictionary_utils from wlsdeploy.util import path_utils _class_name = 'DeploymentsDiscoverer' @@ -224,14 +231,14 @@ def get_applications(self): location.add_name_token(name_token, application) result[application] = OrderedDict() self._populate_model_parameters(result[application], location) - self._add_application_to_archive(application, result[application]) + self._add_application_to_archive(application, result[application], location) self._discover_subfolders(result[application], location) location.remove_name_token(name_token) _logger.exiting(class_name=_class_name, method_name=_method_name, result=result) return model_top_folder_name, result - def _add_application_to_archive(self, application_name, application_dict): + def _add_application_to_archive(self, application_name, application_dict, location): """ Add the binary or directory referenced by the application to the archive file. If the binary can not be located and added to the archive file, un-target the application and log the problem. @@ -241,6 +248,7 @@ def _add_application_to_archive(self, application_name, application_dict): """ _method_name = 'add_application_to_archive' _logger.entering(application_name, class_name=_class_name, method_name=_method_name) + archive_file = self._model_context.get_archive_file() if model_constants.SOURCE_PATH in application_dict: if model_constants.PLAN_DIR in application_dict and \ @@ -265,6 +273,11 @@ def _add_application_to_archive(self, application_name, application_dict): method_name=_method_name) try: new_source_name = archive_file.addApplication(file_name_path) + module_type = dictionary_utils.get_dictionary_element(application_dict, + model_constants.MODULE_TYPE) + if module_type == 'jdbc': + self._jdbc_password_fix(new_source_name) + except IllegalArgumentException, iae: self._disconnect_target(application_name, application_dict, iae.getLocalizedMessage()) except WLSDeployArchiveIOException, wioe: @@ -325,6 +338,54 @@ def _create_app_folder(self, application_name, application_dict): _logger.exiting(class_name=_class_name, method_name=_method_name) + def _jdbc_password_fix(self, source_name): + """ + This will look for password and userid in the jdbc standalone xml and + replace with either fix password token or a token in the xml and variable file. + It extracts the jdbc xml from the archive and then replaces it with the updated file. + :param source_name: Name of the path and file for the standalone xml file + """ + _method_name = '_jdbc_password_fix' + _logger.entering(source_name, class_name=_class_name, method_name=_method_name) + archive_file = self._model_context.get_archive_file() + tmpDir = FileUtils.getTmpDir(); + temp_file = FileUtils.createTempDirectory(tmpDir, 'jdbc-xml') + jdbc_file = archive_file.extractFile(source_name, temp_file) + jdbc_out = FileUtils.createTempDirectory(tmpDir, 'jdbc-out') + jdbc_out = archive_file.extractFile(source_name, jdbc_out) + bis = BufferedReader(FileReader(jdbc_file)) + bos = BufferedWriter(FileWriter(jdbc_out)) + + found = False + while bis.ready(): + line = bis.readLine() + if 'password-encrypted' in line: + bos.write(self._get_pass_replacement(jdbc_file, '.pass.encrypt', 'password-encrypted')) + elif 'user' in line: + found = True + bos.write(line) + elif found and 'value' in line: + bos.write(self._get_pass_replacement(jdbc_file, '.user', 'value')) + + else: + bos.write(line) + bos.newLine() + bis.close() + bos.close() + archive_file.readdApplication(source_name, jdbc_out) + _logger.exiting(class_name=_class_name, method_name=_method_name) + + def _get_pass_replacement(self, jdbc_file, name, type): + if self._credential_injector is not None: + head, tail = os.path.split(jdbc_file) + token = tail[:len(jdbc_file) - len('jdbc.xml')] + token = token + name + result = self._credential_injector.injection_out_of_model(token) + else: + result = PASSWORD_TOKEN + result = '<' + type + '>' + result + '' + return result + def _test_app_folder(self, source_path, plan_dir): app_folder = False app_dir = File(source_path).getParent() diff --git a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py index ba3de4976..7958b15ac 100644 --- a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py @@ -81,6 +81,8 @@ def __init__(self, program_name, model, model_context, version=None, variable_di VariableInjector.__init__(self, program_name, model, model_context, version=version, variable_dictionary=variable_dictionary) self._model_context = model_context + self._no_filter_keys_cache = [] + self._no_filter_keys_cache.append(self.NO_FILTER_KEYS) def check_and_tokenize(self, model_dict, attribute, location): """ @@ -138,6 +140,22 @@ def check_and_tokenize(self, model_dict, attribute, location): assigns.append('%s=%s' % (key, properties[key])) model_dict[attribute] = split_value.join(assigns) + def injection_out_of_model(self, token): + """ + This is for tokenizing variables that are not in the model but need to be in the variable file + :param token: name for cache to create a token for + :param attribute: attribute name + :return: tokenized name + """ + _method_name = 'injection_out_of_model' + _logger.entering(token, class_name=_class_name, method_name=_method_name) + result = self.get_variable_token(None, token) + self.add_to_cache(token_name=token, token_value='') + + self._no_filter_keys_cache.append(token) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=result) + return result + def get_variable_name(self, attribute_location, attribute, suffix=None): """ Override method to possibly create secret token names instead of property names. @@ -222,7 +240,7 @@ def filter_unused_credentials(self, model_dictionary): cache_keys = self.get_variable_cache().keys() for key in cache_keys: - if key in self.NO_FILTER_KEYS: + if key in self._no_filter_keys_cache: continue if credentials_method == SECRETS_METHOD: From 245460e240ffa4d2bf00199443c169a08a1e40db Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Mon, 10 Oct 2022 10:06:56 -0500 Subject: [PATCH 03/23] reset found flag --- .../python/wlsdeploy/tool/discover/deployments_discoverer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index 90b90f774..b76308c6c 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -366,6 +366,7 @@ def _jdbc_password_fix(self, source_name): bos.write(line) elif found and 'value' in line: bos.write(self._get_pass_replacement(jdbc_file, '.user', 'value')) + found = False else: bos.write(line) From f5e67b227b73af1f06b5989470cb2a41079ba638 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Thu, 13 Oct 2022 14:01:36 -0500 Subject: [PATCH 04/23] add ons password token --- .../python/wlsdeploy/tool/discover/deployments_discoverer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index b76308c6c..f1859b9fe 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -359,8 +359,10 @@ def _jdbc_password_fix(self, source_name): found = False while bis.ready(): line = bis.readLine() - if 'password-encrypted' in line: + if '' in line: bos.write(self._get_pass_replacement(jdbc_file, '.pass.encrypt', 'password-encrypted')) + elif '' in line: + bos.write(self._get_pass_replacement(jdbc_file, '.ons.pass.encrypt', 'ons-wallet-password-encrypted')) elif 'user' in line: found = True bos.write(line) From 97dfe13263cabf9faaeed4695c2d9c3f09ae1c78 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Mon, 17 Oct 2022 10:32:40 -0500 Subject: [PATCH 05/23] capture user in properties --- .../deploy/util/WLSDeployArchive.java | 4 +- .../tool/discover/deployments_discoverer.py | 45 ++++++++++++------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java b/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java index 7fd59cd86..88317a059 100644 --- a/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java +++ b/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java @@ -615,8 +615,8 @@ public String addApplication(String appPath) throws WLSDeployArchiveIOException return newName; } - public String readdApplication(String appPath, String tempFile) throws WLSDeployArchiveIOException { - final String METHOD = "readdApplication"; + public String replaceApplication(String appPath, String tempFile) throws WLSDeployArchiveIOException { + final String METHOD = "replaceApplication"; LOGGER.entering(CLASS, METHOD, appPath); getZipFile().removeZipEntry(appPath); String newName = addApplication(tempFile); diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index f1859b9fe..c77f17031 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -10,6 +10,9 @@ from java.io import FileReader from java.io import FileWriter from java.lang import IllegalArgumentException +from java.lang import StringBuilder +from java.util.regex import Matcher +from java.util.regex import Pattern from oracle.weblogic.deploy.util import PyOrderedDict as OrderedDict from oracle.weblogic.deploy.util import FileUtils @@ -355,27 +358,35 @@ def _jdbc_password_fix(self, source_name): jdbc_out = archive_file.extractFile(source_name, jdbc_out) bis = BufferedReader(FileReader(jdbc_file)) bos = BufferedWriter(FileWriter(jdbc_out)) - + cache = StringBuilder() found = False while bis.ready(): - line = bis.readLine() - if '' in line: - bos.write(self._get_pass_replacement(jdbc_file, '.pass.encrypt', 'password-encrypted')) - elif '' in line: - bos.write(self._get_pass_replacement(jdbc_file, '.ons.pass.encrypt', 'ons-wallet-password-encrypted')) - elif 'user' in line: - found = True - bos.write(line) - elif found and 'value' in line: - bos.write(self._get_pass_replacement(jdbc_file, '.user', 'value')) - found = False - - else: - bos.write(line) - bos.newLine() + cache.append(bis.readLine()) bis.close() + pattern = Pattern.compile("(\s?)user(\s?)") + matcher = pattern.matcher(cache.toString()) + end = -1 + if matcher.find(): + end = matcher.end() + result = cache.toString() + if end >= 0: + pattern = Pattern.compile("(.+?)") + matcher = pattern.matcher(result[end:]) + matcher.find() + pattern = Pattern.compile(matcher.group()) + matcher = pattern.matcher(cache.toString()) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.user', 'value')) + + pattern = Pattern.compile('(.+?)') + matcher = pattern.matcher(result) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.pass.encrypt', 'password-encrypted')) + + pattern = Pattern.compile('(.+?)') + matcher = pattern.matcher(result) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.ons.pass.encrypt', 'ons-wallet-password-encrypted')) + bos.write(result) bos.close() - archive_file.readdApplication(source_name, jdbc_out) + archive_file.replaceApplication(source_name, jdbc_out) _logger.exiting(class_name=_class_name, method_name=_method_name) def _get_pass_replacement(self, jdbc_file, name, type): From 4e4961de9139bbc912e326fb1fec7c4d9c30caa1 Mon Sep 17 00:00:00 2001 From: Johnny Shum Date: Mon, 17 Oct 2022 17:17:33 -0500 Subject: [PATCH 06/23] initial merge from discovery of application module and added logic for replacing token with clear text value from properties etc ... --- .../tool/deploy/applications_deployer.py | 35 +++++++++++++++++-- .../deploy/applications_version_helper.py | 4 +-- .../main/python/wlsdeploy/util/variables.py | 21 ++++++++--- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py index d568ac09b..d0e4b9723 100644 --- a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py +++ b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py @@ -52,6 +52,8 @@ from wlsdeploy.util import model_helper from wlsdeploy.util import string_utils +import wlsdeploy.util.variables as variables + class ApplicationsDeployer(Deployer): """ @@ -200,13 +202,18 @@ def __add_applications(self): application_name = \ self.version_helper.get_application_versioned_name(app_source_path, application_name, module_type=module_type) + quoted_application_name = self.wlst_helper.get_quoted_name_for_wlst(application_name) + application_location.add_name_token(application_token, quoted_application_name) self.wlst_helper.cd(root_path) deployer_utils.create_and_cd(application_location, existing_applications, self.aliases) + self._set_attributes_and_add_subfolders(application_location, application) + application_location.remove_name_token(application_token) + self.__substitute_appmodule_token(app_source_path, module_type) if app_source_path.startswith(WLSDeployArchive.ARCHIVE_STRUCT_APPS_TARGET_DIR): plan_dir = dictionary_utils.get_element(application, PLAN_DIR) @@ -215,6 +222,27 @@ def __add_applications(self): self.logger.exiting(class_name=self._class_name, method_name=_method_name) + def __substitute_appmodule_token(self, path, module_type): + if self.version_helper.is_module_type_app_module(module_type): + if os.path.isabs(path): + abspath = path + else: + abspath = self.model_context.get_domain_home() + os.sep + path + + original_variables = {} + variable_file = self.model_context.get_variable_file() + if variable_file is not None and os.path.exists(variable_file): + original_variables = variables.load_variables(variable_file) + fh = open(abspath, 'r') + text = fh.read() + fh.close() + newtext = variables.substitute_value(text, original_variables, self.model_context, encrypt_value=True) + newfh = open(abspath, 'w') + newfh.write(newtext) + newfh.close() + + + def __online_deploy_apps_and_libs(self, base_location): """ Deploy shared libraries and applications in online mode. @@ -994,8 +1022,10 @@ def __deploy_model_applications(self, model_apps, app_location, deployed_applist options=options) location.remove_name_token(token_name) deployed_applist.append(new_app_name) + self.__substitute_appmodule_token(path, module_type) + - def __get_mt_names_from_location(self, app_location): +def __get_mt_names_from_location(self, app_location): dummy_location = LocationContext() token_name = self.aliases.get_name_token(dummy_location) dummy_location.add_name_token(token_name, self.model_context.get_domain_name()) @@ -1108,7 +1138,6 @@ def __extract_source_path_from_archive(self, source_path, model_type, model_name :param model_name: the element name (my-app, etc.), used for logging """ _method_name = '__extract_source_path_from_archive' - # source path may be may be a single file (jar, war, etc.) if self.archive_helper.contains_file(source_path): self.archive_helper.extract_file(source_path) @@ -1242,7 +1271,7 @@ def _get_deploy_options(model_apps, app_name, library_module, application_versio module_type = dictionary_utils.get_element(app, MODULE_TYPE) sub_module_targets = '' - if application_version_helper.is_module_type_resource_type(module_type): + if application_version_helper.is_module_type_app_module(module_type): sub_deployments = dictionary_utils.get_element(app, SUB_DEPLOYMENT) if sub_deployments is not None: for sub_deployment in sub_deployments: diff --git a/core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py b/core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py index 1c4bf979b..5994a8cdd 100644 --- a/core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py +++ b/core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py @@ -100,7 +100,7 @@ def get_application_versioned_name(self, source_path, model_name, from_archive=F # if no manifest version is found, leave the original name unchanged versioned_name = model_name - if self.is_module_type_resource_type(module_type): + if self.is_module_type_app_module(module_type): return model_name try: @@ -125,7 +125,7 @@ def get_application_versioned_name(self, source_path, model_name, from_archive=F self.logger.exiting(class_name=self._class_name, method_name=_method_name, result=versioned_name) return versioned_name - def is_module_type_resource_type(self, module_type): + def is_module_type_app_module(self, module_type): if module_type in [ 'jms', 'jdbc', 'wldf']: return True else: diff --git a/core/src/main/python/wlsdeploy/util/variables.py b/core/src/main/python/wlsdeploy/util/variables.py index e922f853d..218ded29a 100644 --- a/core/src/main/python/wlsdeploy/util/variables.py +++ b/core/src/main/python/wlsdeploy/util/variables.py @@ -21,6 +21,7 @@ from wlsdeploy.logging import platform_logger from wlsdeploy.util import dictionary_utils from wlsdeploy.util.cla_utils import CommandLineArgUtil +from wlsdeploy.util.weblogic_helper import WebLogicHelper _class_name = "variables" _logger = platform_logger.PlatformLogger('wlsdeploy.variables') @@ -29,6 +30,7 @@ _environment_pattern = re.compile("(@@ENV:([\\w.-]+)@@)") _secret_pattern = re.compile("(@@SECRET:([\\w.-]+):([\\w.-]+)@@)") _file_nested_variable_pattern = re.compile("(@@FILE:(@@[\w]+@@[\w.\\\/:-]+)@@)") +__wls_helper = WebLogicHelper(_logger) # these match a string containing ONLY a token _property_string_pattern = re.compile("^(@@PROP:([\\w.-]+)@@)$") @@ -166,7 +168,7 @@ def get_variable_names(text): return names -def substitute_value(text, variables, model_context): +def substitute_value(text, variables, model_context, encrypt_value=False): """ Perform token substitutions on a single text value. If errors occur during substitution, throw a single VariableException. @@ -176,7 +178,7 @@ def substitute_value(text, variables, model_context): """ method_name = 'substitute_value' error_info = {'errorCount': 0} - result = _substitute(text, variables, model_context, error_info) + result = _substitute(text, variables, model_context, error_info, encrypt_value=encrypt_value) error_count = error_info['errorCount'] if error_count: ex = exception_helper.create_variable_exception("WLSDPLY-01740", error_count) @@ -232,19 +234,21 @@ def _process_node(nodes, variables, model_context, error_info): for member in value: if type(member) in [str, unicode]: index = value.index(member) - value[index] = _substitute(member, variables, model_context, error_info, key) + value[index] = _substitute(member, variables, model_context, error_info, attribute_name=key) elif type(value) in [str, unicode]: - nodes[key] = _substitute(value, variables, model_context, error_info, key) + nodes[key] = _substitute(value, variables, model_context, error_info, attribute_name=key) -def _substitute(text, variables, model_context, error_info, attribute_name=None): +def _substitute(text, variables, model_context, error_info, attribute_name=None, encrypt_value=False): """ Substitute token placeholders with their derived values. :param text: the text to process for token placeholders :param variables: the variables to use :param model_context: used to determine the validation method (strict, lax, etc.) :param error_info: collects information about errors encountered + :param encrypt_value: encrypt the clear text value with the domain salt before substitution. It is only used + : for app module application. :return: the replaced text """ method_name = '_substitute' @@ -270,6 +274,8 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None) continue value = variables[key] + if encrypt_value: + value = __wls_helper.encrypt(value, model_context.get_domain_home()) text = text.replace(token, value) # check environment variables before @@FILE:/dir/@@ENV:name@@.txt@@ @@ -297,6 +303,10 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None) _increment_error_count(error_info, allow_unresolved) problem_found = True continue + + if encrypt_value: + value = __wls_helper.encrypt(value, model_context.get_domain_home()) + text = text.replace(token, value) matches = _file_variable_pattern.findall(text) @@ -340,6 +350,7 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None) _report_token_issue("WLSDPLY-01746", method_name, allow_unresolved, attribute_name, text, sample) _increment_error_count(error_info, allow_unresolved) + return text From 0fd46634bab9dd9c9c617756a557a102f2fa2203 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 18 Oct 2022 16:57:34 -0500 Subject: [PATCH 07/23] changes for url and sh script --- core/src/main/python/discover.py | 4 +++- .../tool/discover/deployments_discoverer.py | 18 ++++++++++-------- .../wlsdeploy/tool/util/credential_injector.py | 10 ++++++++-- .../wlsdeploy/tool/util/variable_injector.py | 5 +++-- .../util/target_configuration_helper.py | 9 +++++++++ 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index a8f59cdc6..aed3f313b 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -492,8 +492,10 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject # Apply the injectors specified in model_variable_injector.json, or in the target configuration. # Include the variable mappings that were collected in credential_cache. + extra_cache = dict() variable_injector = VariableInjector(_program_name, model.get_model(), model_context, - WebLogicHelper(__logger).get_actual_weblogic_version(), credential_cache) + WebLogicHelper(__logger).get_actual_weblogic_version(), credential_cache, + extra_cache) inserted, variable_model, variable_file_name = variable_injector.inject_variables_keyword_file() diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index c77f17031..991e49391 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -11,7 +11,6 @@ from java.io import FileWriter from java.lang import IllegalArgumentException from java.lang import StringBuilder -from java.util.regex import Matcher from java.util.regex import Pattern from oracle.weblogic.deploy.util import PyOrderedDict as OrderedDict @@ -359,9 +358,8 @@ def _jdbc_password_fix(self, source_name): bis = BufferedReader(FileReader(jdbc_file)) bos = BufferedWriter(FileWriter(jdbc_out)) cache = StringBuilder() - found = False while bis.ready(): - cache.append(bis.readLine()) + cache.append(bis.readLine()).append("\n") bis.close() pattern = Pattern.compile("(\s?)user(\s?)") matcher = pattern.matcher(cache.toString()) @@ -375,26 +373,30 @@ def _jdbc_password_fix(self, source_name): matcher.find() pattern = Pattern.compile(matcher.group()) matcher = pattern.matcher(cache.toString()) - result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.user', 'value')) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.user:username', 'value')) pattern = Pattern.compile('(.+?)') matcher = pattern.matcher(result) - result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.pass.encrypt', 'password-encrypted')) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.user:password', 'password-encrypted')) + + pattern = Pattern.compile('(.+?)(.+?)') matcher = pattern.matcher(result) - result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.ons.pass.encrypt', 'ons-wallet-password-encrypted')) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.ons.pass.encrypt:password', 'ons-wallet-password-encrypted')) bos.write(result) bos.close() archive_file.replaceApplication(source_name, jdbc_out) _logger.exiting(class_name=_class_name, method_name=_method_name) - def _get_pass_replacement(self, jdbc_file, name, type): + def _get_pass_replacement(self, jdbc_file, name, type, property=False): if self._credential_injector is not None: head, tail = os.path.split(jdbc_file) token = tail[:len(jdbc_file) - len('jdbc.xml')] token = token + name - result = self._credential_injector.injection_out_of_model(token) + result = self._credential_injector.injection_out_of_model(token, property) else: result = PASSWORD_TOKEN result = '<' + type + '>' + result + '' diff --git a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py index 7958b15ac..febfb97fd 100644 --- a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py @@ -140,7 +140,7 @@ def check_and_tokenize(self, model_dict, attribute, location): assigns.append('%s=%s' % (key, properties[key])) model_dict[attribute] = split_value.join(assigns) - def injection_out_of_model(self, token): + def injection_out_of_model(self, token, property=False): """ This is for tokenizing variables that are not in the model but need to be in the variable file :param token: name for cache to create a token for @@ -149,7 +149,10 @@ def injection_out_of_model(self, token): """ _method_name = 'injection_out_of_model' _logger.entering(token, class_name=_class_name, method_name=_method_name) - result = self.get_variable_token(None, token) + if property: + result = self.get_property_token(None, token) + else: + result = self.get_variable_token(None, token) self.add_to_cache(token_name=token, token_value='') self._no_filter_keys_cache.append(token) @@ -209,6 +212,9 @@ def get_variable_token(self, attribute, variable_name): else: return VariableInjector.get_variable_token(self, attribute, variable_name) + def get_property_token(self, attribute, variable_name): + return VariableInjector.get_variable_token(self, attribute, variable_name) + def _check_tokenized(self, attribute_value): """ Override to return true if target uses credentials and the value is formatted like @@SECRET:xyz:abc@@. diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index dd67e9a80..c89a8d8b4 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -80,7 +80,7 @@ class VariableInjector(object): - def __init__(self, program_name, model, model_context, version=None, variable_dictionary=None): + def __init__(self, program_name, model, model_context, version=None, variable_dictionary=None, extra_cache=None): """ Construct an instance of the injector with the model and information used by the injector. :param program_name: name of the calling tool @@ -92,6 +92,7 @@ def __init__(self, program_name, model, model_context, version=None, variable_di self.__program_name = program_name self.__original = copy.deepcopy(model) self.__model = model + self.__extra_cache = extra_cache self.__model_context = model_context if self.__model_context: self.__wlst_mode = self.__model_context.get_target_wlst_mode() @@ -463,7 +464,7 @@ def _process_attribute(self, model, attribute, location, injector_values): def get_variable_name(self, location, attribute, suffix=None): """ - Return the variable name for use in the cache, and in the variable token. + Return the variable name for use in the cache, and in the variable token. v The default behavior is to return the concatenated location paths, with invalid characters cleared. Sub-classes may extend this for other types of tokens, such as @@SECRET. :param location: the location to be used diff --git a/core/src/main/python/wlsdeploy/util/target_configuration_helper.py b/core/src/main/python/wlsdeploy/util/target_configuration_helper.py index 9d697207a..7f520a006 100644 --- a/core/src/main/python/wlsdeploy/util/target_configuration_helper.py +++ b/core/src/main/python/wlsdeploy/util/target_configuration_helper.py @@ -33,6 +33,15 @@ WEBLOGIC_CREDENTIALS_SECRET_NAME = 'weblogic-credentials' WEBLOGIC_CREDENTIALS_SECRET_SUFFIX = '-' + WEBLOGIC_CREDENTIALS_SECRET_NAME +JDBC_CREDENTIALS_SECRET_USER_NAME = 'standalone-jdbc.xml.user' +JDBC_CREDENTIALS_SECRET_USER_SUFFIX = '-' + JDBC_CREDENTIALS_SECRET_USER_NAME + +JDBC_CREDENTIALS_SECRET_PASS_NAME = 'standalone-jdbc.xml.pass.encrypt' +JDBC_CREDENTIALS_SECRET_PASS_SUFFIX = '-' + JDBC_CREDENTIALS_SECRET_PASS_NAME + +JDBC_CREDENTIALS_SECRET_ONS_PASS_NAME = 'standalone-jdbc.xml.ons.pass.encrypt' +JDBC_CREDENTIALS_SECRET_ONS_PASS_SUFFIX = '-' + JDBC_CREDENTIALS_SECRET_ONS_PASS_NAME + RUNTIME_ENCRYPTION_SECRET_NAME = 'runtime-encryption-secret' RUNTIME_ENCRYPTION_SECRET_SUFFIX = '-' + RUNTIME_ENCRYPTION_SECRET_NAME From b5790696b638efa457f3b4b11f14ba4ffc965d9c Mon Sep 17 00:00:00 2001 From: Johnny Shum Date: Thu, 20 Oct 2022 10:06:28 -0500 Subject: [PATCH 08/23] add function to handle encrypt only password fields in app module xml --- .../wlsdeploy/tool/util/appmodule_helper.py | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py diff --git a/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py b/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py new file mode 100644 index 000000000..f773e0c33 --- /dev/null +++ b/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py @@ -0,0 +1,112 @@ +""" +Copyright (c) 2022, Oracle Corporation and/or its affiliates. +Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. +""" + +import re + + +def process_group(begtag, df, handler, result_tokens, search_only): + updated_text = '' + endtag = begtag.replace('<', ' 0: + for matched_group in properties_match.groups(): + if matched_group != endtag: + fixed_groups.append(matched_group) + + for matched_group in fixed_groups: + if matched_group.startswith(begtag): + updated_text += handler.process(matched_group, result_tokens, search_only) + else: + updated_text += matched_group + + return updated_text + + +class GenericTagReplacement(object): + def __init__(self, begtag): + self.begtag = begtag + self.endtag = self.begtag.replace('<', '', xml_text, self.ons_password_handler, result_tokens, + search_only) + return xml_text + + +class ProcessProperties(object): + def __init__(self): + pass + + def process(self, xmltext, result_tokens, search_only): + property_pairs = re.split(r'|', xmltext.replace("","").replace("","")) + result = '' + for token in property_pairs: + if token.find("user 0: + result += '' + if search_only: + result += '' + token + '' + elif token.find("") > 0: + result += '' + token + '' + else: + result += token + + if result == '': + result = xmltext + else: + result = '' + result + '' + + return result + + +class ProcessJdbcDriverParam(object): + def __init__(self, url_handler, properties_handler, password_handler): + self.url_handler = url_handler + self.properties_handler = properties_handler + self.password_handler = password_handler + + def process(self, xml_text, result_tokens, search_only): + xml_text = process_group('', xml_text, self.url_handler, result_tokens, search_only) + xml_text = process_group('', xml_text, self.password_handler, result_tokens, search_only) + xml_text = process_group('', xml_text, self.properties_handler, result_tokens, search_only) + return xml_text + + +def process_jdbc_appmodule_xml(xml_file, search_password_only=True): + result_tokens = {'found_tokens': []} + + url_handler = GenericTagReplacement('') + password_handler = GenericTagReplacement('') + oracle_params_ons_handler = GenericTagReplacement('') + + properties_handler = ProcessProperties() + jdbc_driver_param_handler = ProcessJdbcDriverParam(url_handler, properties_handler, password_handler) + oracle_params_handler = OracleParamsTag(oracle_params_ons_handler) + result = process_group('', xml_file, jdbc_driver_param_handler, result_tokens, + search_password_only) + result = process_group('', result, oracle_params_handler, result_tokens, search_password_only) + return result, result_tokens + From bef54e48015fb5bb5bda07187dad2051f32b08a4 Mon Sep 17 00:00:00 2001 From: Johnny Shum Date: Thu, 20 Oct 2022 10:06:49 -0500 Subject: [PATCH 09/23] update logic to only encrypt password fields in jdbc app module --- .../tool/deploy/applications_deployer.py | 16 +++++++--- .../main/python/wlsdeploy/util/variables.py | 30 ++++++++++++++----- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py index d0e4b9723..ff22e6ac0 100644 --- a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py +++ b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py @@ -51,6 +51,7 @@ from wlsdeploy.util import dictionary_utils from wlsdeploy.util import model_helper from wlsdeploy.util import string_utils +from wlsdeploy.tool.util import appmodule_helper import wlsdeploy.util.variables as variables @@ -223,6 +224,7 @@ def __add_applications(self): self.logger.exiting(class_name=self._class_name, method_name=_method_name) def __substitute_appmodule_token(self, path, module_type): + # we need to substitute any token in the app module xml file if self.version_helper.is_module_type_app_module(module_type): if os.path.isabs(path): abspath = path @@ -233,16 +235,22 @@ def __substitute_appmodule_token(self, path, module_type): variable_file = self.model_context.get_variable_file() if variable_file is not None and os.path.exists(variable_file): original_variables = variables.load_variables(variable_file) + fh = open(abspath, 'r') text = fh.read() fh.close() - newtext = variables.substitute_value(text, original_variables, self.model_context, encrypt_value=True) + tokens = [] + # only jdbc for now + if module_type == 'jdbc': + origtext, result_tokens = appmodule_helper.process_jdbc_appmodule_xml(text, search_password_only=True) + tokens = result_tokens['found_tokens'] + + newtext = variables.substitute_value(text, original_variables, self.model_context, encrypt_token_list=tokens) newfh = open(abspath, 'w') newfh.write(newtext) newfh.close() - def __online_deploy_apps_and_libs(self, base_location): """ Deploy shared libraries and applications in online mode. @@ -1025,7 +1033,7 @@ def __deploy_model_applications(self, model_apps, app_location, deployed_applist self.__substitute_appmodule_token(path, module_type) -def __get_mt_names_from_location(self, app_location): + def __get_mt_names_from_location(self, app_location): dummy_location = LocationContext() token_name = self.aliases.get_name_token(dummy_location) dummy_location.add_name_token(token_name, self.model_context.get_domain_name()) @@ -1047,7 +1055,7 @@ def __get_mt_names_from_location(self, app_location): return resource_group_template_name, resource_group_name, partition_name def __deploy_app_online(self, application_name, source_path, targets, stage_mode=None, plan=None, partition=None, - resource_group=None, resource_group_template=None, sub_module_targets = None, + resource_group=None, resource_group_template=None, sub_module_targets=None, module_type = None, options=None): """ Deploy an application or shared library in online mode. diff --git a/core/src/main/python/wlsdeploy/util/variables.py b/core/src/main/python/wlsdeploy/util/variables.py index 218ded29a..bfaf7f014 100644 --- a/core/src/main/python/wlsdeploy/util/variables.py +++ b/core/src/main/python/wlsdeploy/util/variables.py @@ -168,17 +168,18 @@ def get_variable_names(text): return names -def substitute_value(text, variables, model_context, encrypt_value=False): +def substitute_value(text, variables, model_context, encrypt_token_list=None): """ Perform token substitutions on a single text value. If errors occur during substitution, throw a single VariableException. :param text: the original text :param variables: a dictionary of variables for substitution :param model_context: used to resolve variables in file paths + :param encrypt_token_list: encrypt the value if the token is in this list for app module xml """ method_name = 'substitute_value' error_info = {'errorCount': 0} - result = _substitute(text, variables, model_context, error_info, encrypt_value=encrypt_value) + result = _substitute(text, variables, model_context, error_info, encrypt_token_list=encrypt_token_list) error_count = error_info['errorCount'] if error_count: ex = exception_helper.create_variable_exception("WLSDPLY-01740", error_count) @@ -240,14 +241,30 @@ def _process_node(nodes, variables, model_context, error_info): nodes[key] = _substitute(value, variables, model_context, error_info, attribute_name=key) -def _substitute(text, variables, model_context, error_info, attribute_name=None, encrypt_value=False): +def __in_appmodule_secret_token_list(token, name, key, encrypted_token_list): + if encrypted_token_list: + for item in encrypted_token_list: + env_matches = _environment_pattern.findall(item) + # first substitute @@ENV first, by this time it should have already flagged any error of missing ENV + for env_match_token, env_match_key in env_matches: + value = os.environ.get(str(env_match_key)) + item = item.replace(env_match_token, value) + break + + secret_matches = _secret_pattern.findall(item) + for matched_token, matched_name, matched_key in secret_matches: + if matched_token == token and matched_name == name and matched_key == key: + return True + return False + +def _substitute(text, variables, model_context, error_info, attribute_name=None, encrypt_token_list=None): """ Substitute token placeholders with their derived values. :param text: the text to process for token placeholders :param variables: the variables to use :param model_context: used to determine the validation method (strict, lax, etc.) :param error_info: collects information about errors encountered - :param encrypt_value: encrypt the clear text value with the domain salt before substitution. It is only used + :param encrypt_token_list: encrypt the clear text value with the domain salt before substitution. It is only used : for app module application. :return: the replaced text """ @@ -274,8 +291,6 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None, continue value = variables[key] - if encrypt_value: - value = __wls_helper.encrypt(value, model_context.get_domain_home()) text = text.replace(token, value) # check environment variables before @@FILE:/dir/@@ENV:name@@.txt@@ @@ -304,7 +319,7 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None, problem_found = True continue - if encrypt_value: + if __in_appmodule_secret_token_list(token, name, key, encrypt_token_list): value = __wls_helper.encrypt(value, model_context.get_domain_home()) text = text.replace(token, value) @@ -350,7 +365,6 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None, _report_token_issue("WLSDPLY-01746", method_name, allow_unresolved, attribute_name, text, sample) _increment_error_count(error_info, allow_unresolved) - return text From c019cb0065bd1d71905e2b4b7f5138582c5e1aa2 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Thu, 20 Oct 2022 17:00:39 -0500 Subject: [PATCH 10/23] add userid to sh script --- .../tool/discover/deployments_discoverer.py | 13 ++++++++----- .../wlsdeploy/tool/util/credential_injector.py | 6 ++++-- .../wlsdeploy/util/target_configuration_helper.py | 1 + 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index 991e49391..6ecb4e928 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -371,17 +371,20 @@ def _jdbc_password_fix(self, source_name): pattern = Pattern.compile("(.+?)") matcher = pattern.matcher(result[end:]) matcher.find() + username = matcher.group() + username = username[len(''):len(username) - len('')] pattern = Pattern.compile(matcher.group()) matcher = pattern.matcher(cache.toString()) - result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.user:username', 'value')) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '-user:username', + 'value', username=username)) pattern = Pattern.compile('(.+?)') matcher = pattern.matcher(result) - result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.user:password', 'password-encrypted')) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '-user:password', 'password-encrypted')) pattern = Pattern.compile('(.+?)(.+?)') matcher = pattern.matcher(result) @@ -391,12 +394,12 @@ def _jdbc_password_fix(self, source_name): archive_file.replaceApplication(source_name, jdbc_out) _logger.exiting(class_name=_class_name, method_name=_method_name) - def _get_pass_replacement(self, jdbc_file, name, type, property=False): + def _get_pass_replacement(self, jdbc_file, name, type, property=False, username=''): if self._credential_injector is not None: head, tail = os.path.split(jdbc_file) token = tail[:len(jdbc_file) - len('jdbc.xml')] token = token + name - result = self._credential_injector.injection_out_of_model(token, property) + result = self._credential_injector.injection_out_of_model(token, property, username) else: result = PASSWORD_TOKEN result = '<' + type + '>' + result + '' diff --git a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py index febfb97fd..753f56045 100644 --- a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py @@ -140,7 +140,7 @@ def check_and_tokenize(self, model_dict, attribute, location): assigns.append('%s=%s' % (key, properties[key])) model_dict[attribute] = split_value.join(assigns) - def injection_out_of_model(self, token, property=False): + def injection_out_of_model(self, token, property=False, username=''): """ This is for tokenizing variables that are not in the model but need to be in the variable file :param token: name for cache to create a token for @@ -153,7 +153,9 @@ def injection_out_of_model(self, token, property=False): result = self.get_property_token(None, token) else: result = self.get_variable_token(None, token) - self.add_to_cache(token_name=token, token_value='') + if username is None: + username = '' + self.add_to_cache(token_name=token, token_value=username) self._no_filter_keys_cache.append(token) _logger.exiting(class_name=_class_name, method_name=_method_name, result=result) diff --git a/core/src/main/python/wlsdeploy/util/target_configuration_helper.py b/core/src/main/python/wlsdeploy/util/target_configuration_helper.py index 7f520a006..bfc7c4f0a 100644 --- a/core/src/main/python/wlsdeploy/util/target_configuration_helper.py +++ b/core/src/main/python/wlsdeploy/util/target_configuration_helper.py @@ -142,6 +142,7 @@ def _prepare_k8s_secrets(model_context, token_dictionary, model_dictionary): for secret_name in secret_names: secret_keys = secret_map[secret_name] user_name = dictionary_utils.get_element(secret_keys, SECRET_USERNAME_KEY) + if user_name is None: secrets.append(_build_secret_hash(secret_name, None, PASSWORD_TAG)) else: From d033e1d3b1a378206f37fe5021fb9e38972ae48d Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Fri, 21 Oct 2022 14:33:16 -0500 Subject: [PATCH 11/23] dont clear cache of other than credentials --- core/src/main/python/discover.py | 7 +++---- .../wlsdeploy/tool/util/credential_injector.py | 3 ++- .../python/wlsdeploy/tool/util/variable_injector.py | 12 ++++++++++++ .../oracle/weblogic/deploy/k8s/create_k8s_secrets.sh | 2 +- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index aed3f313b..2393c864d 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -488,14 +488,13 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject # if target handles credential configuration, clear property cache to keep out of variables file. if model_context.get_target_configuration().manages_credentials(): - credential_cache.clear() + credential_injector.clear_cache(credential_cache) # Apply the injectors specified in model_variable_injector.json, or in the target configuration. # Include the variable mappings that were collected in credential_cache. - extra_cache = dict() + variable_injector = VariableInjector(_program_name, model.get_model(), model_context, - WebLogicHelper(__logger).get_actual_weblogic_version(), credential_cache, - extra_cache) + WebLogicHelper(__logger).get_actual_weblogic_version(), credential_cache) inserted, variable_model, variable_file_name = variable_injector.inject_variables_keyword_file() diff --git a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py index 753f56045..c73512adc 100644 --- a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py @@ -144,7 +144,8 @@ def injection_out_of_model(self, token, property=False, username=''): """ This is for tokenizing variables that are not in the model but need to be in the variable file :param token: name for cache to create a token for - :param attribute: attribute name + :param property: Determine if the property is a property or a secret + :param username: usernames appear as part of property value :return: tokenized name """ _method_name = 'injection_out_of_model' diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index c89a8d8b4..d9010dfbf 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -107,6 +107,18 @@ def __init__(self, program_name, model, model_context, version=None, variable_di self.__aliases = Aliases(model_context) self.__variable_dictionary = variable_dictionary + def clear_cache(self, cache): + """ + Clear the cache of password and usernames. + :param cache: to c + """ + entries = list() + for item in cache: + if ':password' in item or ':username' in item: + entries.append(item) + for entry in entries: + cache.pop(entry) + def get_variable_cache(self): """ This caches all variable information, both from running as a tool, and collected during special diff --git a/core/src/main/resources/oracle/weblogic/deploy/k8s/create_k8s_secrets.sh b/core/src/main/resources/oracle/weblogic/deploy/k8s/create_k8s_secrets.sh index c78f950a0..83ca1f055 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/k8s/create_k8s_secrets.sh +++ b/core/src/main/resources/oracle/weblogic/deploy/k8s/create_k8s_secrets.sh @@ -34,7 +34,7 @@ function create_paired_k8s_secret { {{#comments}} # {{{comment}}} {{/comments}} -create_paired_k8s_secret {{{secretName}}} {{{user}}} {{{password}}} +create_paired_k8s_secret {{{secretName}}} "{{{user}}}" {{{password}}} {{/pairedSecrets}} {{#secrets}} From d81bea4aa59e1ba8f90bfe36831f8c98574f64e4 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Mon, 24 Oct 2022 10:05:00 -0500 Subject: [PATCH 12/23] change level of warning message --- alias-test/src/test/python/aliastest/generate/generator_wlst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alias-test/src/test/python/aliastest/generate/generator_wlst.py b/alias-test/src/test/python/aliastest/generate/generator_wlst.py index ae3748029..9a2b21181 100644 --- a/alias-test/src/test/python/aliastest/generate/generator_wlst.py +++ b/alias-test/src/test/python/aliastest/generate/generator_wlst.py @@ -168,7 +168,7 @@ def get_singleton_name(mbean_type): class_name=__class_name, method_name=_method_name) return None if len(name_list) == 0: - __logger.warning('No MBean instance found for {0} at location {1} and was not in listChildTypes', + __logger.fine('No MBean instance found for {0} at location {1} and was not in listChildTypes', mbean_type, current_path(), class_name=__class_name, method_name=_method_name) return None if len(name_list) > 1: From 8ee3a33048d866ea92dc062798c4e00b5273065e Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Mon, 24 Oct 2022 10:29:53 -0500 Subject: [PATCH 13/23] reverse revision --- alias-test/src/test/python/aliastest/generate/generator_wlst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alias-test/src/test/python/aliastest/generate/generator_wlst.py b/alias-test/src/test/python/aliastest/generate/generator_wlst.py index 9a2b21181..ae3748029 100644 --- a/alias-test/src/test/python/aliastest/generate/generator_wlst.py +++ b/alias-test/src/test/python/aliastest/generate/generator_wlst.py @@ -168,7 +168,7 @@ def get_singleton_name(mbean_type): class_name=__class_name, method_name=_method_name) return None if len(name_list) == 0: - __logger.fine('No MBean instance found for {0} at location {1} and was not in listChildTypes', + __logger.warning('No MBean instance found for {0} at location {1} and was not in listChildTypes', mbean_type, current_path(), class_name=__class_name, method_name=_method_name) return None if len(name_list) > 1: From 3f19f53757534536ff5050413d53e825901f49a5 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Mon, 24 Oct 2022 13:41:17 -0500 Subject: [PATCH 14/23] remove extra character in comments --- core/src/main/python/wlsdeploy/tool/util/variable_injector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index d9010dfbf..3afa947d0 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -476,7 +476,7 @@ def _process_attribute(self, model, attribute, location, injector_values): def get_variable_name(self, location, attribute, suffix=None): """ - Return the variable name for use in the cache, and in the variable token. v + Return the variable name for use in the cache, and in the variable token. The default behavior is to return the concatenated location paths, with invalid characters cleared. Sub-classes may extend this for other types of tokens, such as @@SECRET. :param location: the location to be used From 116bae615d9232c15a7f30709a7dd48fcda3e616 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Mon, 24 Oct 2022 13:58:24 -0500 Subject: [PATCH 15/23] remove unneeded argument --- .../python/wlsdeploy/tool/discover/deployments_discoverer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index 6ecb4e928..aeb584284 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -233,14 +233,14 @@ def get_applications(self): location.add_name_token(name_token, application) result[application] = OrderedDict() self._populate_model_parameters(result[application], location) - self._add_application_to_archive(application, result[application], location) + self._add_application_to_archive(application, result[application]) self._discover_subfolders(result[application], location) location.remove_name_token(name_token) _logger.exiting(class_name=_class_name, method_name=_method_name, result=result) return model_top_folder_name, result - def _add_application_to_archive(self, application_name, application_dict, location): + def _add_application_to_archive(self, application_name, application_dict): """ Add the binary or directory referenced by the application to the archive file. If the binary can not be located and added to the archive file, un-target the application and log the problem. From 51af3485b7943ed0021347f522c54917ad143339 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 25 Oct 2022 10:15:56 -0500 Subject: [PATCH 16/23] replace with extra tokens --- core/src/main/python/discover.py | 19 ++++++++++++------- .../tool/discover/deployments_discoverer.py | 18 ++++++++++++------ .../tool/util/credential_injector.py | 8 ++++---- .../wlsdeploy/tool/util/variable_injector.py | 12 ------------ 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index 2393c864d..6d5a98703 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -219,13 +219,14 @@ def __process_domain_home(arg_map, wlst_mode): arg_map[CommandLineArgUtil.DOMAIN_HOME_SWITCH] = full_path -def __discover(model_context, aliases, credential_injector, helper): +def __discover(model_context, aliases, credential_injector, helper, extra_tokens): """ Populate the model from the domain. :param model_context: the model context :param aliases: aliases instance for discover :param credential_injector: credential injector instance :param helper: wlst_helper instance + :param extra_tokens: dictionary to store non-credential tokens during credential search :return: the fully-populated model :raises DiscoverException: if an error occurred while discover the domain """ @@ -233,7 +234,6 @@ def __discover(model_context, aliases, credential_injector, helper): model = Model() base_location = LocationContext() __connect_to_domain(model_context, helper) - try: _add_domain_name(base_location, aliases, helper) DomainInfoDiscoverer(model_context, model.get_model_domain_info(), base_location, wlst_mode=__wlst_mode, @@ -243,7 +243,8 @@ def __discover(model_context, aliases, credential_injector, helper): ResourcesDiscoverer(model_context, model.get_model_resources(), base_location, wlst_mode=__wlst_mode, aliases=aliases, credential_injector=credential_injector).discover() DeploymentsDiscoverer(model_context, model.get_model_app_deployments(), base_location, wlst_mode=__wlst_mode, - aliases=aliases, credential_injector=credential_injector).discover() + aliases=aliases, credential_injector=credential_injector, + extra_tokens=extra_tokens).discover() __discover_multi_tenant(model, model_context, base_location, aliases, credential_injector) except AliasException, ae: wls_version = WebLogicHelper(__logger).get_actual_weblogic_version() @@ -448,7 +449,7 @@ def __persist_model(model, model_context): __logger.exiting(class_name=_class_name, method_name=_method_name) -def __check_and_customize_model(model, model_context, aliases, credential_injector): +def __check_and_customize_model(model, model_context, aliases, credential_injector, extra_tokens): """ Customize the model dictionary before persisting. Validate the model after customization for informational purposes. Any validation errors will not stop the discovered model to be persisted. @@ -456,6 +457,7 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject :param model_context: configuration from command-line :param aliases: used for validation if model changes are made :param credential_injector: injector created to collect and tokenize credentials, possibly None + :param extra_tokens: dictionary to handle non-credential tokenized arguments """ _method_name = '__check_and_customize_model' __logger.entering(class_name=_class_name, method_name=_method_name) @@ -488,7 +490,7 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject # if target handles credential configuration, clear property cache to keep out of variables file. if model_context.get_target_configuration().manages_credentials(): - credential_injector.clear_cache(credential_cache) + credential_cache.clear() # Apply the injectors specified in model_variable_injector.json, or in the target configuration. # Include the variable mappings that were collected in credential_cache. @@ -496,6 +498,8 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject variable_injector = VariableInjector(_program_name, model.get_model(), model_context, WebLogicHelper(__logger).get_actual_weblogic_version(), credential_cache) + variable_injector.add_to_cache(dictionary=extra_tokens) + inserted, variable_model, variable_file_name = variable_injector.inject_variables_keyword_file() if inserted: @@ -590,10 +594,11 @@ def main(model_context): else: __logger.info('WLSDPLY-06024', class_name=_class_name, method_name=_method_name) + extra_tokens = {} try: - model = __discover(model_context, aliases, credential_injector, helper) + model = __discover(model_context, aliases, credential_injector, helper, extra_tokens) - model = __check_and_customize_model(model, model_context, aliases, credential_injector) + model = __check_and_customize_model(model, model_context, aliases, credential_injector, extra_tokens) __remote_report(model_context) except DiscoverException, ex: diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index aeb584284..1aa6ec712 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -41,9 +41,10 @@ class DeploymentsDiscoverer(Discoverer): """ def __init__(self, model_context, deployments_dictionary, base_location, - wlst_mode=WlstModes.OFFLINE, aliases=None, credential_injector=None): + wlst_mode=WlstModes.OFFLINE, aliases=None, credential_injector=None, extra_tokens=None): Discoverer.__init__(self, model_context, base_location, wlst_mode, aliases, credential_injector) self._dictionary = deployments_dictionary + self._extra_tokens = extra_tokens def discover(self): """ @@ -382,24 +383,29 @@ def _jdbc_password_fix(self, source_name): matcher = pattern.matcher(result) result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '-user:password', 'password-encrypted')) - pattern = Pattern.compile('(.+?)(\s*)(.+?)(\s*)(.+?)') matcher = pattern.matcher(result) - result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.ons.pass.encrypt:password', 'ons-wallet-password-encrypted')) + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.ons.pass.encrypt:password', + 'ons-wallet-password-encrypted')) bos.write(result) bos.close() archive_file.replaceApplication(source_name, jdbc_out) _logger.exiting(class_name=_class_name, method_name=_method_name) - def _get_pass_replacement(self, jdbc_file, name, type, property=False, username=''): + def _get_pass_replacement(self, jdbc_file, name, type, properties=None, username=''): if self._credential_injector is not None: head, tail = os.path.split(jdbc_file) token = tail[:len(jdbc_file) - len('jdbc.xml')] token = token + name - result = self._credential_injector.injection_out_of_model(token, property, username) + if properties is not None: + self._extra_tokens[token] = properties + result = self._credential_injector.injection_out_of_model(token, properties, username) else: result = PASSWORD_TOKEN result = '<' + type + '>' + result + '' diff --git a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py index c73512adc..2e40aa004 100644 --- a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py @@ -144,7 +144,7 @@ def injection_out_of_model(self, token, property=False, username=''): """ This is for tokenizing variables that are not in the model but need to be in the variable file :param token: name for cache to create a token for - :param property: Determine if the property is a property or a secret + :param property: value is a property not a secret :param username: usernames appear as part of property value :return: tokenized name """ @@ -154,9 +154,9 @@ def injection_out_of_model(self, token, property=False, username=''): result = self.get_property_token(None, token) else: result = self.get_variable_token(None, token) - if username is None: - username = '' - self.add_to_cache(token_name=token, token_value=username) + if username is None: + username = '' + self.add_to_cache(token_name=token, token_value=username) self._no_filter_keys_cache.append(token) _logger.exiting(class_name=_class_name, method_name=_method_name, result=result) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 3afa947d0..6b3130e0d 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -107,18 +107,6 @@ def __init__(self, program_name, model, model_context, version=None, variable_di self.__aliases = Aliases(model_context) self.__variable_dictionary = variable_dictionary - def clear_cache(self, cache): - """ - Clear the cache of password and usernames. - :param cache: to c - """ - entries = list() - for item in cache: - if ':password' in item or ':username' in item: - entries.append(item) - for entry in entries: - cache.pop(entry) - def get_variable_cache(self): """ This caches all variable information, both from running as a tool, and collected during special From 83cbafdcf415d7856384f43d0ae1bf1d52c70889 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 25 Oct 2022 11:08:36 -0500 Subject: [PATCH 17/23] requested fixes --- .../wlsdeploy/tool/discover/deployments_discoverer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index 1aa6ec712..93980bb54 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -383,7 +383,7 @@ def _jdbc_password_fix(self, source_name): matcher = pattern.matcher(result) result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '-user:password', 'password-encrypted')) - pattern = Pattern.compile('(\s*)(.+?)(\s*)(\s*)(.+?)(\s*)') matcher = pattern.matcher(result) matcher.find() result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '-url', 'url', @@ -391,7 +391,7 @@ def _jdbc_password_fix(self, source_name): pattern = Pattern.compile('(.+?)') matcher = pattern.matcher(result) - result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '.ons.pass.encrypt:password', + result = matcher.replaceFirst(self._get_pass_replacement(jdbc_file, '-ons-pass-encrypt:password', 'ons-wallet-password-encrypted')) bos.write(result) bos.close() @@ -401,7 +401,7 @@ def _jdbc_password_fix(self, source_name): def _get_pass_replacement(self, jdbc_file, name, type, properties=None, username=''): if self._credential_injector is not None: head, tail = os.path.split(jdbc_file) - token = tail[:len(jdbc_file) - len('jdbc.xml')] + token = tail[:len(tail) - len('.xml')] token = token + name if properties is not None: self._extra_tokens[token] = properties From a4e6123c05780cda4830353f7d78815dd3a95859 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 25 Oct 2022 13:51:46 -0500 Subject: [PATCH 18/23] code smells --- .../wlsdeploy/tool/discover/deployments_discoverer.py | 6 +++--- .../main/python/wlsdeploy/tool/util/variable_injector.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index 93980bb54..1047c7303 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -351,10 +351,10 @@ def _jdbc_password_fix(self, source_name): _method_name = '_jdbc_password_fix' _logger.entering(source_name, class_name=_class_name, method_name=_method_name) archive_file = self._model_context.get_archive_file() - tmpDir = FileUtils.getTmpDir(); - temp_file = FileUtils.createTempDirectory(tmpDir, 'jdbc-xml') + tmp_dir = FileUtils.getTmpDir(); + temp_file = FileUtils.createTempDirectory(tmp_dir, 'jdbc-xml') jdbc_file = archive_file.extractFile(source_name, temp_file) - jdbc_out = FileUtils.createTempDirectory(tmpDir, 'jdbc-out') + jdbc_out = FileUtils.createTempDirectory(tmp_dir, 'jdbc-out') jdbc_out = archive_file.extractFile(source_name, jdbc_out) bis = BufferedReader(FileReader(jdbc_file)) bos = BufferedWriter(FileWriter(jdbc_out)) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 6b3130e0d..dd67e9a80 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -80,7 +80,7 @@ class VariableInjector(object): - def __init__(self, program_name, model, model_context, version=None, variable_dictionary=None, extra_cache=None): + def __init__(self, program_name, model, model_context, version=None, variable_dictionary=None): """ Construct an instance of the injector with the model and information used by the injector. :param program_name: name of the calling tool @@ -92,7 +92,6 @@ def __init__(self, program_name, model, model_context, version=None, variable_di self.__program_name = program_name self.__original = copy.deepcopy(model) self.__model = model - self.__extra_cache = extra_cache self.__model_context = model_context if self.__model_context: self.__wlst_mode = self.__model_context.get_target_wlst_mode() From e2d2c71a40abc424f3d8cecc0d74ac5dafb2010a Mon Sep 17 00:00:00 2001 From: Johnny Shum Date: Tue, 25 Oct 2022 14:24:28 -0500 Subject: [PATCH 19/23] encrypt password after all substitutions are done --- .../tool/deploy/applications_deployer.py | 6 +- .../wlsdeploy/tool/util/appmodule_helper.py | 112 +++--------------- .../main/python/wlsdeploy/util/variables.py | 27 +---- 3 files changed, 19 insertions(+), 126 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py index ff22e6ac0..79b37eeee 100644 --- a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py +++ b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py @@ -239,13 +239,11 @@ def __substitute_appmodule_token(self, path, module_type): fh = open(abspath, 'r') text = fh.read() fh.close() - tokens = [] + newtext = variables.substitute_value(text, original_variables, self.model_context) # only jdbc for now if module_type == 'jdbc': - origtext, result_tokens = appmodule_helper.process_jdbc_appmodule_xml(text, search_password_only=True) - tokens = result_tokens['found_tokens'] + newtext = appmodule_helper.process_jdbc_appmodule_xml(newtext, self.model_context) - newtext = variables.substitute_value(text, original_variables, self.model_context, encrypt_token_list=tokens) newfh = open(abspath, 'w') newfh.write(newtext) newfh.close() diff --git a/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py b/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py index f773e0c33..88e5d703f 100644 --- a/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py @@ -5,108 +5,24 @@ import re +from wlsdeploy.util.weblogic_helper import WebLogicHelper +from wlsdeploy.logging import platform_logger -def process_group(begtag, df, handler, result_tokens, search_only): - updated_text = '' - endtag = begtag.replace('<', ' 0: - for matched_group in properties_match.groups(): - if matched_group != endtag: - fixed_groups.append(matched_group) +def process_jdbc_appmodule_xml(xml_file, model_context): - for matched_group in fixed_groups: - if matched_group.startswith(begtag): - updated_text += handler.process(matched_group, result_tokens, search_only) - else: - updated_text += matched_group - - return updated_text - - -class GenericTagReplacement(object): - def __init__(self, begtag): - self.begtag = begtag - self.endtag = self.begtag.replace('<', '", ""]: + end_tag = beg_tag.replace("<", "', xml_text, self.ons_password_handler, result_tokens, - search_only) - return xml_text - - -class ProcessProperties(object): - def __init__(self): - pass - - def process(self, xmltext, result_tokens, search_only): - property_pairs = re.split(r'|', xmltext.replace("","").replace("","")) - result = '' - for token in property_pairs: - if token.find("user 0: - result += '' - if search_only: - result += '' + token + '' - elif token.find("") > 0: - result += '' + token + '' - else: - result += token - - if result == '': - result = xmltext - else: - result = '' + result + '' - - return result - - -class ProcessJdbcDriverParam(object): - def __init__(self, url_handler, properties_handler, password_handler): - self.url_handler = url_handler - self.properties_handler = properties_handler - self.password_handler = password_handler - - def process(self, xml_text, result_tokens, search_only): - xml_text = process_group('', xml_text, self.url_handler, result_tokens, search_only) - xml_text = process_group('', xml_text, self.password_handler, result_tokens, search_only) - xml_text = process_group('', xml_text, self.properties_handler, result_tokens, search_only) - return xml_text - - -def process_jdbc_appmodule_xml(xml_file, search_password_only=True): - result_tokens = {'found_tokens': []} - - url_handler = GenericTagReplacement('') - password_handler = GenericTagReplacement('') - oracle_params_ons_handler = GenericTagReplacement('') + value = matched_groups.groups()[0] + xml_file = xml_file.replace(beg_tag + value + end_tag, + beg_tag + __wls_helper.encrypt(value, model_context.get_domain_home()) + end_tag) - properties_handler = ProcessProperties() - jdbc_driver_param_handler = ProcessJdbcDriverParam(url_handler, properties_handler, password_handler) - oracle_params_handler = OracleParamsTag(oracle_params_ons_handler) - result = process_group('', xml_file, jdbc_driver_param_handler, result_tokens, - search_password_only) - result = process_group('', result, oracle_params_handler, result_tokens, search_password_only) - return result, result_tokens + return xml_file diff --git a/core/src/main/python/wlsdeploy/util/variables.py b/core/src/main/python/wlsdeploy/util/variables.py index bfaf7f014..4dfe102d8 100644 --- a/core/src/main/python/wlsdeploy/util/variables.py +++ b/core/src/main/python/wlsdeploy/util/variables.py @@ -21,7 +21,6 @@ from wlsdeploy.logging import platform_logger from wlsdeploy.util import dictionary_utils from wlsdeploy.util.cla_utils import CommandLineArgUtil -from wlsdeploy.util.weblogic_helper import WebLogicHelper _class_name = "variables" _logger = platform_logger.PlatformLogger('wlsdeploy.variables') @@ -30,7 +29,6 @@ _environment_pattern = re.compile("(@@ENV:([\\w.-]+)@@)") _secret_pattern = re.compile("(@@SECRET:([\\w.-]+):([\\w.-]+)@@)") _file_nested_variable_pattern = re.compile("(@@FILE:(@@[\w]+@@[\w.\\\/:-]+)@@)") -__wls_helper = WebLogicHelper(_logger) # these match a string containing ONLY a token _property_string_pattern = re.compile("^(@@PROP:([\\w.-]+)@@)$") @@ -168,7 +166,7 @@ def get_variable_names(text): return names -def substitute_value(text, variables, model_context, encrypt_token_list=None): +def substitute_value(text, variables, model_context): """ Perform token substitutions on a single text value. If errors occur during substitution, throw a single VariableException. @@ -179,7 +177,7 @@ def substitute_value(text, variables, model_context, encrypt_token_list=None): """ method_name = 'substitute_value' error_info = {'errorCount': 0} - result = _substitute(text, variables, model_context, error_info, encrypt_token_list=encrypt_token_list) + result = _substitute(text, variables, model_context, error_info) error_count = error_info['errorCount'] if error_count: ex = exception_helper.create_variable_exception("WLSDPLY-01740", error_count) @@ -241,23 +239,7 @@ def _process_node(nodes, variables, model_context, error_info): nodes[key] = _substitute(value, variables, model_context, error_info, attribute_name=key) -def __in_appmodule_secret_token_list(token, name, key, encrypted_token_list): - if encrypted_token_list: - for item in encrypted_token_list: - env_matches = _environment_pattern.findall(item) - # first substitute @@ENV first, by this time it should have already flagged any error of missing ENV - for env_match_token, env_match_key in env_matches: - value = os.environ.get(str(env_match_key)) - item = item.replace(env_match_token, value) - break - - secret_matches = _secret_pattern.findall(item) - for matched_token, matched_name, matched_key in secret_matches: - if matched_token == token and matched_name == name and matched_key == key: - return True - return False - -def _substitute(text, variables, model_context, error_info, attribute_name=None, encrypt_token_list=None): +def _substitute(text, variables, model_context, error_info, attribute_name=None): """ Substitute token placeholders with their derived values. :param text: the text to process for token placeholders @@ -319,9 +301,6 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None, problem_found = True continue - if __in_appmodule_secret_token_list(token, name, key, encrypt_token_list): - value = __wls_helper.encrypt(value, model_context.get_domain_home()) - text = text.replace(token, value) matches = _file_variable_pattern.findall(text) From e28d668ddda1a837e34298376010c0bd4c2e0a2c Mon Sep 17 00:00:00 2001 From: Johnny Shum Date: Tue, 25 Oct 2022 14:27:28 -0500 Subject: [PATCH 20/23] revert changes --- core/src/main/python/wlsdeploy/util/variables.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/src/main/python/wlsdeploy/util/variables.py b/core/src/main/python/wlsdeploy/util/variables.py index 4dfe102d8..e922f853d 100644 --- a/core/src/main/python/wlsdeploy/util/variables.py +++ b/core/src/main/python/wlsdeploy/util/variables.py @@ -173,7 +173,6 @@ def substitute_value(text, variables, model_context): :param text: the original text :param variables: a dictionary of variables for substitution :param model_context: used to resolve variables in file paths - :param encrypt_token_list: encrypt the value if the token is in this list for app module xml """ method_name = 'substitute_value' error_info = {'errorCount': 0} @@ -233,10 +232,10 @@ def _process_node(nodes, variables, model_context, error_info): for member in value: if type(member) in [str, unicode]: index = value.index(member) - value[index] = _substitute(member, variables, model_context, error_info, attribute_name=key) + value[index] = _substitute(member, variables, model_context, error_info, key) elif type(value) in [str, unicode]: - nodes[key] = _substitute(value, variables, model_context, error_info, attribute_name=key) + nodes[key] = _substitute(value, variables, model_context, error_info, key) def _substitute(text, variables, model_context, error_info, attribute_name=None): @@ -246,8 +245,6 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None) :param variables: the variables to use :param model_context: used to determine the validation method (strict, lax, etc.) :param error_info: collects information about errors encountered - :param encrypt_token_list: encrypt the clear text value with the domain salt before substitution. It is only used - : for app module application. :return: the replaced text """ method_name = '_substitute' @@ -300,7 +297,6 @@ def _substitute(text, variables, model_context, error_info, attribute_name=None) _increment_error_count(error_info, allow_unresolved) problem_found = True continue - text = text.replace(token, value) matches = _file_variable_pattern.findall(text) From bc641949171008d7b9dfd3e4e2a7f98fd6dd65a2 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 25 Oct 2022 14:33:37 -0500 Subject: [PATCH 21/23] requested change --- .../tool/discover/deployments_discoverer.py | 4 +++- .../wlsdeploy/tool/util/credential_injector.py | 12 +++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index 1047c7303..28ef1f7e0 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -405,7 +405,9 @@ def _get_pass_replacement(self, jdbc_file, name, type, properties=None, username token = token + name if properties is not None: self._extra_tokens[token] = properties - result = self._credential_injector.injection_out_of_model(token, properties, username) + result = self._credential_injector.get_property_token(None, token) + else: + result = self._credential_injector.injection_out_of_model(token, username) else: result = PASSWORD_TOKEN result = '<' + type + '>' + result + '' diff --git a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py index 2e40aa004..c30a54732 100644 --- a/core/src/main/python/wlsdeploy/tool/util/credential_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/credential_injector.py @@ -140,23 +140,17 @@ def check_and_tokenize(self, model_dict, attribute, location): assigns.append('%s=%s' % (key, properties[key])) model_dict[attribute] = split_value.join(assigns) - def injection_out_of_model(self, token, property=False, username=''): + def injection_out_of_model(self, token, username=''): """ This is for tokenizing variables that are not in the model but need to be in the variable file :param token: name for cache to create a token for - :param property: value is a property not a secret :param username: usernames appear as part of property value :return: tokenized name """ _method_name = 'injection_out_of_model' _logger.entering(token, class_name=_class_name, method_name=_method_name) - if property: - result = self.get_property_token(None, token) - else: - result = self.get_variable_token(None, token) - if username is None: - username = '' - self.add_to_cache(token_name=token, token_value=username) + result = self.get_variable_token(None, token) + self.add_to_cache(token_name=token, token_value=username) self._no_filter_keys_cache.append(token) _logger.exiting(class_name=_class_name, method_name=_method_name, result=result) From 3a66857f61b78832dbca22f3dc79ba6edccbe817 Mon Sep 17 00:00:00 2001 From: Johnny Shum Date: Wed, 26 Oct 2022 06:34:38 -0500 Subject: [PATCH 22/23] add jms app module for create --- .../tool/deploy/applications_deployer.py | 4 +++- .../wlsdeploy/tool/util/appmodule_helper.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py index 79b37eeee..eadbef48e 100644 --- a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py +++ b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py @@ -240,9 +240,11 @@ def __substitute_appmodule_token(self, path, module_type): text = fh.read() fh.close() newtext = variables.substitute_value(text, original_variables, self.model_context) - # only jdbc for now + # only jdbc and jms for now if module_type == 'jdbc': newtext = appmodule_helper.process_jdbc_appmodule_xml(newtext, self.model_context) + elif module_type == 'jms': + newtext = appmodule_helper.process_jms_appmodule_xml(newtext, self.model_context) newfh = open(abspath, 'w') newfh.write(newtext) diff --git a/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py b/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py index 88e5d703f..5c99a9bd6 100644 --- a/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/appmodule_helper.py @@ -26,3 +26,19 @@ def process_jdbc_appmodule_xml(xml_file, model_context): return xml_file + +def process_jms_appmodule_xml(xml_file, model_context): + + for beg_tag in [ "", ""]: + end_tag = beg_tag.replace("<", " Date: Wed, 26 Oct 2022 10:25:11 -0500 Subject: [PATCH 23/23] use helper method to be consistent --- .../main/python/wlsdeploy/tool/deploy/applications_deployer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py index eadbef48e..0d5e01430 100644 --- a/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py +++ b/core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py @@ -1129,7 +1129,7 @@ def __deploy_app_online(self, application_name, source_path, targets, stage_mode kwargs[key] = value kwargs['timeout'] = self.model_context.get_model_config().get_deploy_timeout() - if module_type in ['jms', 'jdbc', 'wldf' ] and sub_module_targets is not None: + if self.version_helper.is_module_type_app_module(module_type) and sub_module_targets is not None: kwargs[SUB_MODULE_TARGETS] = sub_module_targets self.logger.fine('WLSDPLY-09320', type_name, application_name, kwargs,