From c745cbbde85e055e5e98c93ba611dad37a107a04 Mon Sep 17 00:00:00 2001 From: Elima Sugunan <97244204+elima-sugunan@users.noreply.github.com> Date: Wed, 7 Dec 2022 22:27:25 +0530 Subject: [PATCH 1/9] MYSQL-23: Create codeql.yml --- .github/workflows/codeql.yml | 74 ++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..5201195 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,74 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '24 4 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # â„šī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From 6e5757d66e1d18b22d67deed5aa2c6946da2aae5 Mon Sep 17 00:00:00 2001 From: "vinay.byrappa" Date: Mon, 19 Dec 2022 23:03:01 +0530 Subject: [PATCH 2/9] MYSQL-7 : Password Values not redacted in the shell logs --- src/dboperations/dboperations.py | 17 ++++------- src/plugin_runner.py | 12 +++----- src/pluginops/pluginops.py | 51 +++++++++---------------------- src/resources/library.sh | 4 +-- src/resources/restore_stage.sh | 10 +++--- src/resources/restore_stage_bi.sh | 13 ++++---- src/resources/restore_stage_si.sh | 5 +-- src/resources/stopVirtual.sh | 2 +- src/resources/tablespaces.sh | 8 ++--- 9 files changed, 47 insertions(+), 75 deletions(-) diff --git a/src/dboperations/dboperations.py b/src/dboperations/dboperations.py index a3e2095..c3a91d4 100644 --- a/src/dboperations/dboperations.py +++ b/src/dboperations/dboperations.py @@ -37,7 +37,7 @@ def stop_mysql(port,connection,baseDir,vdbConn,pwd): if(port_stat == Status.ACTIVE): logger.debug("DB is Running. Shutting down.") shutdown_cmd = "%s/bin/mysqladmin %s'%s' --protocol=TCP --port=%s shutdown" % (baseDir,vdbConn,pwd,port) - logger.debug("Shutdown Command: {}".format(shutdown_cmd)) + # logger.debug("Shutdown Command: {}".format(shutdown_cmd)) result = libs.run_bash(connection, shutdown_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() @@ -46,7 +46,7 @@ def stop_mysql(port,connection,baseDir,vdbConn,pwd): logger.debug("There was an error trying to shutdown the database : "+error) raise MySQLShutdownException(error) else: - logger.debug("Output: "+output) + logger.debug("DB shutdown completed") time.sleep(25) if(Status.ACTIVE == get_port_status(port,connection)): logger.debug("KILL") @@ -97,7 +97,6 @@ def get_port_status(port,connection): except Exception as err: logger.debug("Port Check Failed for second cmd: "+err.message) logger.debug("Port Status Response >") - logger.debug(output) if output== "": logger.debug("MySQL DB is NOT RUNNING at Port:"+myport) @@ -130,7 +129,6 @@ def start_mysql(installPath,baseDir,mountPath,port,serverId,connection): if(port_stat == Status.INACTIVE): logger.debug("DB is not running. Starting the MySQL DB") start_cmd=get_start_cmd(installPath,baseDir,mountPath,port,serverId) - logger.debug("Startup Command: {}".format(start_cmd)) result = libs.run_bash(connection, start_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() @@ -139,7 +137,7 @@ def start_mysql(installPath,baseDir,mountPath,port,serverId,connection): logger.debug("There was an error trying to start the DB : "+error) raise MySQLStartupException(error) else: - logger.debug("Output: "+output) + logger.debug("DB Started") time.sleep(25) if(Status.ACTIVE == get_port_status(port,connection)): logger.debug("DB Started Successfully") @@ -168,12 +166,11 @@ def start_slave(connection,installPath,port,connString,username,pwd,hostIp): raise Exception("One of the required params for MySQL Connection is empty") else: start_slave_cmd=CommandFactory.start_replication(connection,installPath,port,connString,username,pwd,hostIp) - logger.debug("Connection String with {}".format(start_slave_cmd)) try: logger.debug("Starting Slave") result = libs.run_bash(connection, start_slave_cmd,environment_vars,check=True) output = result.stdout.strip() - logger.debug("Start Slave Output: {}".format(output)) + logger.debug("Start Slave Completed") except Exception as err: logger.debug("Starting Slave Failed: "+err.message) raise err @@ -187,7 +184,6 @@ def stop_slave(connection,installPath,port,connString,username,pwd,hostIp): raise Exception("One of the required params for MySQL Connection is empty") else: stop_slave_cmd=CommandFactory.stop_replication(connection,installPath,port,connString,username,pwd,hostIp) - logger.debug("Connection String with {}".format(stop_slave_cmd)) try: logger.debug("Stopping Replication") result = libs.run_bash(connection, stop_slave_cmd,environment_vars,check=True) @@ -195,9 +191,9 @@ def stop_slave(connection,installPath,port,connString,username,pwd,hostIp): _bashErrMsg=result.stderr.strip() _bashErrCode=result.exit_code if _bashErrCode!=0: - logger.debug("Stopping Slave was not succesful") + logger.debug("Stopping Slave was not successful") raise Exception(_bashErrMsg) - logger.debug("Start Slave Response: {}".format(_output)) + logger.debug("Stop Replication successful") except Exception as err: logger.debug("Stop Replication Failed Due To: "+err.message) logger.debug("Ignoring and continuing") @@ -213,7 +209,6 @@ def get_connection_cmd(installPath,port,connString,username,pwd,hostIp): raise ValueError("One of the required params for MySQL Connection is empty") else: connection_cmd=CommandFactory.connect_to_mysql(installPath,port,connString,username,pwd,hostIp) - logger.debug("connaction_cmd >"+connection_cmd) return connection_cmd def get_start_cmd(installPath,baseDir,mountPath,port,serverId): diff --git a/src/plugin_runner.py b/src/plugin_runner.py index ecd48bf..7b80814 100644 --- a/src/plugin_runner.py +++ b/src/plugin_runner.py @@ -75,16 +75,16 @@ def stop_staging(staged_source, repository, source_config): pluginops.stop_staging(staged_source, repository, source_config) @plugin.linked.pre_snapshot() -def linked_pre_snapshot(staged_source, repository, source_config, snapshot_parameters): +def linked_pre_snapshot(staged_source, repository, source_config, optional_snapshot_parameters): logger.debug("linked_pre_snapshot > Start ") # Start Staging if not already running. - pluginops.linked_pre_snapshot(staged_source, repository, source_config, snapshot_parameters) + pluginops.linked_pre_snapshot(staged_source, repository, source_config, optional_snapshot_parameters) logger.debug(" linked_pre_snapshot > End ") @plugin.linked.post_snapshot() -def linked_post_snapshot(staged_source,repository,source_config,snapshot_parameters): +def linked_post_snapshot(staged_source,repository,source_config,optional_snapshot_parameters): logger.debug("linked_post_snapshot - Start ") - snapshot = pluginops.linked_post_snapshot(staged_source,repository,source_config,snapshot_parameters) + snapshot = pluginops.linked_post_snapshot(staged_source,repository,source_config,optional_snapshot_parameters) linked_status(staged_source, repository, source_config) logger.debug("linked_post_snapshot - End ") return snapshot @@ -105,9 +105,6 @@ def configure(virtual_source, snapshot, repository): def reconfigure(virtual_source, repository, source_config, snapshot): logger.debug("virtual.reconfigure > Start") start(virtual_source, repository, source_config) - logger.debug(source_config) - logger.debug("Snapshot") - logger.debug(snapshot) #srcConfig = configure(virtual_source,snapshot,repository) logger.debug("virtual.reconfigure > End") virtual_status(virtual_source, repository, source_config) @@ -140,7 +137,6 @@ def virtual_post_snapshot(virtual_source, repository, source_config): snapshot.snap_backup_path="" snapshot.snap_time=utils.get_current_time() logger.debug("SnapShot Definition Created") - logger.debug(snapshot) return snapshot @plugin.virtual.start() diff --git a/src/pluginops/pluginops.py b/src/pluginops/pluginops.py index a7cbd4b..a353ce0 100644 --- a/src/pluginops/pluginops.py +++ b/src/pluginops/pluginops.py @@ -124,8 +124,6 @@ def start_staging(staged_source, repository, source_config): result = libs.run_bash(staged_source.staged_connection, start_staging_script,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() - logger.debug("output:"+output) - logger.debug("error:"+error) exit_code = result.exit_code if exit_code !=0: logger.debug("There was an error> "+error) @@ -151,8 +149,6 @@ def start_staging(staged_source, repository, source_config): result = libs.run_bash(staged_source.staged_connection, start_staging_script,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() - logger.debug("output:"+output) - logger.debug("error:"+error) exit_code = result.exit_code if exit_code !=0: logger.debug("Error is : "+error) @@ -188,7 +184,7 @@ def stop_staging(staged_source, repository, source_config): logger.debug("Error is : "+error) raise LinkingException("Exception while stopping staging:"+error) else: - logger.debug("Stop Staging - Successful: "+output) + logger.debug("Stop Staging - Successful") elif staged_source.parameters.d_source_type == "Manual Backup Ingestion": logger.debug("plugin_operations.stop_staging > Manual Backup Ingestion") library_script=pkgutil.get_data('resources','library.sh') @@ -210,7 +206,7 @@ def stop_staging(staged_source, repository, source_config): logger.debug("Error is : "+error) raise LinkingException("Exception while stopping staging:"+error) else: - logger.debug("Stop Staging - Successful: "+output) + logger.debug("Stop Staging - Successful") else : logger.debug("dSourceType is Simple Tablespace Backup. ") library_script=pkgutil.get_data('resources','library.sh') @@ -232,7 +228,7 @@ def stop_staging(staged_source, repository, source_config): logger.debug("Error is : "+error) raise LinkingException("Exception while stopping staging:"+error) else: - logger.debug("Stop Staging - Successful: "+output) + logger.debug("Stop Staging - Successful: ") #TODO: Integration ################################################## @@ -321,18 +317,17 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.error(err) raise err else: - logger.debug("Pre-Snapshot/Restore successful "+output) + logger.debug("Pre-Snapshot/Restore successful ") logger.debug("Restoring Backup to Stage") restore_script = pkgutil.get_data('resources', 'restore_stage.sh') result = libs.run_bash(staged_source.staged_connection, restore_script,environment_vars,check=False) - logger.debug(result) output = result.stdout.strip() std_err=result.stderr.strip() exit_code = result.exit_code logger.debug(std_err) logger.debug(exit_code) if exit_code == 0: - logger.debug("Creation of Staging DB(Pre-Snapshot) successful."+output) + logger.debug("Creation of Staging DB(Pre-Snapshot) successful.") else: logger.debug("There was an error while creating the staging DB.Check error.log for details.") err = utils.process_exit_codes(exit_code,"DBLINK",std_err) @@ -369,7 +364,7 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.error(err) raise err else: - logger.debug("Pre-Snapshot/Restore_DB successful "+output) + logger.debug("Pre-Snapshot/Restore_DB successful ") else: # Simple Tablespace Option is hidden from the plugin. # This section will not get triggered until the option gets added back in schema.json @@ -407,7 +402,7 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.debug("There was an error while resync : "+error) raise LinkingException("Exception in pre-snapshot/restore_db:"+error) else: - logger.debug("Pre-Snapshot/Restore_DB successful "+output) + logger.debug("Pre-Snapshot/Restore_DB successful") # Simple Tablespace Option is hidden from the plugin. # This section will not get triggered until the option gets added back in schema.json @@ -471,7 +466,6 @@ def linked_post_snapshot( logger.debug("plugin_opertions.linked_post_snapshot - Start ") dSourceType = staged_source.parameters.d_source_type start_staging(staged_source,repository,source_config) - logger.debug(snapshot_parameters) mount_path=staged_source.parameters.mount_path snapshot = SnapshotDefinition(validate=False) snapshot.snapshot_id= str(utils.get_snapshot_id()) @@ -482,7 +476,6 @@ def linked_post_snapshot( snapshot.snap_pass=staged_source.parameters.staging_pass snapshot.snap_backup_path=staged_source.parameters.backup_path snapshot.snap_time=utils.get_current_time() - logger.debug(snapshot) logger.debug("linked_post_snapshot - End ") return snapshot @@ -510,7 +503,7 @@ def linked_status(staged_source, repository, source_config): logger.debug("Exception while checking Staging DB Status : "+error) #ignore status? else: - logger.debug("Staging Status Check: "+output) + logger.debug("Staging Status Check successful") if output == "ACTIVE": return Status.ACTIVE else: @@ -532,11 +525,9 @@ def configure(virtual_source, snapshot, repository): ) logger.debug("Mount Path:"+mount_path) logger.debug("Snapshot Settings:") - logger.debug(snapshot) logger.debug("Snapshot_id"+snapshot.snapshot_id) logger.debug("Config Settings: ") config_settings_prov = virtual_source.parameters.config_settings_prov - logger.debug(config_settings_prov) config_params="" ################################################################### # TODO: Operation fails if there are config settings. Must revisit. @@ -576,19 +567,18 @@ def configure(virtual_source, snapshot, repository): } configure_script = pkgutil.get_data('resources', 'provision.sh') result = libs.run_bash(virtual_source.connection, configure_script,environment_vars,check=False) - logger.debug(result) output = result.stdout.strip() std_err=result.stderr.strip() exit_code = result.exit_code if exit_code == 0: - logger.debug("Pre-Snapshot/Restore_DB successful "+output) + logger.debug("Pre-Snapshot/Restore_DB successful") else: err = utils.process_exit_codes(exit_code,"PROVISION",std_err) logger.debug("There was an error while provisioning.Check error.log for details.") logger.error(err) raise err return SourceConfigDefinition( - db_name=output, + db_name=virtual_source.parameters.t_database_name, base_dir=virtual_source.parameters.base_dir, port=virtual_source.parameters.port, data_dir=mount_path @@ -610,7 +600,6 @@ def stop_mysql(port,connection,baseDir,user,pwd,host): if(port_stat == Status.ACTIVE): logger.debug("DB is Running. Shutting down.") shutdown_cmd = "%s/bin/mysqladmin %s'%s' --protocol=TCP --port=%s shutdown" % (baseDir,vdbConn,pwd,port) - logger.debug("Shutdown Command: {}".format(shutdown_cmd)) result = libs.run_bash(connection, shutdown_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() @@ -619,7 +608,7 @@ def stop_mysql(port,connection,baseDir,user,pwd,host): logger.debug("There was an error trying to shutdown the database : "+error) #raise MySQLShutdownException(error) else: - logger.debug("Output: "+output) + logger.debug("Shutdown of database successful") time.sleep(20) if(Status.ACTIVE == get_port_status(port,connection)): logger.debug("MySQL has not shutdown after 20 seconds. Killing process.") @@ -676,8 +665,7 @@ def get_port_status(port,connection): output = result.stdout.strip() except Exception as err: logger.debug("Port Check Failed for second cmd: "+err.message) - logger.debug("Port Status Response >") - logger.debug(output) + logger.debug("Get port status successful") if output== "": logger.debug("MySQL DB is NOT RUNNING at Port:"+myport) @@ -714,7 +702,6 @@ def start_mysql(installPath,baseDir,mountPath,port,serverId,connection): if(port_stat == Status.INACTIVE): logger.debug("DB is not running. Starting the MySQL DB") start_cmd=get_start_cmd(installPath,baseDir,mountPath,port,serverId) - logger.debug("Startup Command: {}".format(start_cmd)) result = libs.run_bash(connection, start_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() @@ -726,8 +713,6 @@ def start_mysql(installPath,baseDir,mountPath,port,serverId,connection): constants.ERR_START_ACTION, "ExitCode:{} \n {}".format(exit_code,error) ) - else: - logger.debug("Output: "+output) time.sleep(30) if(Status.ACTIVE == get_port_status(port,connection)): logger.debug("DB Started Successfully") @@ -750,12 +735,11 @@ def start_slave(connection,installPath,port,connString,username,pwd,hostIp): raise Exception("One of the required params for MySQL Connection is empty") else: start_slave_cmd=CommandFactory.start_replication(connection,installPath,port,connString,username,pwd,hostIp) - logger.debug("Connection String with {}".format(start_slave_cmd)) try: logger.debug("Starting Slave") result = libs.run_bash(connection, start_slave_cmd,environment_vars,check=True) output = result.stdout.strip() - logger.debug("Start Slave Output: {}".format(output)) + logger.debug("Start Slave successful") except Exception as err: logger.debug("Starting Slave Failed: "+err.message) raise err @@ -773,7 +757,6 @@ def stop_slave(connection,installPath,port,connString,username,pwd,hostIp): raise Exception("One of the required params for MySQL Connection is empty") else: stop_slave_cmd=CommandFactory.stop_replication(connection,installPath,port,connString,username,pwd,hostIp) - logger.debug("Connection String with {}".format(stop_slave_cmd)) try: logger.debug("Stopping Replication") result = libs.run_bash(connection, stop_slave_cmd,environment_vars,check=True) @@ -781,9 +764,9 @@ def stop_slave(connection,installPath,port,connString,username,pwd,hostIp): _bashErrMsg=result.stderr.strip() _bashErrCode=result.exit_code if _bashErrCode!=0: - logger.debug("Stopping Slave was not succesful") + logger.debug("Stopping Slave was not successful") raise Exception(_bashErrMsg) - logger.debug("Start Slave Response: {}".format(_output)) + logger.debug("Stopping Slave successful") except Exception as err: logger.debug("Stop Replication Failed Due To: "+err.message) logger.debug("Ignoring and continuing") @@ -802,7 +785,6 @@ def get_connection_cmd(installPath,port,connString,username,pwd,hostIp): raise ValueError("One of the required params for MySQL Connection is empty") else: connection_cmd=CommandFactory.connect_to_mysql(installPath,port,connString,username,pwd,hostIp) - logger.debug("connaction_cmd >"+connection_cmd) return connection_cmd ################################################## @@ -850,16 +832,13 @@ def repository_discovery(source_connection): logger.debug("Error is : "+error) raise RepositoryDiscoveryError("Exception while discovering:"+error) else: - logger.debug("Output: "+output) #process repository json repos_js=json.loads(output) # print the keys and values for repo_js in repos_js: - #logger.debug("Adding repository:"+repo_js+" to list") path = repo_js['installPath'] version = repo_js['version'] prettyName = repo_js['prettyName'].split("/bin")[1] repository = RepositoryDefinition(name=prettyName, install_path=path, version=version) repositories.append(repository) - logger.debug("output:"+output) return repositories diff --git a/src/resources/library.sh b/src/resources/library.sh index ff5d0a5..2c1ff7b 100644 --- a/src/resources/library.sh +++ b/src/resources/library.sh @@ -296,7 +296,7 @@ stopDatabase() { ZLOGSYNC=`echo "${ZRESULTS}" | $DLPX_BIN_JQ --raw-output ".logSync"` echo "ZCONN:" - echo "$ZCONN" + masklog "$ZCONN" # Found valid process ... if [[ "${ZPORT}" != "" ]] && [[ "${ZPSID}" != "" ]] @@ -495,7 +495,7 @@ function terminate { # Runs a given command and exits with code if error function command_runner { - masklog "command_runner: CMD : $1" + #masklog "command_runner: CMD : $1" return_msg=$(eval $1 2>&1 1>&2 > /dev/null) return_code=$? log "Return Status: ${return_code}" diff --git a/src/resources/restore_stage.sh b/src/resources/restore_stage.sh index 328d711..ad09556 100644 --- a/src/resources/restore_stage.sh +++ b/src/resources/restore_stage.sh @@ -43,7 +43,7 @@ STAGINGPASS=`echo "'"${STAGINGPASS}"'"` masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" +#echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" @@ -101,15 +101,15 @@ ${MYSQLD}/mysqld --initialize --user=mysql --datadir=${NEW_DATA_DIR} --log-error PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` # sudo grep 'temporary password' ${NEW_DATA_DIR}/mysqld.log` # 2019-04-11T14:40:34.032576Z 1 [Note] A temporary password is generated for root@localhost: L0qXNZ8?C3Us -log "init temporary password: ${PWD_LINE}" +# log "init temporary password: ${PWD_LINE}" TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # These temporary passwords contain special characters so need to wrap in single / literal quotes ... TMP_PWD=`echo "'"$TMP_PWD"'"` -log "Temporary Password: ${TMP_PWD}" +# log "Temporary Password: ${TMP_PWD}" masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" +#echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" log "Creation Results: ${RESULTS}" @@ -390,7 +390,7 @@ command_runner "${CMD}" 5 # masklog "Staging Connection Prior to updaging password : ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" +#echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection after updating password: ${STAGING_CONN}" diff --git a/src/resources/restore_stage_bi.sh b/src/resources/restore_stage_bi.sh index 464ee01..586d80b 100644 --- a/src/resources/restore_stage_bi.sh +++ b/src/resources/restore_stage_bi.sh @@ -36,7 +36,7 @@ log "Staging Port: ${TARGET_PORT}" STAGINGPASS=`echo "'"${STAGINGPASS}"'"` masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" +# echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" @@ -75,15 +75,15 @@ ${MYSQLD}/mysqld --initialize --user=mysql --datadir=${NEW_DATA_DIR} --log-error PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` # sudo grep 'temporary password' ${NEW_DATA_DIR}/mysqld.log` -log "init temporary password: ${PWD_LINE}" +# log "init temporary password: ${PWD_LINE}" TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # These temporary passwords contain special characters so need to wrap in single / literal quotes ... TMP_PWD=`echo "'"$TMP_PWD"'"` -log "Temporary Password: ${TMP_PWD}" +# log "Temporary Password: ${TMP_PWD}" masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" +# echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" masklog "Creation Results: ${RESULTS}" @@ -341,7 +341,8 @@ fi ######################################################################## # Change Password for Staging Conn ... CMD="${INSTALL_BIN}/mysql ${STAGING_CONN} --connect-expired-password -se \"ALTER USER 'root'@'localhost' IDENTIFIED BY ${STAGINGPASS};UPDATE mysql.user SET authentication_string=PASSWORD(${STAGINGPASS}) where USER='root';FLUSH PRIVILEGES;\"" -masklog "Final Command to Change Password is : ${CMD}" +CMDLOG="${INSTALL_BIN}/mysql ${STAGING_CONN} --connect-expired-password -se \"ALTER USER 'root'@'localhost' IDENTIFIED BY '********';UPDATE mysql.user SET authentication_string=PASSWORD('********') where USER='root';FLUSH PRIVILEGES;\"" +masklog "Final Command to Change Password is : ${CMDLOG}" command_runner "${CMD}" 5 #eval ${CMD} 1>>${DEBUG_LOG} 2>&1 @@ -349,7 +350,7 @@ command_runner "${CMD}" 5 # Update Staging Connection with supplied password ... masklog "Staging Connection Prior to updating password : ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" +# echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` log "============================================================" masklog "Staging Connection after updating password: ${STAGING_CONN}" diff --git a/src/resources/restore_stage_si.sh b/src/resources/restore_stage_si.sh index 22ed1e3..b64187f 100644 --- a/src/resources/restore_stage_si.sh +++ b/src/resources/restore_stage_si.sh @@ -211,8 +211,9 @@ fi # Change Password for Staging Conn ... # CMD="${INSTALL_BIN}/mysql ${STAGING_CONN} --connect-expired-password -se \"ALTER USER 'root'@'localhost' IDENTIFIED BY ${STAGINGPASS}; FLUSH PRIVILEGES;\"" +CMDLOG="${INSTALL_BIN}/mysql ${STAGING_CONN} --connect-expired-password -se \"ALTER USER 'root'@'localhost' IDENTIFIED BY '********'; FLUSH PRIVILEGES;\"" -log "Change Password: ${CMD}" +log "Change Password: ${CMDLOG}" eval ${CMD} 1>>${DEBUG_LOG} 2>&1 @@ -223,7 +224,7 @@ log "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" ) #log "${RESULTS}" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` -log "Staging Connection: ${STAGING_CONN}" +masklog "Staging Connection: ${STAGING_CONN}" log "Validating Restore Databases ..." CMD="${INSTALL_BIN}/mysql ${STAGING_CONN} -e \"show databases;\"" diff --git a/src/resources/stopVirtual.sh b/src/resources/stopVirtual.sh index f58243e..c1086c9 100644 --- a/src/resources/stopVirtual.sh +++ b/src/resources/stopVirtual.sh @@ -27,7 +27,7 @@ log "VDB Connection: ${VDBCONN}" RESULTS=$( buildConnectionString "${VDBCONN}" "${VDBPASS}" "${PORT}" ) #log "${RESULTS}" VDB_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` -log "Staging Connection: ${VDB_CONN}" +masklog "Staging Connection: ${VDB_CONN}" RESULTS=$( portStatus "${PORT}" ) #echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".status" diff --git a/src/resources/tablespaces.sh b/src/resources/tablespaces.sh index c888a31..69dedd4 100644 --- a/src/resources/tablespaces.sh +++ b/src/resources/tablespaces.sh @@ -38,11 +38,11 @@ NEW_TMP_DIR="${NEW_MOUNT_DIR}/tmp" #SOURCE_CONN="-uroot -pdelphix --protocol=TCP --port=3306" SOURCEPASS=`echo "'"${SOURCEPASS}"'"` -log "Source Connection: ${SOURCECONN}" +masklog "Source Connection: ${SOURCECONN}" RESULTS=$( buildConnectionString "${SOURCECONN}" "${SOURCEPASS}" "${SOURCEPORT}" ) #log "${RESULTS}" SOURCE_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` -log "Source Connection: ${SOURCE_CONN}" +masklog "Source Connection: ${SOURCE_CONN}" SOURCE_DATA_DIR="${SOURCEDATADIR}" # "/usr/local/mysql/data" SOURCE_BASE_DIR="${SOURCEBASEDIR}" # "/usr/local/mysql" @@ -55,11 +55,11 @@ SOURCE_TABLES="${SOURCETABLES}" # "ALL" "employees,patient,patient_details,m #TARGET_CONN="-uroot -pdelphix --protocol=TCP --port=3307" STAGINGPASS=`echo "'"${STAGINGPASS}"'"` -log "Staging Connection: ${STAGINGCONN}" +masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" ) #log "${RESULTS}" TARGET_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` -log "Staging Connection: ${TARGET_CONN}" +masklog "Staging Connection: ${TARGET_CONN}" #TARGET_DATA_DIR="/mnt/provision/my_stage/data" TARGET_DATA_DIR="${STAGINGDATADIR}"/data From 5236b236193f102124df1edb03b571b58bc8fe23 Mon Sep 17 00:00:00 2001 From: "vinay.byrappa" Date: Tue, 20 Dec 2022 15:20:31 +0530 Subject: [PATCH 3/9] VDB Name in UI Fix --- src/pluginops/pluginops.py | 5 +++-- src/resources/library.sh | 2 +- src/resources/provision.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pluginops/pluginops.py b/src/pluginops/pluginops.py index a353ce0..489ac4e 100644 --- a/src/pluginops/pluginops.py +++ b/src/pluginops/pluginops.py @@ -15,6 +15,7 @@ import json import re import os +import json from datetime import datetime from common import utils,constants #from dboperations import dboperations @@ -571,14 +572,14 @@ def configure(virtual_source, snapshot, repository): std_err=result.stderr.strip() exit_code = result.exit_code if exit_code == 0: - logger.debug("Pre-Snapshot/Restore_DB successful") + logger.debug("Pre-Snapshot/Restore_DB successful"+output) else: err = utils.process_exit_codes(exit_code,"PROVISION",std_err) logger.debug("There was an error while provisioning.Check error.log for details.") logger.error(err) raise err return SourceConfigDefinition( - db_name=virtual_source.parameters.t_database_name, + db_name=output, base_dir=virtual_source.parameters.base_dir, port=virtual_source.parameters.port, data_dir=mount_path diff --git a/src/resources/library.sh b/src/resources/library.sh index 2c1ff7b..c12b845 100644 --- a/src/resources/library.sh +++ b/src/resources/library.sh @@ -295,7 +295,7 @@ stopDatabase() { ZBASEDIR=`echo "${ZRESULTS}" | $DLPX_BIN_JQ --raw-output ".baseDir"` ZLOGSYNC=`echo "${ZRESULTS}" | $DLPX_BIN_JQ --raw-output ".logSync"` - echo "ZCONN:" + #echo "ZCONN:" masklog "$ZCONN" # Found valid process ... diff --git a/src/resources/provision.sh b/src/resources/provision.sh index 5b4b3ab..a4c3d65 100644 --- a/src/resources/provision.sh +++ b/src/resources/provision.sh @@ -216,7 +216,7 @@ outputJSON=$($DLPX_BIN_JQ ".baseDir = $(jqQuote "$CONFIG_BASEDIR")" <<< "$output outputJSON=$($DLPX_BIN_JQ ".dbName = $(jqQuote "$prettyName")" <<< "$outputJSON") printf "$outputJSON" > "$DLPX_OUTPUT_FILE" -log "Output: $outputJSON" +#log "Output: $outputJSON" echo "${prettyName}" #log "Environment: " #export DLPX_LIBRARY_SOURCE="" From f6d3b52add6297eb94eca3130f8742db26641b5d Mon Sep 17 00:00:00 2001 From: "vinay.byrappa" Date: Tue, 20 Dec 2022 15:38:53 +0530 Subject: [PATCH 4/9] Cleanup of files --- src/dboperations/dboperations.py | 1 - src/resources/library.sh | 2 -- src/resources/restore_stage.sh | 4 ---- src/resources/restore_stage_bi.sh | 4 ---- 4 files changed, 11 deletions(-) diff --git a/src/dboperations/dboperations.py b/src/dboperations/dboperations.py index c3a91d4..9ca92c7 100644 --- a/src/dboperations/dboperations.py +++ b/src/dboperations/dboperations.py @@ -37,7 +37,6 @@ def stop_mysql(port,connection,baseDir,vdbConn,pwd): if(port_stat == Status.ACTIVE): logger.debug("DB is Running. Shutting down.") shutdown_cmd = "%s/bin/mysqladmin %s'%s' --protocol=TCP --port=%s shutdown" % (baseDir,vdbConn,pwd,port) - # logger.debug("Shutdown Command: {}".format(shutdown_cmd)) result = libs.run_bash(connection, shutdown_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() diff --git a/src/resources/library.sh b/src/resources/library.sh index c12b845..18e6ede 100644 --- a/src/resources/library.sh +++ b/src/resources/library.sh @@ -295,7 +295,6 @@ stopDatabase() { ZBASEDIR=`echo "${ZRESULTS}" | $DLPX_BIN_JQ --raw-output ".baseDir"` ZLOGSYNC=`echo "${ZRESULTS}" | $DLPX_BIN_JQ --raw-output ".logSync"` - #echo "ZCONN:" masklog "$ZCONN" # Found valid process ... @@ -495,7 +494,6 @@ function terminate { # Runs a given command and exits with code if error function command_runner { - #masklog "command_runner: CMD : $1" return_msg=$(eval $1 2>&1 1>&2 > /dev/null) return_code=$? log "Return Status: ${return_code}" diff --git a/src/resources/restore_stage.sh b/src/resources/restore_stage.sh index ad09556..209f3e9 100644 --- a/src/resources/restore_stage.sh +++ b/src/resources/restore_stage.sh @@ -43,7 +43,6 @@ STAGINGPASS=`echo "'"${STAGINGPASS}"'"` masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -#echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" @@ -101,7 +100,6 @@ ${MYSQLD}/mysqld --initialize --user=mysql --datadir=${NEW_DATA_DIR} --log-error PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` # sudo grep 'temporary password' ${NEW_DATA_DIR}/mysqld.log` # 2019-04-11T14:40:34.032576Z 1 [Note] A temporary password is generated for root@localhost: L0qXNZ8?C3Us -# log "init temporary password: ${PWD_LINE}" TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # These temporary passwords contain special characters so need to wrap in single / literal quotes ... @@ -109,7 +107,6 @@ TMP_PWD=`echo "'"$TMP_PWD"'"` # log "Temporary Password: ${TMP_PWD}" masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -#echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" log "Creation Results: ${RESULTS}" @@ -390,7 +387,6 @@ command_runner "${CMD}" 5 # masklog "Staging Connection Prior to updaging password : ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -#echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection after updating password: ${STAGING_CONN}" diff --git a/src/resources/restore_stage_bi.sh b/src/resources/restore_stage_bi.sh index 586d80b..9cca3e9 100644 --- a/src/resources/restore_stage_bi.sh +++ b/src/resources/restore_stage_bi.sh @@ -36,7 +36,6 @@ log "Staging Port: ${TARGET_PORT}" STAGINGPASS=`echo "'"${STAGINGPASS}"'"` masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -# echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" @@ -80,10 +79,8 @@ TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # These temporary passwords contain special characters so need to wrap in single / literal quotes ... TMP_PWD=`echo "'"$TMP_PWD"'"` -# log "Temporary Password: ${TMP_PWD}" masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -# echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" masklog "Creation Results: ${RESULTS}" @@ -350,7 +347,6 @@ command_runner "${CMD}" 5 # Update Staging Connection with supplied password ... masklog "Staging Connection Prior to updating password : ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -# echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` log "============================================================" masklog "Staging Connection after updating password: ${STAGING_CONN}" From d54d0e2490cbe82d00d2582bd672f95a3dddd9d9 Mon Sep 17 00:00:00 2001 From: "vinay.byrappa" Date: Tue, 17 Jan 2023 20:35:35 +0530 Subject: [PATCH 5/9] Review comments addressed --- src/pluginops/pluginops.py | 2 +- src/resources/provision.sh | 7 ------- src/resources/restore_stage.sh | 3 +-- src/resources/restore_stage_bi.sh | 6 +----- src/resources/restore_stage_si.sh | 7 +------ 5 files changed, 4 insertions(+), 21 deletions(-) diff --git a/src/pluginops/pluginops.py b/src/pluginops/pluginops.py index 489ac4e..74665de 100644 --- a/src/pluginops/pluginops.py +++ b/src/pluginops/pluginops.py @@ -572,7 +572,7 @@ def configure(virtual_source, snapshot, repository): std_err=result.stderr.strip() exit_code = result.exit_code if exit_code == 0: - logger.debug("Pre-Snapshot/Restore_DB successful"+output) + logger.debug("Pre-Snapshot/Restore_DB successful") else: err = utils.process_exit_codes(exit_code,"PROVISION",std_err) logger.debug("There was an error while provisioning.Check error.log for details.") diff --git a/src/resources/provision.sh b/src/resources/provision.sh index a4c3d65..09c3d41 100644 --- a/src/resources/provision.sh +++ b/src/resources/provision.sh @@ -216,13 +216,6 @@ outputJSON=$($DLPX_BIN_JQ ".baseDir = $(jqQuote "$CONFIG_BASEDIR")" <<< "$output outputJSON=$($DLPX_BIN_JQ ".dbName = $(jqQuote "$prettyName")" <<< "$outputJSON") printf "$outputJSON" > "$DLPX_OUTPUT_FILE" -#log "Output: $outputJSON" echo "${prettyName}" -#log "Environment: " -#export DLPX_LIBRARY_SOURCE="" -#export REPLICATION_PASS="" -#export SNAPSHOT_METADATA="" -#export STAGINGPASS="" -#env | sort >>$DEBUG_LOG log "End" exit 0 diff --git a/src/resources/restore_stage.sh b/src/resources/restore_stage.sh index 209f3e9..14d9afc 100644 --- a/src/resources/restore_stage.sh +++ b/src/resources/restore_stage.sh @@ -109,7 +109,6 @@ masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" -log "Creation Results: ${RESULTS}" ############################################################ log "Creating Staging Directories on NFS Mounted Path from Delphix" @@ -584,6 +583,6 @@ export DLPX_LIBRARY_SOURCE="" export REPLICATION_PASS="" export STAGINGPASS="" export SOURCEPASS="" -env | sort >>$DEBUG_LOG +env | grep -v 'SOURCEPASS' | sort >>$DEBUG_LOG log " <<>${DEBUG_LOG} 2>&1 PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` -# sudo grep 'temporary password' ${NEW_DATA_DIR}/mysqld.log` -# log "init temporary password: ${PWD_LINE}" TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # These temporary passwords contain special characters so need to wrap in single / literal quotes ... @@ -83,7 +81,6 @@ masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" -masklog "Creation Results: ${RESULTS}" ############################################################ ## @@ -361,7 +358,6 @@ masklog "Granting privileges command: ${CMD}" return_msg=$(eval ${CMD} 2>&1 1>&2 > /dev/null) return_code=$? log "Return Status: ${return_code}" -log "Return message:${return_msg}" if [ $return_code != 0 ]; then errorlog "Unable to grant required permissions to delphix database user. This may have to be done manually" fi @@ -471,7 +467,7 @@ command_runner "${CMD}" 11 log "Environment:" export DLPX_LIBRARY_SOURCE="" export STAGINGPASS="" -env | sort >>$DEBUG_LOG +env | grep -v 'STAGINGPASS' | sort >>$DEBUG_LOG log "End" echo "Staging Started" exit 0 diff --git a/src/resources/restore_stage_si.sh b/src/resources/restore_stage_si.sh index b64187f..807ffbe 100644 --- a/src/resources/restore_stage_si.sh +++ b/src/resources/restore_stage_si.sh @@ -92,16 +92,13 @@ ${MYSQLD}/mysqld --initialize --user=mysql --datadir=${NEW_DATA_DIR} --log-error PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` # sudo grep 'temporary password' ${NEW_DATA_DIR}/mysqld.log` # 2019-04-11T14:40:34.032576Z 1 [Note] A temporary password is generated for root@localhost: L0qXNZ8?C3Us -log "init temporary password: ${PWD_LINE}" TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # # These temporary passwords contain special characters so need to wrap in single / literal quotes ... # TMP_PWD=`echo "'"$TMP_PWD"'"` -log "Temporary Password: ${TMP_PWD}" -##log "Creation Results: ${RESULTS}" log "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" ) @@ -264,8 +261,6 @@ eval ${CMD} 1>>${DEBUG_LOG} 2>&1 log "Environment: " export DLPX_LIBRARY_SOURCE="" -#export REPLICATION_PASS="" -#export STAGINGPASS="" -env | sort >>$DEBUG_LOG +env | grep -v 'STAGINGPASS' | sort >>$DEBUG_LOG log "------------------------- End" exit 0 From c6c68d03a4031d1968019d4b5fa8aa2169eb5fc5 Mon Sep 17 00:00:00 2001 From: "vinay.byrappa" Date: Fri, 20 Jan 2023 11:22:55 +0530 Subject: [PATCH 6/9] Review comments addressed and Copyrights added --- src/common/utils.py | 4 ++++ src/dboperations/dboperations.py | 4 ++++ src/plugin_runner.py | 4 ++++ src/pluginops/pluginops.py | 2 +- src/resources/library.sh | 2 +- src/resources/provision.sh | 2 +- src/resources/restore_stage.sh | 4 ++-- src/resources/restore_stage_bi.sh | 4 ++-- src/resources/restore_stage_si.sh | 4 ++-- src/resources/startVirtual.sh | 3 +++ src/resources/stopVirtual.sh | 2 +- src/resources/tablespaces.sh | 2 +- 12 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/common/utils.py b/src/common/utils.py index e6e0815..0907d37 100644 --- a/src/common/utils.py +++ b/src/common/utils.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# + import logging import random import time diff --git a/src/dboperations/dboperations.py b/src/dboperations/dboperations.py index 9ca92c7..1fc7e1c 100644 --- a/src/dboperations/dboperations.py +++ b/src/dboperations/dboperations.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# + import logging import random import time diff --git a/src/plugin_runner.py b/src/plugin_runner.py index 7b80814..d2a3ea0 100644 --- a/src/plugin_runner.py +++ b/src/plugin_runner.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# + import pkgutil import logging import sys diff --git a/src/pluginops/pluginops.py b/src/pluginops/pluginops.py index 74665de..5a7ad36 100644 --- a/src/pluginops/pluginops.py +++ b/src/pluginops/pluginops.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 by Delphix. All rights reserved. +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. # ####################################################################################################################### diff --git a/src/resources/library.sh b/src/resources/library.sh index 18e6ede..ad230f3 100644 --- a/src/resources/library.sh +++ b/src/resources/library.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 by Delphix. All rights reserved. +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. # # library.sh # diff --git a/src/resources/provision.sh b/src/resources/provision.sh index 09c3d41..887f233 100644 --- a/src/resources/provision.sh +++ b/src/resources/provision.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log diff --git a/src/resources/restore_stage.sh b/src/resources/restore_stage.sh index 14d9afc..1d6d1a5 100644 --- a/src/resources/restore_stage.sh +++ b/src/resources/restore_stage.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. PGM_NAME='restore_stage.sh' # Load Library ... @@ -583,6 +583,6 @@ export DLPX_LIBRARY_SOURCE="" export REPLICATION_PASS="" export STAGINGPASS="" export SOURCEPASS="" -env | grep -v 'SOURCEPASS' | sort >>$DEBUG_LOG +env | grep -v 'PASS' | sort >>$DEBUG_LOG log " <<>$DEBUG_LOG +env | grep -v 'PASS' | sort >>$DEBUG_LOG log "End" echo "Staging Started" exit 0 diff --git a/src/resources/restore_stage_si.sh b/src/resources/restore_stage_si.sh index 807ffbe..2ff4f0b 100644 --- a/src/resources/restore_stage_si.sh +++ b/src/resources/restore_stage_si.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log @@ -261,6 +261,6 @@ eval ${CMD} 1>>${DEBUG_LOG} 2>&1 log "Environment: " export DLPX_LIBRARY_SOURCE="" -env | grep -v 'STAGINGPASS' | sort >>$DEBUG_LOG +env | grep -v 'PASS' | sort >>$DEBUG_LOG log "------------------------- End" exit 0 diff --git a/src/resources/startVirtual.sh b/src/resources/startVirtual.sh index 395451f..6aff22c 100644 --- a/src/resources/startVirtual.sh +++ b/src/resources/startVirtual.sh @@ -1,4 +1,7 @@ # +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# +# set -x PGM_NAME='startVirtual.sh' diff --git a/src/resources/stopVirtual.sh b/src/resources/stopVirtual.sh index c1086c9..24d0ee8 100644 --- a/src/resources/stopVirtual.sh +++ b/src/resources/stopVirtual.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log set -x diff --git a/src/resources/tablespaces.sh b/src/resources/tablespaces.sh index 69dedd4..b47136f 100644 --- a/src/resources/tablespaces.sh +++ b/src/resources/tablespaces.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log From f1b912f83fde7ed6951191a0b2668bdc1bc897dd Mon Sep 17 00:00:00 2001 From: "vinay.byrappa" Date: Fri, 20 Jan 2023 11:35:28 +0530 Subject: [PATCH 7/9] Review comments addressed --- src/pluginops/pluginops.py | 2 +- src/resources/library.sh | 2 +- src/resources/provision.sh | 2 +- src/resources/restore_stage.sh | 3 +-- src/resources/restore_stage_bi.sh | 2 +- src/resources/restore_stage_si.sh | 2 +- src/resources/stopVirtual.sh | 2 +- src/resources/tablespaces.sh | 2 +- 8 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/pluginops/pluginops.py b/src/pluginops/pluginops.py index 5a7ad36..58ec516 100644 --- a/src/pluginops/pluginops.py +++ b/src/pluginops/pluginops.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# Copyright (c) 2020, 2023 by Delphix. All rights reserved. # ####################################################################################################################### diff --git a/src/resources/library.sh b/src/resources/library.sh index ad230f3..9186353 100644 --- a/src/resources/library.sh +++ b/src/resources/library.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# Copyright (c) 2020, 2023 by Delphix. All rights reserved. # # library.sh # diff --git a/src/resources/provision.sh b/src/resources/provision.sh index 887f233..76dd1e6 100644 --- a/src/resources/provision.sh +++ b/src/resources/provision.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log diff --git a/src/resources/restore_stage.sh b/src/resources/restore_stage.sh index 1d6d1a5..f9cc31d 100644 --- a/src/resources/restore_stage.sh +++ b/src/resources/restore_stage.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. PGM_NAME='restore_stage.sh' # Load Library ... @@ -104,7 +104,6 @@ PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # These temporary passwords contain special characters so need to wrap in single / literal quotes ... TMP_PWD=`echo "'"$TMP_PWD"'"` -# log "Temporary Password: ${TMP_PWD}" masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` diff --git a/src/resources/restore_stage_bi.sh b/src/resources/restore_stage_bi.sh index 76f9492..74f65ef 100644 --- a/src/resources/restore_stage_bi.sh +++ b/src/resources/restore_stage_bi.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # Program Name PGM_NAME='restore_stage_bi.sh' diff --git a/src/resources/restore_stage_si.sh b/src/resources/restore_stage_si.sh index 2ff4f0b..f3c4470 100644 --- a/src/resources/restore_stage_si.sh +++ b/src/resources/restore_stage_si.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log diff --git a/src/resources/stopVirtual.sh b/src/resources/stopVirtual.sh index 24d0ee8..1890624 100644 --- a/src/resources/stopVirtual.sh +++ b/src/resources/stopVirtual.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log set -x diff --git a/src/resources/tablespaces.sh b/src/resources/tablespaces.sh index b47136f..f5ae183 100644 --- a/src/resources/tablespaces.sh +++ b/src/resources/tablespaces.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log From 2997fd9a473fe28455467c93625d03ef00028117 Mon Sep 17 00:00:00 2001 From: "vinay.byrappa" Date: Fri, 20 Jan 2023 12:58:28 +0530 Subject: [PATCH 8/9] Review comments addressed --- src/resources/restore.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/restore.sh b/src/resources/restore.sh index 39300c1..91accc0 100644 --- a/src/resources/restore.sh +++ b/src/resources/restore.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # Program Name PGM_NAME='restore.sh' @@ -139,6 +139,6 @@ export DLPX_LIBRARY_SOURCE="" export REPLICATION_PASS="" export STAGINGPASS="" export SOURCEPASS="" -env | sort >>$DEBUG_LOG +env | grep -v 'PASS' | sort >>$DEBUG_LOG log "-- End --" exit 0 From 3c9b0bce1478242c1e4e0473778799b77c57ee1a Mon Sep 17 00:00:00 2001 From: "vinay.byrappa" Date: Fri, 20 Jan 2023 13:09:28 +0530 Subject: [PATCH 9/9] Review comments addressed --- src/pluginops/pluginops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pluginops/pluginops.py b/src/pluginops/pluginops.py index 58ec516..54e50f2 100644 --- a/src/pluginops/pluginops.py +++ b/src/pluginops/pluginops.py @@ -229,7 +229,7 @@ def stop_staging(staged_source, repository, source_config): logger.debug("Error is : "+error) raise LinkingException("Exception while stopping staging:"+error) else: - logger.debug("Stop Staging - Successful: ") + logger.debug("Stop Staging - Successful") #TODO: Integration ################################################## @@ -318,7 +318,7 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.error(err) raise err else: - logger.debug("Pre-Snapshot/Restore successful ") + logger.debug("Pre-Snapshot/Restore successful") logger.debug("Restoring Backup to Stage") restore_script = pkgutil.get_data('resources', 'restore_stage.sh') result = libs.run_bash(staged_source.staged_connection, restore_script,environment_vars,check=False) @@ -328,7 +328,7 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.debug(std_err) logger.debug(exit_code) if exit_code == 0: - logger.debug("Creation of Staging DB(Pre-Snapshot) successful.") + logger.debug("Creation of Staging DB(Pre-Snapshot) successful") else: logger.debug("There was an error while creating the staging DB.Check error.log for details.") err = utils.process_exit_codes(exit_code,"DBLINK",std_err) @@ -365,7 +365,7 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.error(err) raise err else: - logger.debug("Pre-Snapshot/Restore_DB successful ") + logger.debug("Pre-Snapshot/Restore_DB successful") else: # Simple Tablespace Option is hidden from the plugin. # This section will not get triggered until the option gets added back in schema.json