diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..5201195 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,74 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '24 4 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # â„šī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/src/common/utils.py b/src/common/utils.py index e6e0815..0907d37 100644 --- a/src/common/utils.py +++ b/src/common/utils.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# + import logging import random import time diff --git a/src/dboperations/dboperations.py b/src/dboperations/dboperations.py index a3e2095..1fc7e1c 100644 --- a/src/dboperations/dboperations.py +++ b/src/dboperations/dboperations.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# + import logging import random import time @@ -37,7 +41,6 @@ def stop_mysql(port,connection,baseDir,vdbConn,pwd): if(port_stat == Status.ACTIVE): logger.debug("DB is Running. Shutting down.") shutdown_cmd = "%s/bin/mysqladmin %s'%s' --protocol=TCP --port=%s shutdown" % (baseDir,vdbConn,pwd,port) - logger.debug("Shutdown Command: {}".format(shutdown_cmd)) result = libs.run_bash(connection, shutdown_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() @@ -46,7 +49,7 @@ def stop_mysql(port,connection,baseDir,vdbConn,pwd): logger.debug("There was an error trying to shutdown the database : "+error) raise MySQLShutdownException(error) else: - logger.debug("Output: "+output) + logger.debug("DB shutdown completed") time.sleep(25) if(Status.ACTIVE == get_port_status(port,connection)): logger.debug("KILL") @@ -97,7 +100,6 @@ def get_port_status(port,connection): except Exception as err: logger.debug("Port Check Failed for second cmd: "+err.message) logger.debug("Port Status Response >") - logger.debug(output) if output== "": logger.debug("MySQL DB is NOT RUNNING at Port:"+myport) @@ -130,7 +132,6 @@ def start_mysql(installPath,baseDir,mountPath,port,serverId,connection): if(port_stat == Status.INACTIVE): logger.debug("DB is not running. Starting the MySQL DB") start_cmd=get_start_cmd(installPath,baseDir,mountPath,port,serverId) - logger.debug("Startup Command: {}".format(start_cmd)) result = libs.run_bash(connection, start_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() @@ -139,7 +140,7 @@ def start_mysql(installPath,baseDir,mountPath,port,serverId,connection): logger.debug("There was an error trying to start the DB : "+error) raise MySQLStartupException(error) else: - logger.debug("Output: "+output) + logger.debug("DB Started") time.sleep(25) if(Status.ACTIVE == get_port_status(port,connection)): logger.debug("DB Started Successfully") @@ -168,12 +169,11 @@ def start_slave(connection,installPath,port,connString,username,pwd,hostIp): raise Exception("One of the required params for MySQL Connection is empty") else: start_slave_cmd=CommandFactory.start_replication(connection,installPath,port,connString,username,pwd,hostIp) - logger.debug("Connection String with {}".format(start_slave_cmd)) try: logger.debug("Starting Slave") result = libs.run_bash(connection, start_slave_cmd,environment_vars,check=True) output = result.stdout.strip() - logger.debug("Start Slave Output: {}".format(output)) + logger.debug("Start Slave Completed") except Exception as err: logger.debug("Starting Slave Failed: "+err.message) raise err @@ -187,7 +187,6 @@ def stop_slave(connection,installPath,port,connString,username,pwd,hostIp): raise Exception("One of the required params for MySQL Connection is empty") else: stop_slave_cmd=CommandFactory.stop_replication(connection,installPath,port,connString,username,pwd,hostIp) - logger.debug("Connection String with {}".format(stop_slave_cmd)) try: logger.debug("Stopping Replication") result = libs.run_bash(connection, stop_slave_cmd,environment_vars,check=True) @@ -195,9 +194,9 @@ def stop_slave(connection,installPath,port,connString,username,pwd,hostIp): _bashErrMsg=result.stderr.strip() _bashErrCode=result.exit_code if _bashErrCode!=0: - logger.debug("Stopping Slave was not succesful") + logger.debug("Stopping Slave was not successful") raise Exception(_bashErrMsg) - logger.debug("Start Slave Response: {}".format(_output)) + logger.debug("Stop Replication successful") except Exception as err: logger.debug("Stop Replication Failed Due To: "+err.message) logger.debug("Ignoring and continuing") @@ -213,7 +212,6 @@ def get_connection_cmd(installPath,port,connString,username,pwd,hostIp): raise ValueError("One of the required params for MySQL Connection is empty") else: connection_cmd=CommandFactory.connect_to_mysql(installPath,port,connString,username,pwd,hostIp) - logger.debug("connaction_cmd >"+connection_cmd) return connection_cmd def get_start_cmd(installPath,baseDir,mountPath,port,serverId): diff --git a/src/plugin_runner.py b/src/plugin_runner.py index ecd48bf..d2a3ea0 100644 --- a/src/plugin_runner.py +++ b/src/plugin_runner.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# + import pkgutil import logging import sys @@ -75,16 +79,16 @@ def stop_staging(staged_source, repository, source_config): pluginops.stop_staging(staged_source, repository, source_config) @plugin.linked.pre_snapshot() -def linked_pre_snapshot(staged_source, repository, source_config, snapshot_parameters): +def linked_pre_snapshot(staged_source, repository, source_config, optional_snapshot_parameters): logger.debug("linked_pre_snapshot > Start ") # Start Staging if not already running. - pluginops.linked_pre_snapshot(staged_source, repository, source_config, snapshot_parameters) + pluginops.linked_pre_snapshot(staged_source, repository, source_config, optional_snapshot_parameters) logger.debug(" linked_pre_snapshot > End ") @plugin.linked.post_snapshot() -def linked_post_snapshot(staged_source,repository,source_config,snapshot_parameters): +def linked_post_snapshot(staged_source,repository,source_config,optional_snapshot_parameters): logger.debug("linked_post_snapshot - Start ") - snapshot = pluginops.linked_post_snapshot(staged_source,repository,source_config,snapshot_parameters) + snapshot = pluginops.linked_post_snapshot(staged_source,repository,source_config,optional_snapshot_parameters) linked_status(staged_source, repository, source_config) logger.debug("linked_post_snapshot - End ") return snapshot @@ -105,9 +109,6 @@ def configure(virtual_source, snapshot, repository): def reconfigure(virtual_source, repository, source_config, snapshot): logger.debug("virtual.reconfigure > Start") start(virtual_source, repository, source_config) - logger.debug(source_config) - logger.debug("Snapshot") - logger.debug(snapshot) #srcConfig = configure(virtual_source,snapshot,repository) logger.debug("virtual.reconfigure > End") virtual_status(virtual_source, repository, source_config) @@ -140,7 +141,6 @@ def virtual_post_snapshot(virtual_source, repository, source_config): snapshot.snap_backup_path="" snapshot.snap_time=utils.get_current_time() logger.debug("SnapShot Definition Created") - logger.debug(snapshot) return snapshot @plugin.virtual.start() diff --git a/src/pluginops/pluginops.py b/src/pluginops/pluginops.py index a7cbd4b..54e50f2 100644 --- a/src/pluginops/pluginops.py +++ b/src/pluginops/pluginops.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 by Delphix. All rights reserved. +# Copyright (c) 2020, 2023 by Delphix. All rights reserved. # ####################################################################################################################### @@ -15,6 +15,7 @@ import json import re import os +import json from datetime import datetime from common import utils,constants #from dboperations import dboperations @@ -124,8 +125,6 @@ def start_staging(staged_source, repository, source_config): result = libs.run_bash(staged_source.staged_connection, start_staging_script,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() - logger.debug("output:"+output) - logger.debug("error:"+error) exit_code = result.exit_code if exit_code !=0: logger.debug("There was an error> "+error) @@ -151,8 +150,6 @@ def start_staging(staged_source, repository, source_config): result = libs.run_bash(staged_source.staged_connection, start_staging_script,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() - logger.debug("output:"+output) - logger.debug("error:"+error) exit_code = result.exit_code if exit_code !=0: logger.debug("Error is : "+error) @@ -188,7 +185,7 @@ def stop_staging(staged_source, repository, source_config): logger.debug("Error is : "+error) raise LinkingException("Exception while stopping staging:"+error) else: - logger.debug("Stop Staging - Successful: "+output) + logger.debug("Stop Staging - Successful") elif staged_source.parameters.d_source_type == "Manual Backup Ingestion": logger.debug("plugin_operations.stop_staging > Manual Backup Ingestion") library_script=pkgutil.get_data('resources','library.sh') @@ -210,7 +207,7 @@ def stop_staging(staged_source, repository, source_config): logger.debug("Error is : "+error) raise LinkingException("Exception while stopping staging:"+error) else: - logger.debug("Stop Staging - Successful: "+output) + logger.debug("Stop Staging - Successful") else : logger.debug("dSourceType is Simple Tablespace Backup. ") library_script=pkgutil.get_data('resources','library.sh') @@ -232,7 +229,7 @@ def stop_staging(staged_source, repository, source_config): logger.debug("Error is : "+error) raise LinkingException("Exception while stopping staging:"+error) else: - logger.debug("Stop Staging - Successful: "+output) + logger.debug("Stop Staging - Successful") #TODO: Integration ################################################## @@ -321,18 +318,17 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.error(err) raise err else: - logger.debug("Pre-Snapshot/Restore successful "+output) + logger.debug("Pre-Snapshot/Restore successful") logger.debug("Restoring Backup to Stage") restore_script = pkgutil.get_data('resources', 'restore_stage.sh') result = libs.run_bash(staged_source.staged_connection, restore_script,environment_vars,check=False) - logger.debug(result) output = result.stdout.strip() std_err=result.stderr.strip() exit_code = result.exit_code logger.debug(std_err) logger.debug(exit_code) if exit_code == 0: - logger.debug("Creation of Staging DB(Pre-Snapshot) successful."+output) + logger.debug("Creation of Staging DB(Pre-Snapshot) successful") else: logger.debug("There was an error while creating the staging DB.Check error.log for details.") err = utils.process_exit_codes(exit_code,"DBLINK",std_err) @@ -369,7 +365,7 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.error(err) raise err else: - logger.debug("Pre-Snapshot/Restore_DB successful "+output) + logger.debug("Pre-Snapshot/Restore_DB successful") else: # Simple Tablespace Option is hidden from the plugin. # This section will not get triggered until the option gets added back in schema.json @@ -407,7 +403,7 @@ def linked_pre_snapshot(staged_source, repository, source_config, snapshot_param logger.debug("There was an error while resync : "+error) raise LinkingException("Exception in pre-snapshot/restore_db:"+error) else: - logger.debug("Pre-Snapshot/Restore_DB successful "+output) + logger.debug("Pre-Snapshot/Restore_DB successful") # Simple Tablespace Option is hidden from the plugin. # This section will not get triggered until the option gets added back in schema.json @@ -471,7 +467,6 @@ def linked_post_snapshot( logger.debug("plugin_opertions.linked_post_snapshot - Start ") dSourceType = staged_source.parameters.d_source_type start_staging(staged_source,repository,source_config) - logger.debug(snapshot_parameters) mount_path=staged_source.parameters.mount_path snapshot = SnapshotDefinition(validate=False) snapshot.snapshot_id= str(utils.get_snapshot_id()) @@ -482,7 +477,6 @@ def linked_post_snapshot( snapshot.snap_pass=staged_source.parameters.staging_pass snapshot.snap_backup_path=staged_source.parameters.backup_path snapshot.snap_time=utils.get_current_time() - logger.debug(snapshot) logger.debug("linked_post_snapshot - End ") return snapshot @@ -510,7 +504,7 @@ def linked_status(staged_source, repository, source_config): logger.debug("Exception while checking Staging DB Status : "+error) #ignore status? else: - logger.debug("Staging Status Check: "+output) + logger.debug("Staging Status Check successful") if output == "ACTIVE": return Status.ACTIVE else: @@ -532,11 +526,9 @@ def configure(virtual_source, snapshot, repository): ) logger.debug("Mount Path:"+mount_path) logger.debug("Snapshot Settings:") - logger.debug(snapshot) logger.debug("Snapshot_id"+snapshot.snapshot_id) logger.debug("Config Settings: ") config_settings_prov = virtual_source.parameters.config_settings_prov - logger.debug(config_settings_prov) config_params="" ################################################################### # TODO: Operation fails if there are config settings. Must revisit. @@ -576,12 +568,11 @@ def configure(virtual_source, snapshot, repository): } configure_script = pkgutil.get_data('resources', 'provision.sh') result = libs.run_bash(virtual_source.connection, configure_script,environment_vars,check=False) - logger.debug(result) output = result.stdout.strip() std_err=result.stderr.strip() exit_code = result.exit_code if exit_code == 0: - logger.debug("Pre-Snapshot/Restore_DB successful "+output) + logger.debug("Pre-Snapshot/Restore_DB successful") else: err = utils.process_exit_codes(exit_code,"PROVISION",std_err) logger.debug("There was an error while provisioning.Check error.log for details.") @@ -610,7 +601,6 @@ def stop_mysql(port,connection,baseDir,user,pwd,host): if(port_stat == Status.ACTIVE): logger.debug("DB is Running. Shutting down.") shutdown_cmd = "%s/bin/mysqladmin %s'%s' --protocol=TCP --port=%s shutdown" % (baseDir,vdbConn,pwd,port) - logger.debug("Shutdown Command: {}".format(shutdown_cmd)) result = libs.run_bash(connection, shutdown_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() @@ -619,7 +609,7 @@ def stop_mysql(port,connection,baseDir,user,pwd,host): logger.debug("There was an error trying to shutdown the database : "+error) #raise MySQLShutdownException(error) else: - logger.debug("Output: "+output) + logger.debug("Shutdown of database successful") time.sleep(20) if(Status.ACTIVE == get_port_status(port,connection)): logger.debug("MySQL has not shutdown after 20 seconds. Killing process.") @@ -676,8 +666,7 @@ def get_port_status(port,connection): output = result.stdout.strip() except Exception as err: logger.debug("Port Check Failed for second cmd: "+err.message) - logger.debug("Port Status Response >") - logger.debug(output) + logger.debug("Get port status successful") if output== "": logger.debug("MySQL DB is NOT RUNNING at Port:"+myport) @@ -714,7 +703,6 @@ def start_mysql(installPath,baseDir,mountPath,port,serverId,connection): if(port_stat == Status.INACTIVE): logger.debug("DB is not running. Starting the MySQL DB") start_cmd=get_start_cmd(installPath,baseDir,mountPath,port,serverId) - logger.debug("Startup Command: {}".format(start_cmd)) result = libs.run_bash(connection, start_cmd,environment_vars,check=True) output = result.stdout.strip() error = result.stderr.strip() @@ -726,8 +714,6 @@ def start_mysql(installPath,baseDir,mountPath,port,serverId,connection): constants.ERR_START_ACTION, "ExitCode:{} \n {}".format(exit_code,error) ) - else: - logger.debug("Output: "+output) time.sleep(30) if(Status.ACTIVE == get_port_status(port,connection)): logger.debug("DB Started Successfully") @@ -750,12 +736,11 @@ def start_slave(connection,installPath,port,connString,username,pwd,hostIp): raise Exception("One of the required params for MySQL Connection is empty") else: start_slave_cmd=CommandFactory.start_replication(connection,installPath,port,connString,username,pwd,hostIp) - logger.debug("Connection String with {}".format(start_slave_cmd)) try: logger.debug("Starting Slave") result = libs.run_bash(connection, start_slave_cmd,environment_vars,check=True) output = result.stdout.strip() - logger.debug("Start Slave Output: {}".format(output)) + logger.debug("Start Slave successful") except Exception as err: logger.debug("Starting Slave Failed: "+err.message) raise err @@ -773,7 +758,6 @@ def stop_slave(connection,installPath,port,connString,username,pwd,hostIp): raise Exception("One of the required params for MySQL Connection is empty") else: stop_slave_cmd=CommandFactory.stop_replication(connection,installPath,port,connString,username,pwd,hostIp) - logger.debug("Connection String with {}".format(stop_slave_cmd)) try: logger.debug("Stopping Replication") result = libs.run_bash(connection, stop_slave_cmd,environment_vars,check=True) @@ -781,9 +765,9 @@ def stop_slave(connection,installPath,port,connString,username,pwd,hostIp): _bashErrMsg=result.stderr.strip() _bashErrCode=result.exit_code if _bashErrCode!=0: - logger.debug("Stopping Slave was not succesful") + logger.debug("Stopping Slave was not successful") raise Exception(_bashErrMsg) - logger.debug("Start Slave Response: {}".format(_output)) + logger.debug("Stopping Slave successful") except Exception as err: logger.debug("Stop Replication Failed Due To: "+err.message) logger.debug("Ignoring and continuing") @@ -802,7 +786,6 @@ def get_connection_cmd(installPath,port,connString,username,pwd,hostIp): raise ValueError("One of the required params for MySQL Connection is empty") else: connection_cmd=CommandFactory.connect_to_mysql(installPath,port,connString,username,pwd,hostIp) - logger.debug("connaction_cmd >"+connection_cmd) return connection_cmd ################################################## @@ -850,16 +833,13 @@ def repository_discovery(source_connection): logger.debug("Error is : "+error) raise RepositoryDiscoveryError("Exception while discovering:"+error) else: - logger.debug("Output: "+output) #process repository json repos_js=json.loads(output) # print the keys and values for repo_js in repos_js: - #logger.debug("Adding repository:"+repo_js+" to list") path = repo_js['installPath'] version = repo_js['version'] prettyName = repo_js['prettyName'].split("/bin")[1] repository = RepositoryDefinition(name=prettyName, install_path=path, version=version) repositories.append(repository) - logger.debug("output:"+output) return repositories diff --git a/src/resources/library.sh b/src/resources/library.sh index ff5d0a5..9186353 100644 --- a/src/resources/library.sh +++ b/src/resources/library.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 by Delphix. All rights reserved. +# Copyright (c) 2020, 2023 by Delphix. All rights reserved. # # library.sh # @@ -295,8 +295,7 @@ stopDatabase() { ZBASEDIR=`echo "${ZRESULTS}" | $DLPX_BIN_JQ --raw-output ".baseDir"` ZLOGSYNC=`echo "${ZRESULTS}" | $DLPX_BIN_JQ --raw-output ".logSync"` - echo "ZCONN:" - echo "$ZCONN" + masklog "$ZCONN" # Found valid process ... if [[ "${ZPORT}" != "" ]] && [[ "${ZPSID}" != "" ]] @@ -495,7 +494,6 @@ function terminate { # Runs a given command and exits with code if error function command_runner { - masklog "command_runner: CMD : $1" return_msg=$(eval $1 2>&1 1>&2 > /dev/null) return_code=$? log "Return Status: ${return_code}" diff --git a/src/resources/provision.sh b/src/resources/provision.sh index 5b4b3ab..76dd1e6 100644 --- a/src/resources/provision.sh +++ b/src/resources/provision.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log @@ -216,13 +216,6 @@ outputJSON=$($DLPX_BIN_JQ ".baseDir = $(jqQuote "$CONFIG_BASEDIR")" <<< "$output outputJSON=$($DLPX_BIN_JQ ".dbName = $(jqQuote "$prettyName")" <<< "$outputJSON") printf "$outputJSON" > "$DLPX_OUTPUT_FILE" -log "Output: $outputJSON" echo "${prettyName}" -#log "Environment: " -#export DLPX_LIBRARY_SOURCE="" -#export REPLICATION_PASS="" -#export SNAPSHOT_METADATA="" -#export STAGINGPASS="" -#env | sort >>$DEBUG_LOG log "End" exit 0 diff --git a/src/resources/restore.sh b/src/resources/restore.sh index 39300c1..91accc0 100644 --- a/src/resources/restore.sh +++ b/src/resources/restore.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # Program Name PGM_NAME='restore.sh' @@ -139,6 +139,6 @@ export DLPX_LIBRARY_SOURCE="" export REPLICATION_PASS="" export STAGINGPASS="" export SOURCEPASS="" -env | sort >>$DEBUG_LOG +env | grep -v 'PASS' | sort >>$DEBUG_LOG log "-- End --" exit 0 diff --git a/src/resources/restore_stage.sh b/src/resources/restore_stage.sh index 328d711..f9cc31d 100644 --- a/src/resources/restore_stage.sh +++ b/src/resources/restore_stage.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. PGM_NAME='restore_stage.sh' # Load Library ... @@ -43,7 +43,6 @@ STAGINGPASS=`echo "'"${STAGINGPASS}"'"` masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" @@ -101,18 +100,14 @@ ${MYSQLD}/mysqld --initialize --user=mysql --datadir=${NEW_DATA_DIR} --log-error PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` # sudo grep 'temporary password' ${NEW_DATA_DIR}/mysqld.log` # 2019-04-11T14:40:34.032576Z 1 [Note] A temporary password is generated for root@localhost: L0qXNZ8?C3Us -log "init temporary password: ${PWD_LINE}" TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # These temporary passwords contain special characters so need to wrap in single / literal quotes ... TMP_PWD=`echo "'"$TMP_PWD"'"` -log "Temporary Password: ${TMP_PWD}" masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" -log "Creation Results: ${RESULTS}" ############################################################ log "Creating Staging Directories on NFS Mounted Path from Delphix" @@ -390,7 +385,6 @@ command_runner "${CMD}" 5 # masklog "Staging Connection Prior to updaging password : ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection after updating password: ${STAGING_CONN}" @@ -588,6 +582,6 @@ export DLPX_LIBRARY_SOURCE="" export REPLICATION_PASS="" export STAGINGPASS="" export SOURCEPASS="" -env | sort >>$DEBUG_LOG +env | grep -v 'PASS' | sort >>$DEBUG_LOG log " <<>${DEBUG_LOG} 2>&1 PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` -# sudo grep 'temporary password' ${NEW_DATA_DIR}/mysqld.log` -log "init temporary password: ${PWD_LINE}" TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # These temporary passwords contain special characters so need to wrap in single / literal quotes ... TMP_PWD=`echo "'"$TMP_PWD"'"` -log "Temporary Password: ${TMP_PWD}" masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` masklog "Staging Connection: ${STAGING_CONN}" -masklog "Creation Results: ${RESULTS}" ############################################################ ## @@ -341,7 +335,8 @@ fi ######################################################################## # Change Password for Staging Conn ... CMD="${INSTALL_BIN}/mysql ${STAGING_CONN} --connect-expired-password -se \"ALTER USER 'root'@'localhost' IDENTIFIED BY ${STAGINGPASS};UPDATE mysql.user SET authentication_string=PASSWORD(${STAGINGPASS}) where USER='root';FLUSH PRIVILEGES;\"" -masklog "Final Command to Change Password is : ${CMD}" +CMDLOG="${INSTALL_BIN}/mysql ${STAGING_CONN} --connect-expired-password -se \"ALTER USER 'root'@'localhost' IDENTIFIED BY '********';UPDATE mysql.user SET authentication_string=PASSWORD('********') where USER='root';FLUSH PRIVILEGES;\"" +masklog "Final Command to Change Password is : ${CMDLOG}" command_runner "${CMD}" 5 #eval ${CMD} 1>>${DEBUG_LOG} 2>&1 @@ -349,7 +344,6 @@ command_runner "${CMD}" 5 # Update Staging Connection with supplied password ... masklog "Staging Connection Prior to updating password : ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" "${STAGINGHOSTIP}" ) -echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` log "============================================================" masklog "Staging Connection after updating password: ${STAGING_CONN}" @@ -364,7 +358,6 @@ masklog "Granting privileges command: ${CMD}" return_msg=$(eval ${CMD} 2>&1 1>&2 > /dev/null) return_code=$? log "Return Status: ${return_code}" -log "Return message:${return_msg}" if [ $return_code != 0 ]; then errorlog "Unable to grant required permissions to delphix database user. This may have to be done manually" fi @@ -474,7 +467,7 @@ command_runner "${CMD}" 11 log "Environment:" export DLPX_LIBRARY_SOURCE="" export STAGINGPASS="" -env | sort >>$DEBUG_LOG +env | grep -v 'PASS' | sort >>$DEBUG_LOG log "End" echo "Staging Started" exit 0 diff --git a/src/resources/restore_stage_si.sh b/src/resources/restore_stage_si.sh index 22ed1e3..f3c4470 100644 --- a/src/resources/restore_stage_si.sh +++ b/src/resources/restore_stage_si.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log @@ -92,16 +92,13 @@ ${MYSQLD}/mysqld --initialize --user=mysql --datadir=${NEW_DATA_DIR} --log-error PWD_LINE=`cat ${NEW_DATA_DIR}/mysqld.log | grep 'temporary password'` # sudo grep 'temporary password' ${NEW_DATA_DIR}/mysqld.log` # 2019-04-11T14:40:34.032576Z 1 [Note] A temporary password is generated for root@localhost: L0qXNZ8?C3Us -log "init temporary password: ${PWD_LINE}" TMP_PWD=`echo "${PWD_LINE}" | ${AWK} -F": " '{print $2}' | xargs` # # These temporary passwords contain special characters so need to wrap in single / literal quotes ... # TMP_PWD=`echo "'"$TMP_PWD"'"` -log "Temporary Password: ${TMP_PWD}" -##log "Creation Results: ${RESULTS}" log "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${TMP_PWD}" "${STAGINGPORT}" ) @@ -211,8 +208,9 @@ fi # Change Password for Staging Conn ... # CMD="${INSTALL_BIN}/mysql ${STAGING_CONN} --connect-expired-password -se \"ALTER USER 'root'@'localhost' IDENTIFIED BY ${STAGINGPASS}; FLUSH PRIVILEGES;\"" +CMDLOG="${INSTALL_BIN}/mysql ${STAGING_CONN} --connect-expired-password -se \"ALTER USER 'root'@'localhost' IDENTIFIED BY '********'; FLUSH PRIVILEGES;\"" -log "Change Password: ${CMD}" +log "Change Password: ${CMDLOG}" eval ${CMD} 1>>${DEBUG_LOG} 2>&1 @@ -223,7 +221,7 @@ log "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" ) #log "${RESULTS}" STAGING_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` -log "Staging Connection: ${STAGING_CONN}" +masklog "Staging Connection: ${STAGING_CONN}" log "Validating Restore Databases ..." CMD="${INSTALL_BIN}/mysql ${STAGING_CONN} -e \"show databases;\"" @@ -263,8 +261,6 @@ eval ${CMD} 1>>${DEBUG_LOG} 2>&1 log "Environment: " export DLPX_LIBRARY_SOURCE="" -#export REPLICATION_PASS="" -#export STAGINGPASS="" -env | sort >>$DEBUG_LOG +env | grep -v 'PASS' | sort >>$DEBUG_LOG log "------------------------- End" exit 0 diff --git a/src/resources/startVirtual.sh b/src/resources/startVirtual.sh index 395451f..6aff22c 100644 --- a/src/resources/startVirtual.sh +++ b/src/resources/startVirtual.sh @@ -1,4 +1,7 @@ # +# Copyright (c) 2021, 2023 by Delphix. All rights reserved. +# +# set -x PGM_NAME='startVirtual.sh' diff --git a/src/resources/stopVirtual.sh b/src/resources/stopVirtual.sh index f58243e..1890624 100644 --- a/src/resources/stopVirtual.sh +++ b/src/resources/stopVirtual.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log set -x @@ -27,7 +27,7 @@ log "VDB Connection: ${VDBCONN}" RESULTS=$( buildConnectionString "${VDBCONN}" "${VDBPASS}" "${PORT}" ) #log "${RESULTS}" VDB_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` -log "Staging Connection: ${VDB_CONN}" +masklog "Staging Connection: ${VDB_CONN}" RESULTS=$( portStatus "${PORT}" ) #echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".status" diff --git a/src/resources/tablespaces.sh b/src/resources/tablespaces.sh index c888a31..f5ae183 100644 --- a/src/resources/tablespaces.sh +++ b/src/resources/tablespaces.sh @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2018, 2023 by Delphix. All rights reserved. # ##DEBUG## In Delphix debug.log @@ -38,11 +38,11 @@ NEW_TMP_DIR="${NEW_MOUNT_DIR}/tmp" #SOURCE_CONN="-uroot -pdelphix --protocol=TCP --port=3306" SOURCEPASS=`echo "'"${SOURCEPASS}"'"` -log "Source Connection: ${SOURCECONN}" +masklog "Source Connection: ${SOURCECONN}" RESULTS=$( buildConnectionString "${SOURCECONN}" "${SOURCEPASS}" "${SOURCEPORT}" ) #log "${RESULTS}" SOURCE_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` -log "Source Connection: ${SOURCE_CONN}" +masklog "Source Connection: ${SOURCE_CONN}" SOURCE_DATA_DIR="${SOURCEDATADIR}" # "/usr/local/mysql/data" SOURCE_BASE_DIR="${SOURCEBASEDIR}" # "/usr/local/mysql" @@ -55,11 +55,11 @@ SOURCE_TABLES="${SOURCETABLES}" # "ALL" "employees,patient,patient_details,m #TARGET_CONN="-uroot -pdelphix --protocol=TCP --port=3307" STAGINGPASS=`echo "'"${STAGINGPASS}"'"` -log "Staging Connection: ${STAGINGCONN}" +masklog "Staging Connection: ${STAGINGCONN}" RESULTS=$( buildConnectionString "${STAGINGCONN}" "${STAGINGPASS}" "${STAGINGPORT}" ) #log "${RESULTS}" TARGET_CONN=`echo "${RESULTS}" | $DLPX_BIN_JQ --raw-output ".string"` -log "Staging Connection: ${TARGET_CONN}" +masklog "Staging Connection: ${TARGET_CONN}" #TARGET_DATA_DIR="/mnt/provision/my_stage/data" TARGET_DATA_DIR="${STAGINGDATADIR}"/data