From b6f9b7e54dbb013a4173bacd962a2d8eff37563c Mon Sep 17 00:00:00 2001 From: xue_meng_en <1836611252@qq.com> Date: Mon, 9 Jan 2023 20:11:24 +0800 Subject: [PATCH 01/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=90=AD=E5=BB=BA?= =?UTF-8?q?=E7=81=BE=E5=A4=87=E5=85=B3=E7=B3=BB=E5=A4=B1=E8=B4=A5=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/streaming_disaster_recovery/streaming_base.py | 1 + .../streaming_modules/streaming_diaster_recovery_start.py | 1 + .../streaming_modules/streaming_disaster_recovery_switchover.py | 1 + 3 files changed, 3 insertions(+) diff --git a/script/impl/streaming_disaster_recovery/streaming_base.py b/script/impl/streaming_disaster_recovery/streaming_base.py index acbc1a57..7f221dc4 100644 --- a/script/impl/streaming_disaster_recovery/streaming_base.py +++ b/script/impl/streaming_disaster_recovery/streaming_base.py @@ -2262,6 +2262,7 @@ class StreamingBase(object): if stream_disaster_step < 3: self.set_cmserver_guc("backup_open", "0", "set") self.stream_disaster_set_cmagent_guc("agent_backup_open", "0", "set") + self.set_stream_cluster_run_mode_guc("set", fail_over=True) self.write_streaming_step("3_set_backup_open_for_failover") # 4.Delete the relevant guc parameters and remove the disaster tolerance relationship # based on streaming disaster recovery cluster, No need to delete for switchover. diff --git a/script/impl/streaming_disaster_recovery/streaming_modules/streaming_diaster_recovery_start.py b/script/impl/streaming_disaster_recovery/streaming_modules/streaming_diaster_recovery_start.py index 39cc3e96..8f6abe16 100644 --- a/script/impl/streaming_disaster_recovery/streaming_modules/streaming_diaster_recovery_start.py +++ b/script/impl/streaming_disaster_recovery/streaming_modules/streaming_diaster_recovery_start.py @@ -123,6 +123,7 @@ class StreamingStartHandler(StreamingBase): if step >= 5: return self.logger.debug("Start fifth step of streaming start.") + self.set_stream_cluster_run_mode_guc("set") self.set_data_in_dcc(self.backup_open_key, "0", only_mode='primary') self.set_data_in_dcc(self.backup_open_key, "2", only_mode='disaster_standby') self.stop_cluster_by_node(only_mode='disaster_standby') diff --git a/script/impl/streaming_disaster_recovery/streaming_modules/streaming_disaster_recovery_switchover.py b/script/impl/streaming_disaster_recovery/streaming_modules/streaming_disaster_recovery_switchover.py index 5318effe..25ff78e7 100644 --- a/script/impl/streaming_disaster_recovery/streaming_modules/streaming_disaster_recovery_switchover.py +++ b/script/impl/streaming_disaster_recovery/streaming_modules/streaming_disaster_recovery_switchover.py @@ -104,6 +104,7 @@ class StreamingSwitchoverHandler(StreamingBase): if stream_disaster_step < 3: self.set_cmserver_guc("backup_open", "2", "set") self.set_cmagent_guc("agent_backup_open", "2", "set") + self.set_stream_cluster_run_mode_guc("set") self.write_streaming_step("3_set_backup_open_2_done") if stream_disaster_step < 4: self.update_streaming_info(StreamingConstants.ACTION_SWITCHOVER, "50%") -- Gitee From 711b93d8b153bd4f4086ac9409be21a941cdd05f Mon Sep 17 00:00:00 2001 From: chentingting <2393940156@qq.com> Date: Tue, 7 Mar 2023 17:16:09 +0800 Subject: [PATCH 02/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8D=87=E7=BA=A7?= =?UTF-8?q?=E5=89=8D=E7=BD=AE=E5=A4=B1=E8=B4=A5=E5=88=A0=E9=99=A4GPHOME?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/preinstall/PreinstallImpl.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/script/impl/preinstall/PreinstallImpl.py b/script/impl/preinstall/PreinstallImpl.py index 4eb869a6..fb868d88 100644 --- a/script/impl/preinstall/PreinstallImpl.py +++ b/script/impl/preinstall/PreinstallImpl.py @@ -36,6 +36,7 @@ from base_utils.os.file_util import FileUtil from domain_utils.cluster_file.package_info import PackageInfo from base_utils.os.password_util import PasswordUtil from base_utils.os.net_util import NetUtil +from base_utils.os.env_util import EnvUtil from domain_utils.cluster_file.profile_file import ProfileFile # action name @@ -1621,6 +1622,13 @@ class PreinstallImpl: else: FileUtil.removeFile(rmPath) elif os.path.isdir(rmPath): - FileUtil.removeDirectory(rmPath) + if not EnvUtil.is_fuzzy_upgrade( + self.context.user, + logger=self.context.logger, + env_file=self.context.mpprcFile): + FileUtil.removeDirectory(rmPath) + else: + self.context.logger.debug( + f'In upgrade process, no need to delete {rmPath}.') self.context.logger.logExit(str(e)) sys.exit(0) -- Gitee From 4ad71d02cf9e4be577ada4b803bfff0838508e5c Mon Sep 17 00:00:00 2001 From: shirley_zhengx Date: Tue, 21 Mar 2023 10:03:05 +0800 Subject: [PATCH 03/96] fix set guc upgrade_mode failed --- script/impl/upgrade/UpgradeImpl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index d8cee5bd..2bba63f5 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -3199,8 +3199,10 @@ class UpgradeImpl: self.context.logger.debug("Not post upgrade.") self.setUpgradeFromParam(self.context.oldClusterNumber) if self.context.action == const.ACTION_INPLACE_UPGRADE: - self.setUpgradeMode(1, "set") + # Must set guc after start cluster by setUpgradeMode, because checking guc + # needs to connect database to execute sql statement. self.start_strategy(is_final=False) + self.setUpgradeMode(1, "reload") self.touchInitFile() else: # the guc parameter upgrade_from need to restart cmagent to take effect -- Gitee From d36561a05f7b72fa536ccd6a1d13080f60b736c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=8F=B2?= Date: Fri, 24 Mar 2023 17:55:33 +0800 Subject: [PATCH 04/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8DCM=E7=8B=AC=E7=AB=8B?= =?UTF-8?q?=E5=8D=87=E7=BA=A7=E5=A4=B1=E8=B4=A5=EF=BC=8C=E6=8A=A5=E9=94=99?= =?UTF-8?q?module=20'sys'=20has=20no=20attribute=20'exist'=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/local/upgrade_cm_utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/local/upgrade_cm_utility.py b/script/local/upgrade_cm_utility.py index b77f38b4..a544fcd5 100644 --- a/script/local/upgrade_cm_utility.py +++ b/script/local/upgrade_cm_utility.py @@ -310,7 +310,7 @@ class UpgradeCmPrepareUtility(UpgradeCmUtility): if file_not_exist_list: self.logger.error("The CM package file is incorrect.") self.logger.error("The following file does not exist: {0}".format(file_not_exist_list)) - sys.exist(-1) + sys.exit(-1) self.logger.log("Upgrade CM package is correct.") def generate_new_upgrade_package(self, decompress_dir): -- Gitee From 88522a16c1d89339e99b22bf93e7bd49b3c1bfcd Mon Sep 17 00:00:00 2001 From: liuheng Date: Thu, 30 Mar 2023 15:38:24 +0800 Subject: [PATCH 05/96] =?UTF-8?q?=E4=BF=AE=E6=94=B9getNetWorkConfFile?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E4=B8=AD=E5=A4=A7=E5=B0=8F=E5=86=99=E4=B8=8D?= =?UTF-8?q?=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/common/Common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/gspylib/common/Common.py b/script/gspylib/common/Common.py index 87238638..7e07779d 100644 --- a/script/gspylib/common/Common.py +++ b/script/gspylib/common/Common.py @@ -598,7 +598,7 @@ class DefaultValue(): NetWorkConfFile = "" distname, version, idnum = LinuxDistro.linux_distribution() distname = distname.lower() - if (distname in ("redhat", "centos", "euleros", "openEuler", "fusionos")): + if (distname in ("redhat", "centos", "euleros", "openeuler", "fusionos")): NetWorkConfFile = "%s/ifcfg-%s" % (RedHatNetWorkConfPath, networkCardNum) else: -- Gitee From 7a8c191aec73f5febf4224e16a62f7d5806fd71b Mon Sep 17 00:00:00 2001 From: chentingting <2393940156@qq.com> Date: Tue, 21 Mar 2023 15:53:32 +0800 Subject: [PATCH 06/96] =?UTF-8?q?=E5=8F=96=E6=B6=88=E6=95=B4=E4=BD=93?= =?UTF-8?q?=E5=AF=B9=E7=A3=81=E7=9B=98=E8=B0=83=E5=BA=A6=E7=9A=84=E6=A3=80?= =?UTF-8?q?=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_checkos | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/gs_checkos b/script/gs_checkos index 038d6950..55ab92d4 100644 --- a/script/gs_checkos +++ b/script/gs_checkos @@ -1489,8 +1489,12 @@ def main(): if ("A" in g_opts.item_detail): itemList = CHECK_ITEMNUMLIST + if 'A10' in itemList: + itemList.remove('A10') elif ("B" in g_opts.item_detail): itemList = SET_ITEMNUMLIST + if 'B4' in itemList: + itemList.remove('B4') else: sortList = sorted(g_opts.item_detail) itemList = sortList -- Gitee From ef52b5896e8a0d04ba191a8d757ee6f23bc98563 Mon Sep 17 00:00:00 2001 From: chentingting <2393940156@qq.com> Date: Tue, 14 Mar 2023 11:15:45 +0800 Subject: [PATCH 07/96] fix getcap not found --- script/base_utils/os/file_util.py | 8 +++++--- script/gspylib/common/ErrorCode.py | 3 ++- script/local/Uninstall.py | 14 +++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/script/base_utils/os/file_util.py b/script/base_utils/os/file_util.py index 40fbce0a..55e20b95 100644 --- a/script/base_utils/os/file_util.py +++ b/script/base_utils/os/file_util.py @@ -405,7 +405,8 @@ class FileUtil(object): Add the read and write permissions of some root users. ''' - cmd = 'setcap {}+ep {}'.format(cap_mode, path) + cmd = 'export PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin; ' + cmd = cmd + 'setcap {}+ep {}'.format(cap_mode, path) status, output = subprocess.getstatusoutput(cmd) if status != 0: raise Exception(ErrorCode.GAUSS_501["GAUSS_50107"] % path + @@ -416,10 +417,11 @@ class FileUtil(object): ''' Get the permissions of some root users. ''' - cmd = f'getcap {path}' + cmd = 'export PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin; ' + cmd = cmd + f'getcap {path}' status, output = subprocess.getstatusoutput(cmd) if status != 0: - raise Exception(ErrorCode.GAUSS_501["GAUSS_50107"] % path + + raise Exception(ErrorCode.GAUSS_501["GAUSS_50112"] % path + " Error:\n%s." % output + "The cmd is %s" % cmd) return output.strip() diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index 920d13bd..8964539d 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -131,7 +131,8 @@ class ErrorCode(): 'GAUSS_50109': "[GAUSS-50109] : Only a user with the root permission " "can check SSD information.", 'GAUSS_50110': "[GAUSS-50110] : Cannot execute this script on %s.", - 'GAUSS_50111': "[GAUSS-50111] : The %s directory has no permission." + 'GAUSS_50111': "[GAUSS-50111] : The %s directory has no permission.", + 'GAUSS_50112': "[GAUSS-50112] : Failed to get the permission of %s.", } ########################################################################### diff --git a/script/local/Uninstall.py b/script/local/Uninstall.py index 27622cb8..ec201fb3 100644 --- a/script/local/Uninstall.py +++ b/script/local/Uninstall.py @@ -108,7 +108,6 @@ class Uninstall(LocalBaseOM): ''' Deregistering a Disk in dss-mode ''' - self.logger.log("Start to unregist the lun.") gausshome = ClusterDir.getInstallDir(self.user) dsscmd = os.path.realpath(os.path.join(gausshome, 'bin', 'dsscmd')) perctrl = os.path.realpath(os.path.join(gausshome, 'bin', 'perctrl')) @@ -119,6 +118,7 @@ class Uninstall(LocalBaseOM): dss_home = EnvUtil.get_dss_home(self.user) cfg = os.path.join(dss_home, 'cfg', 'dss_inst.ini') if os.path.isfile(cfg): + self.logger.log("Start to unregist the lun.") Dss.unreg_disk(dss_home, logger=self.logger) self.logger.log("Successfully unregist the lun.") else: @@ -306,6 +306,18 @@ class Uninstall(LocalBaseOM): self.logger.log("Removing the installation directory.") try: + + dss_app = os.path.realpath( + os.path.join( + os.path.dirname(self.installPath), + f'dss_app_{os.path.realpath(self.installPath)[-8:]}')) + if os.path.isdir(dss_app): + for fn in os.listdir(dss_app): + fp = os.path.realpath(os.path.join(dss_app, fn)) + if os.path.isfile(fp): + os.remove(fp) + self.logger.debug("Remove path:%s." % fp) + fileList = os.listdir(self.installPath) for fileName in fileList: fileName = fileName.replace("/", "").replace("..", "") -- Gitee From 68462b1ccee7ea4e1a0cfffd6a4067c84c3c14b1 Mon Sep 17 00:00:00 2001 From: jianghongbo Date: Mon, 3 Apr 2023 14:41:19 +0800 Subject: [PATCH 08/96] =?UTF-8?q?om=E5=B1=8F=E8=94=BD=E7=94=A8=E6=88=B7pyt?= =?UTF-8?q?hon=E7=89=88=E6=9C=AC=E5=B7=AE=E5=BC=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_preinstall | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/script/gs_preinstall b/script/gs_preinstall index 7c6c8f6b..20a36f55 100644 --- a/script/gs_preinstall +++ b/script/gs_preinstall @@ -44,6 +44,45 @@ if "--unused-third-party" in sys.argv: import paramiko from gspylib.common.GaussLog import GaussLog + +source = os.path.join(sys.path[0], '../lib/bcrypt/lib3.' + \ + str(sys.version_info[1]), '_bcrypt.abi3.so') +dest = os.path.join(sys.path[0], '../lib/bcrypt/') +cmd = f"cp {source} {dest}" +(status, output) = subprocess.getstatusoutput(cmd) +if status != 0: + GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + +source = os.path.join(sys.path[0], '../lib/_cffi_backend_3.' + \ + str(sys.version_info[1]), '_cffi_backend.so') +dest = os.path.join(sys.path[0], '../lib/') +cmd = f"cp {source} {dest}" +(status, output) = subprocess.getstatusoutput(cmd) +if status != 0: + GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + + +source = os.path.join(sys.path[0], '../lib/cryptography/hazmat/bindings/lib3.' + \ + str(sys.version_info[1]), '*.so') +dest = os.path.join(sys.path[0], '../lib/cryptography/hazmat/bindings/') +cmd = f"cp {source} {dest}" +(status, output) = subprocess.getstatusoutput(cmd) +if status != 0: + GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + + +source = os.path.join(sys.path[0], '../lib/nacl/lib3.' + \ + str(sys.version_info[1]), '_sodium.abi3.so') +dest = os.path.join(sys.path[0], '../lib/nacl/') +cmd = f"cp {source} {dest}" +(status, output) = subprocess.getstatusoutput(cmd) +if status != 0: + GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + from gspylib.common.Common import DefaultValue from gspylib.common.ErrorCode import ErrorCode from gspylib.common.ParallelBaseOM import ParallelBaseOM -- Gitee From bb0c2f678c06d16067d9f8522eeac4f26cb6060d Mon Sep 17 00:00:00 2001 From: vimiix Date: Tue, 11 Apr 2023 10:33:32 +0800 Subject: [PATCH 09/96] =?UTF-8?q?fix(TaskPool):=E4=BF=AE=E5=A4=8D=E6=89=A7?= =?UTF-8?q?=E8=A1=8Cssh=E5=91=BD=E4=BB=A4=E6=97=B6=E6=89=BE=E4=B8=8D?= =?UTF-8?q?=E5=88=B0ssh=E5=8F=AF=E6=89=A7=E8=A1=8C=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/pssh/bin/TaskPool.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/gspylib/pssh/bin/TaskPool.py b/script/gspylib/pssh/bin/TaskPool.py index 1adeb5df..512116fd 100644 --- a/script/gspylib/pssh/bin/TaskPool.py +++ b/script/gspylib/pssh/bin/TaskPool.py @@ -277,8 +277,9 @@ class TaskThread(threading.Thread): ".bashrc") ssh_auth_sock = self.get_env_variable("SSH_AUTH_SOCK", bashrc_file) ssh_agent_pid = self.get_env_variable("SSH_AGENT_PID", bashrc_file) - env = {"SSH_AUTH_SOCK": ssh_auth_sock, - "SSH_AGENT_PID": ssh_agent_pid} + env = os.environ + env.update({"SSH_AUTH_SOCK": ssh_auth_sock, + "SSH_AGENT_PID": ssh_agent_pid}) self.proc = FastPopen(self.cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, close_fds=True) -- Gitee From 54621724149a6b0e573353543a94fef2b260708d Mon Sep 17 00:00:00 2001 From: liuheng Date: Thu, 13 Apr 2023 09:10:43 +0800 Subject: [PATCH 10/96] =?UTF-8?q?=E4=BF=AE=E6=94=B9CheckEtcHosts=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/inspection/items/os/CheckEtcHosts.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/script/gspylib/inspection/items/os/CheckEtcHosts.py b/script/gspylib/inspection/items/os/CheckEtcHosts.py index e0f4a774..f6d1c9b9 100644 --- a/script/gspylib/inspection/items/os/CheckEtcHosts.py +++ b/script/gspylib/inspection/items/os/CheckEtcHosts.py @@ -68,18 +68,10 @@ class CheckEtcHosts(BaseItem): flag = "Error_conflicts" else: IPMapping[ip] = host - if (len(IPInfo.split()) > 2 and IPInfo.split()[2] == "#Gauss"): - commentsMapping.append(IPInfo + " IP Hosts Mapping") - flag = "Error_comments" if (flag == "Normal"): self.result.rst = ResultStatus.OK self.result.val = "The /etc/hosts is configured correctly." - elif (flag == "Error_comments"): - self.result.rst = ResultStatus.NG - self.result.val = "The /etc/hosts has comments Mapping:\n" \ - + "\n".join( - commentsMapping) else: self.result.rst = ResultStatus.NG self.result.val = "The /etc/hosts has conflicts Mapping:\n" \ -- Gitee From 5d511689ec9366f96afcd0d35d81dee872a5d1dd Mon Sep 17 00:00:00 2001 From: leiziwei Date: Wed, 29 Mar 2023 10:24:50 +0800 Subject: [PATCH 11/96] =?UTF-8?q?opengauss=E9=80=82=E9=85=8Duos=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.sh | 6 ++++-- build/get_PlatForm_str.sh | 2 ++ script/gspylib/os/gsplatform.py | 13 ++++++++++++- script/local/LocalCheckOS.py | 2 +- script/os_platform/common.py | 9 ++++++--- script/os_platform/linux_platform.py | 9 +++++---- script/os_platform/rhel_platform.py | 3 +++ 7 files changed, 33 insertions(+), 11 deletions(-) diff --git a/build.sh b/build.sh index f1a02a1f..0274802a 100644 --- a/build.sh +++ b/build.sh @@ -48,7 +48,7 @@ done PLAT_FORM_STR=$(sh "${ROOT_DIR}/build/get_PlatForm_str.sh") if [ "${PLAT_FORM_STR}"x == "Failed"x ]; then - echo "We only support openEuler(aarch64), EulerOS(aarch64), FusionOS, CentOS platform." + echo "We only support openEuler(aarch64), EulerOS(aarch64), FusionOS, CentOS, UnionTech(X86) platform." exit 1; fi @@ -70,8 +70,10 @@ elif [ X$(echo $PLAT_FORM_STR | grep "ubuntu") != X"" ]; then dist_version="Ubuntu" elif [ X$(echo $PLAT_FORM_STR | grep "asianux") != X"" ]; then dist_version="Asianux" +elif [ X$(echo $PLAT_FORM_STR | grep "UnionTech") != X"" ]; then + dist_version="UnionTech" else - echo "We only support openEuler(aarch64), EulerOS(aarch64), FusionOS, CentOS, Ubuntu(x86) platform." + echo "We only support openEuler(aarch64), EulerOS(aarch64), FusionOS, CentOS, Ubuntu(x86), UnionTech(x86) platform." echo "Kernel is $kernel" exit 1 fi diff --git a/build/get_PlatForm_str.sh b/build/get_PlatForm_str.sh index cf8c84f9..2bd8af9b 100644 --- a/build/get_PlatForm_str.sh +++ b/build/get_PlatForm_str.sh @@ -37,6 +37,8 @@ function get_os_str() { os_str=asianux7.6_x86_64 elif [ "$os_name"x = "asianux"x ] && [ "$cpu_arc"x = "aarch64"x ]; then os_str=asianux7.5_aarch64 + elif [ "$os_name"x = "uos"x ] && [ "$cpu_arc"x = "x86_64"x ]; then + os_str=UnionTech_x86_64 else os_str="Failed" fi diff --git a/script/gspylib/os/gsplatform.py b/script/gspylib/os/gsplatform.py index d379af03..c67c9c7b 100644 --- a/script/gspylib/os/gsplatform.py +++ b/script/gspylib/os/gsplatform.py @@ -67,8 +67,9 @@ OPENEULER = "openeuler" ASIANUX = "asianux" DEBIAN = "debian" UBUNTU = "ubuntu" +UNIONTECH = "uniontech" SUPPORT_WHOLE_PLATFORM_LIST = [SUSE, REDHAT, CENTOS, EULEROS, FUSIONOS, - OPENEULER, KYLIN, ASIANUX, DEBIAN, UBUNTU] + OPENEULER, KYLIN, ASIANUX, DEBIAN, UBUNTU, UNIONTECH] # RedhatX platform SUPPORT_RHEL_SERIES_PLATFORM_LIST = [REDHAT, CENTOS, "kylin", "asianux"] SUPPORT_RHEL6X_VERSION_LIST = ["6.4", "6.5", "6.6", "6.7", "6.8", "6.9", "10"] @@ -102,6 +103,7 @@ PAK_REDHAT = "RedHat" PAK_ASIANUX = "Asianux" PAK_UBUNTU = "Ubuntu" PAK_SUSE = "SUSE" +PAK_UNIONTECH = "UnionTech" ####################################################### _supported_dists = ( @@ -1551,6 +1553,15 @@ class LinuxPlatform(GenericPlatform): prefixStr, packageVersion, PAK_UBUNTU, BIT_VERSION, postfixStr)) +<<<<<<< HEAD +======= + elif distname in UNIONTECH: + fileName = os.path.join(dirName, "./../../../", + "%s-%s-%s-%s.%s" % ( + prefixStr, packageVersion, + PAK_UNIONTECH, + BIT_VERSION, postfixStr)) +>>>>>>> 12441ba (opengauss适配uos系统) else: raise Exception(ErrorCode.GAUSS_519["GAUSS_51900"] + "Supported platforms are: %s." % str( diff --git a/script/local/LocalCheckOS.py b/script/local/LocalCheckOS.py index f66375b5..1fef3d32 100644 --- a/script/local/LocalCheckOS.py +++ b/script/local/LocalCheckOS.py @@ -1791,7 +1791,7 @@ def CheckPlatformInfo(): data.bits) g_logger.log("False %s %s" % (data.distname, platform_str)) return - elif (data.distname == "euleros" or data.distname == "openEuler" or data.distname == "FusionOS" or data.distname == "kylin"): + elif (data.distname == "euleros" or data.distname == "openEuler" or data.distname == "FusionOS" or data.distname == "kylin" or data.distname == "UnionTech"): mixed_type = "%s" % data.distname platform_str = "%s_%s_%s" % (data.distname, data.version, data.bits) elif (data.distname == "debian" or data.version == "buster/sid"): diff --git a/script/os_platform/common.py b/script/os_platform/common.py index 143c82c8..fa4eaa87 100644 --- a/script/os_platform/common.py +++ b/script/os_platform/common.py @@ -31,8 +31,9 @@ FUSIONOS = "fusionos" ASIANUX = "asianux" DEBIAN = "debian" UBUNTU = "ubuntu" +UNIONTECH = "uniontech" SUPPORT_WHOLE_PLATFORM_LIST = [SUSE, REDHAT, CENTOS, EULEROS, OPENEULER, KYLIN, - FUSIONOS, ASIANUX, DEBIAN, UBUNTU] + FUSIONOS, ASIANUX, DEBIAN, UBUNTU, UNIONTECH] # RedhatX platform SUPPORT_RHEL_SERIES_PLATFORM_LIST = [REDHAT, CENTOS, "kylin", "asianux"] SUPPORT_RHEL6X_VERSION_LIST = ["6.4", "6.5", "6.6", "6.7", "6.8", "6.9", "10"] @@ -44,6 +45,8 @@ SUPPORT_RHEL_SERIES_VERSION_LIST = (SUPPORT_RHEL6X_VERSION_LIST + SUPPORT_RHEL8X_VERSION_LIST) # EulerOS 2.3 -> 2.0 SP3 SUPPORT_EULEROS_VERSION_LIST = ["2.0"] +# UnionTech 20 +SUPPORT_UNIONTECH_VERSION_LIST = ["20"] # SuSE platform SUSE11 = "11" SUSE12 = "12" @@ -68,9 +71,9 @@ PAK_UBUNTU = "Ubuntu" PAK_KYLIN = "Kylin" PAK_SUSE = "SUSE" PAK_DEBIAN = "Debian" - +PAK_UNIONTECH = "UnionTech" ####################################################### _supported_dists = ( 'SuSE', 'debian', 'fedora', 'redhat', 'centos', 'euleros', "openEuler", 'mandrake', 'mandriva', 'rocks', 'slackware', 'yellowdog', 'gentoo', - "FusionOS", 'UnitedLinux', 'turbolinux', 'ubuntu', 'kylin', 'asianux') + "FusionOS", 'UnitedLinux', 'turbolinux', 'ubuntu', 'kylin', 'asianux', "UnionTech") diff --git a/script/os_platform/linux_platform.py b/script/os_platform/linux_platform.py index 1f21d235..a46ed0a4 100644 --- a/script/os_platform/linux_platform.py +++ b/script/os_platform/linux_platform.py @@ -23,9 +23,9 @@ import os from gspylib.common.ErrorCode import ErrorCode from os_platform.common import REDHAT, PAK_REDHAT, BIT_VERSION, \ - CENTOS, PAK_EULER, PAK_CENTOS, ASIANUX, SUSE, PAK_ASIANUX, \ + CENTOS, UNIONTECH, PAK_EULER, PAK_CENTOS, ASIANUX, SUSE, PAK_ASIANUX, \ EULEROS, OPENEULER, KYLIN, PAK_OPENEULER, SUPPORT_WHOLE_PLATFORM_LIST,\ - BLANK_SPACE, PAK_UBUNTU, DEBIAN, PAK_KYLIN, PAK_SUSE, PAK_DEBIAN, \ + BLANK_SPACE, PAK_UBUNTU, DEBIAN, PAK_KYLIN, PAK_UNIONTECH, PAK_SUSE, PAK_DEBIAN, \ FUSIONOS, PAK_FUSIONOS from os_platform.linux_distro import LinuxDistro @@ -213,10 +213,11 @@ class LinuxPlatform(object): self.package_file_path(prefix_str, packageVersion, PAK_CENTOS, postfix_str) ] - elif distname == OPENEULER or distname == KYLIN: + elif distname == OPENEULER or distname == KYLIN or distname == UNIONTECH: file_name_list = [ self.package_file_path(prefix_str, packageVersion, PAK_OPENEULER, postfix_str), - self.package_file_path(prefix_str, packageVersion, PAK_KYLIN, postfix_str) + self.package_file_path(prefix_str, packageVersion, PAK_KYLIN, postfix_str), + self.package_file_path(prefix_str, packageVersion, PAK_UNIONTECH, postfix_str) ] elif distname == DEBIAN: diff --git a/script/os_platform/rhel_platform.py b/script/os_platform/rhel_platform.py index d21ffbd5..6266b9dc 100644 --- a/script/os_platform/rhel_platform.py +++ b/script/os_platform/rhel_platform.py @@ -26,6 +26,7 @@ from gspylib.common.ErrorCode import ErrorCode from os_platform.common import BIT_VERSION, EULEROS, SUPPORT_EULEROS_VERSION_LIST, \ FUSIONOS, SUPPORT_RHEL_SERIES_PLATFORM_LIST, \ SUPPORT_RHEL_SERIES_VERSION_LIST, OPENEULER, CENTOS, \ + UNIONTECH, SUPPORT_UNIONTECH_VERSION_LIST, \ SUPPORT_RHEL7X_VERSION_LIST, DEBIAN, BLANK_SPACE from os_platform.linux_distro import LinuxDistro from os_platform.linux_platform import LinuxPlatform @@ -167,6 +168,8 @@ class RHELPlatform(LinuxPlatform): if ((bits == BIT_VERSION and ((dist_name.lower() == EULEROS and version[0:3] in SUPPORT_EULEROS_VERSION_LIST) or + (dist_name.lower() == UNIONTECH and version[0:3] in + SUPPORT_UNIONTECH_VERSION_LIST) or (dist_name.lower() in SUPPORT_RHEL_SERIES_PLATFORM_LIST and version[0:3] in SUPPORT_RHEL_SERIES_VERSION_LIST)) or (dist_name.lower() == OPENEULER) or -- Gitee From 6c1ced1617dc24535552730852db988813f6a219 Mon Sep 17 00:00:00 2001 From: leiziwei Date: Thu, 13 Apr 2023 10:25:48 +0800 Subject: [PATCH 12/96] =?UTF-8?q?=E8=A7=A3=E5=86=B3UOS=E5=90=88=E5=B9=B6?= =?UTF-8?q?=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/os/gsplatform.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/script/gspylib/os/gsplatform.py b/script/gspylib/os/gsplatform.py index c67c9c7b..1e12c307 100644 --- a/script/gspylib/os/gsplatform.py +++ b/script/gspylib/os/gsplatform.py @@ -1553,15 +1553,12 @@ class LinuxPlatform(GenericPlatform): prefixStr, packageVersion, PAK_UBUNTU, BIT_VERSION, postfixStr)) -<<<<<<< HEAD -======= elif distname in UNIONTECH: fileName = os.path.join(dirName, "./../../../", "%s-%s-%s-%s.%s" % ( prefixStr, packageVersion, PAK_UNIONTECH, BIT_VERSION, postfixStr)) ->>>>>>> 12441ba (opengauss适配uos系统) else: raise Exception(ErrorCode.GAUSS_519["GAUSS_51900"] + "Supported platforms are: %s." % str( -- Gitee From f30036207fc4bc82cf77f19ccfdf2b078f68192b Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Fri, 14 Apr 2023 11:44:05 +0800 Subject: [PATCH 13/96] =?UTF-8?q?gs=5Fsdr=E6=90=AD=E5=BB=BA=E6=B5=81?= =?UTF-8?q?=E5=BC=8F=E9=9B=86=E7=BE=A4=E8=AE=BE=E7=BD=AEmost=5Favailable?= =?UTF-8?q?=5Fsync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../streaming_modules/streaming_diaster_recovery_start.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/impl/streaming_disaster_recovery/streaming_modules/streaming_diaster_recovery_start.py b/script/impl/streaming_disaster_recovery/streaming_modules/streaming_diaster_recovery_start.py index 39cc3e96..491f1145 100644 --- a/script/impl/streaming_disaster_recovery/streaming_modules/streaming_diaster_recovery_start.py +++ b/script/impl/streaming_disaster_recovery/streaming_modules/streaming_diaster_recovery_start.py @@ -125,6 +125,7 @@ class StreamingStartHandler(StreamingBase): self.logger.debug("Start fifth step of streaming start.") self.set_data_in_dcc(self.backup_open_key, "0", only_mode='primary') self.set_data_in_dcc(self.backup_open_key, "2", only_mode='disaster_standby') + self.set_most_available(mode="reload", raise_error=False) self.stop_cluster_by_node(only_mode='disaster_standby') self.write_streaming_step("5_set_wal_segments_step") -- Gitee From d23ea53bd465c189c1e81e6e36e92d9f772ac7bc Mon Sep 17 00:00:00 2001 From: jianghongbo Date: Sat, 15 Apr 2023 15:48:56 +0800 Subject: [PATCH 14/96] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7=E8=87=B35.1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.sh | 2 +- simpleInstall/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index f1a02a1f..4f73ff9b 100644 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ declare binarylib_dir='None' declare module_name="openGauss" -declare version_number='5.0.0' +declare version_number='5.1.0' declare version_Kernel='92.298' ROOT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) echo "ROOT_DIR : $ROOT_DIR" diff --git a/simpleInstall/install.sh b/simpleInstall/install.sh index 88f49162..c1dc3de6 100644 --- a/simpleInstall/install.sh +++ b/simpleInstall/install.sh @@ -8,7 +8,7 @@ then fi readonly cur_path=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd && cd - &>/dev/null) -readonly version="5.0.0" +readonly version="5.1.0" source $cur_path"/common.sh" -- Gitee From 74fcb4b4dac19074234f24cfa1ca829f5bb14335 Mon Sep 17 00:00:00 2001 From: jianghongbo Date: Tue, 18 Apr 2023 10:52:50 +0800 Subject: [PATCH 15/96] fix I6W9B4 --- script/gs_preinstall | 75 ++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/script/gs_preinstall b/script/gs_preinstall index 20a36f55..87580bb8 100644 --- a/script/gs_preinstall +++ b/script/gs_preinstall @@ -30,6 +30,7 @@ from gspylib.common.CheckPythonVersion import checkPythonVersion, \ checkPythonVersion() check_python_compiler_option() from base_utils.os.file_util import FileUtil +from gspylib.common.GaussLog import GaussLog if "--unused-third-party" in sys.argv: package_path = os.path.dirname(os.path.realpath(__file__)) lib_path = os.path.join(package_path, "..", "lib") @@ -42,46 +43,44 @@ if "--unused-third-party" in sys.argv: import netifaces import cryptography import paramiko +else: + source = os.path.join(sys.path[0], '../lib/bcrypt/lib3.' + \ + str(sys.version_info[1]), '_bcrypt.abi3.so') + dest = os.path.join(sys.path[0], '../lib/bcrypt/') + cmd = f"cp {source} {dest}" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + + source = os.path.join(sys.path[0], '../lib/_cffi_backend_3.' + \ + str(sys.version_info[1]), '_cffi_backend.so') + dest = os.path.join(sys.path[0], '../lib/') + cmd = f"cp {source} {dest}" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) -from gspylib.common.GaussLog import GaussLog -source = os.path.join(sys.path[0], '../lib/bcrypt/lib3.' + \ - str(sys.version_info[1]), '_bcrypt.abi3.so') -dest = os.path.join(sys.path[0], '../lib/bcrypt/') -cmd = f"cp {source} {dest}" -(status, output) = subprocess.getstatusoutput(cmd) -if status != 0: - GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % - (output, cmd)) - -source = os.path.join(sys.path[0], '../lib/_cffi_backend_3.' + \ - str(sys.version_info[1]), '_cffi_backend.so') -dest = os.path.join(sys.path[0], '../lib/') -cmd = f"cp {source} {dest}" -(status, output) = subprocess.getstatusoutput(cmd) -if status != 0: - GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % - (output, cmd)) - - -source = os.path.join(sys.path[0], '../lib/cryptography/hazmat/bindings/lib3.' + \ - str(sys.version_info[1]), '*.so') -dest = os.path.join(sys.path[0], '../lib/cryptography/hazmat/bindings/') -cmd = f"cp {source} {dest}" -(status, output) = subprocess.getstatusoutput(cmd) -if status != 0: - GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % - (output, cmd)) - - -source = os.path.join(sys.path[0], '../lib/nacl/lib3.' + \ - str(sys.version_info[1]), '_sodium.abi3.so') -dest = os.path.join(sys.path[0], '../lib/nacl/') -cmd = f"cp {source} {dest}" -(status, output) = subprocess.getstatusoutput(cmd) -if status != 0: - GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % - (output, cmd)) + source = os.path.join(sys.path[0], '../lib/cryptography/hazmat/bindings/lib3.' + \ + str(sys.version_info[1]), '*.so') + dest = os.path.join(sys.path[0], '../lib/cryptography/hazmat/bindings/') + cmd = f"cp {source} {dest}" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + + + source = os.path.join(sys.path[0], '../lib/nacl/lib3.' + \ + str(sys.version_info[1]), '_sodium.abi3.so') + dest = os.path.join(sys.path[0], '../lib/nacl/') + cmd = f"cp {source} {dest}" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) from gspylib.common.Common import DefaultValue from gspylib.common.ErrorCode import ErrorCode -- Gitee From 2f650ae0991fdb2a6181915be2bccf7ed718b0a9 Mon Sep 17 00:00:00 2001 From: vimiix Date: Tue, 18 Apr 2023 18:46:31 +0800 Subject: [PATCH 16/96] =?UTF-8?q?fix(cmd=5Fexecutor):=E5=88=A0=E9=99=A4exe?= =?UTF-8?q?cCommandWithMode=E6=96=B9=E6=B3=95=E4=B8=AD=E6=97=A0=E6=95=88?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/base_utils/executor/cmd_executor.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/script/base_utils/executor/cmd_executor.py b/script/base_utils/executor/cmd_executor.py index 4f4a7526..2a22d0ad 100644 --- a/script/base_utils/executor/cmd_executor.py +++ b/script/base_utils/executor/cmd_executor.py @@ -62,8 +62,6 @@ class CmdExecutor(object): local_mode=False, mpprc_file='', host_list=None, - logger="", - timeout=0, parallelism=True): """ function: check the mode, if local mode, exec only on local node, @@ -79,11 +77,6 @@ class CmdExecutor(object): CmdExecutor.execCommandLocally(cmd) else: # Non-native mode - if logger != "": - g_ssh_tool.executeCommand(cmd, ConstantsBase.SUCCESS, host_list, mpprc_file, - 300, False, logger, timeout) - return - g_ssh_tool.executeCommand(cmd, ConstantsBase.SUCCESS, host_list, -- Gitee From 5c3a853978c96817abffc040dd349d1fdc2a19bc Mon Sep 17 00:00:00 2001 From: "jerad.k" Date: Wed, 19 Apr 2023 11:23:39 +0800 Subject: [PATCH 17/96] fix(cluster_static_config): verify crc value --- script/gspylib/common/DbClusterInfo.py | 104 +++++++++++++++++++------ script/gspylib/common/ErrorCode.py | 1 + 2 files changed, 82 insertions(+), 23 deletions(-) diff --git a/script/gspylib/common/DbClusterInfo.py b/script/gspylib/common/DbClusterInfo.py index 5959f207..6b3166f2 100644 --- a/script/gspylib/common/DbClusterInfo.py +++ b/script/gspylib/common/DbClusterInfo.py @@ -1803,6 +1803,12 @@ class dbClusterInfo(): info = fp.read(28) (crc, lenth, version, currenttime, nodeNum, localNodeId) = struct.unpack("=IIIqiI", info) + + wait_info = struct.pack("IIqiI",lenth, version, + currenttime, nodeNum, localNodeId) + if self.__verify_crc(crc, wait_info) == False: + raise Exception(ErrorCode.GAUSS_512["GAUSS_51258"] + % ("headInfo", staticConfigFile)) self.version = version self.installTime = currenttime self.localNodeId = localNodeId @@ -1873,35 +1879,42 @@ class dbClusterInfo(): (crc, nodeId, nodeName) = struct.unpack("=II64s", info) nodeName = nodeName.decode().strip('\x00') dbNode = dbNodeInfo(nodeId, nodeName) + wait_info = struct.pack("I64s",nodeId,nodeName.encode("utf-8")) info = fp.read(68) + wait_info += info (azName, azPriority) = struct.unpack("=64sI", info) dbNode.azName = azName.decode().strip('\x00') dbNode.azPriority = azPriority # get backIps - self.__unPackIps(fp, dbNode.backIps) + wait_info += self.__unPackIps(fp, dbNode.backIps) # get sshIps - self.__unPackIps(fp, dbNode.sshIps) + wait_info += self.__unPackIps(fp, dbNode.sshIps) if (not isLCCluster): # get cm_server information - self.__unPackCmsInfo(fp, dbNode) + wait_info += self.__unPackCmsInfo(fp, dbNode) # get cm_agent information - self.__unpackAgentInfo(fp, dbNode) + wait_info += self.__unpackAgentInfo(fp, dbNode) # get gtm information - self.__unpackGtmInfo(fp, dbNode) + wait_info += self.__unpackGtmInfo(fp, dbNode) info = fp.read(404) + wait_info += info # get cn information - self.__unpackCooInfo(fp, dbNode) + wait_info += self.__unpackCooInfo(fp, dbNode) # get DB information - self.__unpackDataNode(fp, dbNode) + wait_info += self.__unpackDataNode(fp, dbNode) if (not isLCCluster): # get etcd information - self.__unpackEtcdInfo(fp, dbNode) + wait_info += self.__unpackEtcdInfo(fp, dbNode) info = fp.read(8) + wait_info += info # set DB azName for OLAP for inst in dbNode.datanodes: inst.azName = dbNode.azName + if self.__verify_crc(crc,wait_info) == False: + raise Exception(ErrorCode.GAUSS_512["GAUSS_51258"] + % ("nodeInfo", staticConfigFile)) return dbNode def __unpackEtcdInfo(self, fp, dbNode): @@ -1915,14 +1928,17 @@ class dbClusterInfo(): etcdInst.hostname = dbNode.name etcdInst.instanceType = INSTANCE_TYPE_UNDEFINED info = fp.read(1100) + wait_info = info (etcdNum, etcdInst.instanceId, etcdInst.mirrorId, etcdhostname, etcdInst.datadir) = struct.unpack("=IIi64s1024s", info) etcdInst.datadir = etcdInst.datadir.decode().strip('\x00') - self.__unPackIps(fp, etcdInst.listenIps) + wait_info += self.__unPackIps(fp, etcdInst.listenIps) info = fp.read(4) + wait_info += info (etcdInst.port,) = struct.unpack("=I", info) - self.__unPackIps(fp, etcdInst.haIps) + wait_info += self.__unPackIps(fp, etcdInst.haIps) info = fp.read(4) + wait_info += info (etcdInst.haPort,) = struct.unpack("=I", info) if (etcdNum == 1): dbNode.etcdNum = 1 @@ -1931,6 +1947,7 @@ class dbClusterInfo(): else: dbNode.etcdNum = 0 dbNode.etcds = [] + return wait_info def __unPackIps(self, fp, ips): """ @@ -1939,13 +1956,17 @@ class dbClusterInfo(): output : NA """ info = fp.read(4) + wait_info = info (n,) = struct.unpack("=i", info) for i in range(int(n)): info = fp.read(128) + wait_info += info (currentIp,) = struct.unpack("=128s", info) currentIp = currentIp.decode().strip('\x00') ips.append(str(currentIp.strip())) info = fp.read(128 * (MAX_IP_NUM - n)) + wait_info += info + return wait_info def __unPackCmsInfo(self, fp, dbNode): """ @@ -1957,16 +1978,19 @@ class dbClusterInfo(): cmsInst.instanceRole = INSTANCE_ROLE_CMSERVER cmsInst.hostname = dbNode.name info = fp.read(1164) + wait_info = info (cmsInst.instanceId, cmsInst.mirrorId, dbNode.cmDataDir, cmsInst.level, self.cmsFloatIp) = struct.unpack("=II1024sI128s", info) dbNode.cmDataDir = dbNode.cmDataDir.decode().strip('\x00') self.cmsFloatIp = self.cmsFloatIp.decode().strip('\x00') cmsInst.datadir = "%s/cm_server" % dbNode.cmDataDir - self.__unPackIps(fp, cmsInst.listenIps) + wait_info += self.__unPackIps(fp, cmsInst.listenIps) info = fp.read(4) + wait_info += info (cmsInst.port,) = struct.unpack("=I", info) - self.__unPackIps(fp, cmsInst.haIps) + wait_info += self.__unPackIps(fp, cmsInst.haIps) info = fp.read(8) + wait_info += info (cmsInst.haPort, cmsInst.instanceType) = struct.unpack("=II", info) if (cmsInst.instanceType == MASTER_INSTANCE): dbNode.cmsNum = 1 @@ -1976,13 +2000,15 @@ class dbClusterInfo(): raise Exception(ErrorCode.GAUSS_512["GAUSS_51204"] % ("CMServer", cmsInst.instanceType)) info = fp.read(4 + 128 * MAX_IP_NUM + 4) - + wait_info += info if (cmsInst.instanceId): dbNode.cmservers.append(cmsInst) self.cmscount += 1 else: dbNode.cmservers = [] + return wait_info + def __unpackAgentInfo(self, fp, dbNode): """ function : Unpack the info of agent. It should be called after @@ -1996,10 +2022,12 @@ class dbClusterInfo(): cmaInst.hostname = dbNode.name cmaInst.instanceType = INSTANCE_TYPE_UNDEFINED info = fp.read(8) + wait_info = info (cmaInst.instanceId, cmaInst.mirrorId) = struct.unpack("=Ii", info) - self.__unPackIps(fp, cmaInst.listenIps) + wait_info += self.__unPackIps(fp, cmaInst.listenIps) cmaInst.datadir = "%s/cm_agent" % dbNode.cmDataDir dbNode.cmagents.append(cmaInst) + return wait_info def __unpackGtmInfo(self, fp, dbNode): """ @@ -2011,11 +2039,13 @@ class dbClusterInfo(): gtmInst.instanceRole = INSTANCE_ROLE_GTM gtmInst.hostname = dbNode.name info = fp.read(1036) + wait_info = info (gtmInst.instanceId, gtmInst.mirrorId, gtmNum, gtmInst.datadir) = struct.unpack("=III1024s", info) gtmInst.datadir = gtmInst.datadir.decode().strip('\x00') - self.__unPackIps(fp, gtmInst.listenIps) + wait_info += self.__unPackIps(fp, gtmInst.listenIps) info = fp.read(8) + wait_info += info (gtmInst.port, gtmInst.instanceType) = struct.unpack("=II", info) if (gtmInst.instanceType == MASTER_INSTANCE): dbNode.gtmNum = 1 @@ -2024,17 +2054,20 @@ class dbClusterInfo(): else: raise Exception(ErrorCode.GAUSS_512["GAUSS_51204"] % ( "GTM", gtmInst.instanceType)) - self.__unPackIps(fp, gtmInst.haIps) + wait_info += self.__unPackIps(fp, gtmInst.haIps) info = fp.read(4) + wait_info += info (gtmInst.haPort,) = struct.unpack("=I", info) info = fp.read(1024 + 4 + 128 * MAX_IP_NUM + 4) - + wait_info += info if (gtmNum == 1): dbNode.gtms.append(gtmInst) self.gtmcount += 1 else: dbNode.gtms = [] + return wait_info + def __unpackCooInfo(self, fp, dbNode): """ function : Unpack the info of coordinator @@ -2046,12 +2079,14 @@ class dbClusterInfo(): cooInst.hostname = dbNode.name cooInst.instanceType = INSTANCE_TYPE_UNDEFINED info = fp.read(2060) + wait_info = info (cooInst.instanceId, cooInst.mirrorId, cooNum, cooInst.datadir, cooInst.ssdDir) = struct.unpack("=IiI1024s1024s", info) cooInst.datadir = cooInst.datadir.decode().strip('\x00') cooInst.ssdDir = cooInst.ssdDir.decode().strip('\x00') - self.__unPackIps(fp, cooInst.listenIps) + wait_info += self.__unPackIps(fp, cooInst.listenIps) info = fp.read(8) + wait_info += info (cooInst.port, cooInst.haPort) = struct.unpack("=II", info) if (cooNum == 1): dbNode.cooNum = 1 @@ -2059,6 +2094,7 @@ class dbClusterInfo(): else: dbNode.cooNum = 0 dbNode.coordinators = [] + return wait_info def __unpackDataNode(self, fp, dbNode): """ @@ -2067,6 +2103,7 @@ class dbClusterInfo(): output : NA """ info = fp.read(4) + wait_info = info (dataNodeNums,) = struct.unpack("=I", info) dbNode.dataNum = 0 @@ -2082,12 +2119,14 @@ class dbClusterInfo(): # then rollback by fp.seek(), and exchange its(xlogdir) value # with ssddir. info = fp.read(2056) + wait_info += info (dnInst.instanceId, dnInst.mirrorId, dnInst.datadir, dnInst.xlogdir) = struct.unpack("=II1024s1024s", info) dnInst.datadir = dnInst.datadir.decode().strip('\x00') dnInst.xlogdir = dnInst.xlogdir.decode().strip('\x00') info = fp.read(1024) + wait_info += info (dnInst.ssdDir) = struct.unpack("=1024s", info) dnInst.ssdDir = dnInst.ssdDir[0].decode().strip('\x00') # if notsetXlog,ssdDir should not be null.use by upgrade. @@ -2096,8 +2135,9 @@ class dbClusterInfo(): dnInst.ssdDir = dnInst.xlogdir dnInst.xlogdir = "" - self.__unPackIps(fp, dnInst.listenIps) + wait_info += self.__unPackIps(fp, dnInst.listenIps) info = fp.read(8) + wait_info += info (dnInst.port, dnInst.instanceType) = struct.unpack("=II", info) if (dnInst.instanceType == MASTER_INSTANCE): dbNode.dataNum += 1 @@ -2107,8 +2147,9 @@ class dbClusterInfo(): else: raise Exception(ErrorCode.GAUSS_512["GAUSS_51204"] % ("DN", dnInst.instanceType)) - self.__unPackIps(fp, dnInst.haIps) + wait_info += self.__unPackIps(fp, dnInst.haIps) info = fp.read(4) + wait_info += info (dnInst.haPort,) = struct.unpack("=I", info) if ( self.clusterType == @@ -2118,34 +2159,42 @@ class dbClusterInfo(): for j in range(maxStandbyCount): peerDbInst = peerInstanceInfo() info = fp.read(1024) + wait_info += info (peerDbInst.peerDataPath,) = struct.unpack("=1024s", info) peerDbInst.peerDataPath = \ peerDbInst.peerDataPath.decode().strip('\x00') - self.__unPackIps(fp, peerDbInst.peerHAIPs) + wait_info += self.__unPackIps(fp, peerDbInst.peerHAIPs) info = fp.read(8) + wait_info += info (peerDbInst.peerHAPort, peerDbInst.peerRole) = struct.unpack("=II", info) dnInst.peerInstanceInfos.append(peerDbInst) else: peerDbInst = peerInstanceInfo() info = fp.read(1024) + wait_info += info (peerDbInst.peerDataPath,) = struct.unpack("=1024s", info) peerDbInst.peerDataPath = \ peerDbInst.peerDataPath.decode().strip('\x00') - self.__unPackIps(fp, peerDbInst.peerHAIPs) + wait_info += self.__unPackIps(fp, peerDbInst.peerHAIPs) info = fp.read(8) + wait_info += info (peerDbInst.peerHAPort, peerDbInst.peerRole) = \ struct.unpack("=II", info) info = fp.read(1024) + wait_info += info (peerDbInst.peerData2Path,) = struct.unpack("=1024s", info) peerDbInst.peerData2Path = \ peerDbInst.peerDataPath.decode().strip('\x00') - self.__unPackIps(fp, peerDbInst.peer2HAIPs) + wait_info += self.__unPackIps(fp, peerDbInst.peer2HAIPs) info = fp.read(8) + wait_info += info (peerDbInst.peer2HAPort, peerDbInst.peer2Role) = \ struct.unpack("=II", info) dnInst.peerInstanceInfos.append(peerDbInst) dbNode.datanodes.append(dnInst) + return wait_info + def initFromStaticConfigWithoutUser(self, staticConfigFile): """ @@ -2170,6 +2219,11 @@ class dbClusterInfo(): info = fp.read(28) (crc, lenth, version, currenttime, nodeNum, localNodeId) = struct.unpack("=IIIqiI", info) + + wait_info = struct.pack("IIqiI",lenth, version, currenttime, nodeNum, localNodeId) + if self.__verify_crc(crc, wait_info) == False: + raise Exception(ErrorCode.GAUSS_512["GAUSS_51258"] + % ("headInfo", staticConfigFile)) if (version <= 100): raise Exception(ErrorCode.GAUSS_516["GAUSS_51637"] % ("cluster static config version[%s]" @@ -4903,3 +4957,7 @@ class dbClusterInfo(): else: raise Exception(ErrorCode.GAUSS_502["GAUSS_50204"] % \ "float IP." + " Error: \n%s" % ret_value) + + @classmethod + def __verify_crc(self, crc, info): + return crc == binascii.crc32(info) diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index 8964539d..755ea95e 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -494,6 +494,7 @@ class ErrorCode(): 'GAUSS_51255': "[GAUSS-51255] : Failed to reencrypt the password with dsscmd", 'GAUSS_51256': "[GAUSS-51256] : Failed to get the encrypted text with dsscmd", 'GAUSS_51257': "[GAUSS-51257] : There are some errors about dsscmd. ", + 'GAUSS_51258': "[GAUSS-51258] : Failed to verify the %s crc value in file: %s", } -- Gitee From 79f82acbef8d03422f792f0f01c17e6376165ce5 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Thu, 20 Apr 2023 10:30:38 +0800 Subject: [PATCH 18/96] =?UTF-8?q?=E6=81=A2=E5=A4=8Dglobal=5Fsyscache?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E4=B8=BAon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/etc/conf/guc_list.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/script/gspylib/etc/conf/guc_list.xml b/script/gspylib/etc/conf/guc_list.xml index e8699321..98815474 100644 --- a/script/gspylib/etc/conf/guc_list.xml +++ b/script/gspylib/etc/conf/guc_list.xml @@ -16,9 +16,6 @@ - - - -- Gitee From 5ef2c58240301250f302ad407912b6486afff298 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Fri, 21 Apr 2023 09:55:08 +0800 Subject: [PATCH 19/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D3.0.3=E6=9C=80=E6=96=B0?= =?UTF-8?q?=E5=88=B0master=E5=B0=B1=E5=9C=B0=E5=8D=87=E7=BA=A7=E6=8A=A5?= =?UTF-8?q?=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/upgrade/UpgradeImpl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index 24343ad4..18d78873 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -3841,7 +3841,7 @@ class UpgradeImpl: raise Exception(ErrorCode.GAUSS_513["GAUSS_51300"] % sql + " Error: \n%s" % str(output)) sql = """START TRANSACTION;SET IsInplaceUpgrade = on; - drop index pg_proc_proname_args_nsp_index;SET LOCAL + drop index if exists pg_proc_proname_args_nsp_index;SET LOCAL inplace_upgrade_next_system_object_oids=IUO_CATALOG,false, true,0,0,0,2691;create UNIQUE INDEX pg_proc_proname_args_nsp_index ON pg_proc USING btree (proname, proargtypes, pronamespace);SET -- Gitee From 0d15ecfaf86c420205cff9e9cd7bfd945b157167 Mon Sep 17 00:00:00 2001 From: liuheng Date: Fri, 21 Apr 2023 18:21:59 +0800 Subject: [PATCH 20/96] =?UTF-8?q?=E5=9C=A8preinstall=E5=BC=80=E5=A7=8B?= =?UTF-8?q?=EF=BC=8C=E6=B7=BB=E5=8A=A0os=E5=92=8C=E5=8C=85=E6=9E=B6?= =?UTF-8?q?=E6=9E=84=E7=9A=84=E6=98=AF=E5=90=A6=E4=B8=80=E8=87=B4=E7=9A=84?= =?UTF-8?q?=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_preinstall | 11 +++--- script/gspylib/common/CheckPythonVersion.py | 42 +++++++++++++++++---- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/script/gs_preinstall b/script/gs_preinstall index 20a36f55..236b6683 100644 --- a/script/gs_preinstall +++ b/script/gs_preinstall @@ -25,9 +25,10 @@ import pwd import sys import grp import subprocess -from gspylib.common.CheckPythonVersion import checkPythonVersion, \ - check_python_compiler_option -checkPythonVersion() +from gspylib.common.CheckPythonVersion import check_python_version, \ + check_python_compiler_option, check_os_and_package_arch +check_os_and_package_arch() +check_python_version() check_python_compiler_option() from base_utils.os.file_util import FileUtil if "--unused-third-party" in sys.argv: @@ -545,8 +546,8 @@ def clearHistTimeFormat(): (status, output) = subprocess.getstatusoutput(cmd) if status != 0: GaussLog.exitWithError("Clear HISTTIMEFORMAT from /etc/profile " - "failed.\nError: %s\nThe cmd is: %s\n" % - (output,cmd)) + "failed.\nError: %s\nThe cmd is: %s\n" % + (output,cmd)) if __name__ == '__main__': """ diff --git a/script/gspylib/common/CheckPythonVersion.py b/script/gspylib/common/CheckPythonVersion.py index 639bc0ae..bc06d798 100644 --- a/script/gspylib/common/CheckPythonVersion.py +++ b/script/gspylib/common/CheckPythonVersion.py @@ -19,15 +19,17 @@ import sys import sysconfig import platform import re +import os +import subprocess -def checkPythonVersion(): - pythonVersion = sys.version_info[0:2] - distName = platform.platform() - if pythonVersion < (3, 0): +def check_python_version(): + python_version = sys.version_info[0:2] + dist_name = platform.platform() + if python_version < (3, 0): raise Exception("[GAUSS-52200] : version of python" " is not correct: %s." % - distName + " should use Python 3.*") + dist_name + " should use Python 3.*") return True def check_python_compiler_option(): @@ -37,11 +39,37 @@ def check_python_compiler_option(): carry the -enable-shared parameters") return True +def check_os_and_package_arch(): + """ + check os and package arch + """ + clib_path = os.path.realpath( + os.path.join(os.path.realpath(__file__), "../../clib")) + package_cmd = "cd " + clib_path + "&& file libcrypto.so.1.1 2>/dev/null" + (status, output) = subprocess.getstatusoutput(package_cmd) + if status != 0: + raise Exception("%s command faile." % (package_cmd)) + package_arch = "" + if ("x86-64" in output): + package_arch = "x86_64" + if ("aarch64" in output): + package_arch = "aarch64" + + os_cmd = "uname -p" + (status, output) = subprocess.getstatusoutput(os_cmd) + if status != 0: + raise Exception("%s command failed." % (os_cmd)) + os_arch = output + + if (package_arch == os_arch): + return + raise Exception("System and software package architecture mismatch.\n" + + "Error: os architecture is %s, package architecture is %s" % (os_arch, package_arch)) if __name__ == '__main__': try: - check_python = checkPythonVersion() - if check_python: + CHECK_PYTHON = check_python_version() + if CHECK_PYTHON: check_python_compiler_option() except Exception as e: raise Exception(e) -- Gitee From 93b44ab117d8564de349126c45c2ce0c4e179bd6 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Tue, 25 Apr 2023 20:28:11 +0800 Subject: [PATCH 21/96] =?UTF-8?q?om=E9=80=82=E9=85=8D=E5=85=B1=E4=BA=AB?= =?UTF-8?q?=E5=AD=98=E5=82=A8dorado=E6=A8=A1=E5=BC=8F=EF=BC=88=E4=B8=BB?= =?UTF-8?q?=E9=9B=86=E7=BE=A4=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_install | 3 +++ script/gspylib/common/LocalBaseOM.py | 5 ++++- script/gspylib/common/ParallelBaseOM.py | 1 + script/gspylib/common/ParameterParsecheck.py | 5 ++++- script/gspylib/component/BaseComponent.py | 1 + script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py | 6 ++++++ script/impl/install/OLAP/InstallImplOLAP.py | 2 +- script/local/InitInstance.py | 14 ++++++++++---- 8 files changed, 30 insertions(+), 7 deletions(-) diff --git a/script/gs_install b/script/gs_install index 54696c1f..17b234f2 100644 --- a/script/gs_install +++ b/script/gs_install @@ -216,6 +216,9 @@ General options: # parameter --time-out if (ParaDict.__contains__("time_out")): self.time_out = ParaDict.get("time_out") + # parameter --dorado-info + if (ParaDict.__contains__("dorado-config")): + self.dorado_config = ParaDict.get("dorado-config") def checkUser(self): """ diff --git a/script/gspylib/common/LocalBaseOM.py b/script/gspylib/common/LocalBaseOM.py index 578d25b2..8073f861 100644 --- a/script/gspylib/common/LocalBaseOM.py +++ b/script/gspylib/common/LocalBaseOM.py @@ -49,7 +49,8 @@ class LocalBaseOM(object): gtmInitParas=None, paxos_mode=False, dss_mode=False, - dss_config=""): + dss_config="", + dorado_config=""): ''' Constructor ''' @@ -79,6 +80,7 @@ class LocalBaseOM(object): self.paxos_mode = paxos_mode self.dss_mode = dss_mode self.dss_config = dss_config + self.dorado_config = dorado_config def initComponent(self, paxos_mode=False): """ @@ -153,6 +155,7 @@ class LocalBaseOM(object): component.paxos_mode = paxos_mode self.initComponentAttributes(component) component.initParas = self.initParas + component.dorado_config = self.dorado_config self.dnCons.append(component) def readConfigInfo(self): diff --git a/script/gspylib/common/ParallelBaseOM.py b/script/gspylib/common/ParallelBaseOM.py index e80b7915..3eac95ff 100644 --- a/script/gspylib/common/ParallelBaseOM.py +++ b/script/gspylib/common/ParallelBaseOM.py @@ -80,6 +80,7 @@ class ParallelBaseOM(object): self.cnCons = [] self.dnCons = [] self.dss_cons = [] + self.dorado_config = "" # localMode is same as isSingle in all OM script, expect for # gs_preinstall. # in gs_preinstall, localMode means local mode for master-standby diff --git a/script/gspylib/common/ParameterParsecheck.py b/script/gspylib/common/ParameterParsecheck.py index 2fa68bbc..6a29fc7d 100644 --- a/script/gspylib/common/ParameterParsecheck.py +++ b/script/gspylib/common/ParameterParsecheck.py @@ -68,7 +68,7 @@ gs_preinstall = ["-?", "--help", "-V", "--version", "-U:", "-G:", "-L", "-l:", "--non-interactive", "--delete-root-trust", "--unused-third-party"] gs_install = ["-?", "--help", "-V", "--version", "-X:", "-l:", "--gsinit-parameter=", "--dn-guc=", "--cms-guc=", - "--time-out=", "--alarm-component="] + "--time-out=", "--dorado-config=", "--alarm-component="] gs_uninstall = ["-?", "--help", "-V", "--version", "-l:", "-L", "--delete-data"] gs_postuninstall = ["-?", "--help", "-V", "--version", "--delete-user", @@ -302,6 +302,7 @@ class Parameter(): "--alarm-type": "warningType", "--alarm-server-addr": "warningserverip", "--time-out": "time_out", "": "", + "--dorado-config": "dorado-config", "--alarm-component": "alarm_component", "--SSD-fault-time": "SSDFaultTime", "--begin-time": "begintime", @@ -473,6 +474,8 @@ class Parameter(): PARAMETER_VALUEDICT['new_values'] = value.strip().split(",") elif key == "--upgrade-package": PARAMETER_VALUEDICT["upgrade-package"] = value.strip() + elif key == "--dorado-config": + PARAMETER_VALUEDICT["dorado-config"] = value.strip() # Only check / symbol for gs_lcct. if key in ("--name", "--nodegroup-name"): self.checkLcGroupName(key, value) diff --git a/script/gspylib/component/BaseComponent.py b/script/gspylib/component/BaseComponent.py index 9a631f42..b2fe6f7e 100644 --- a/script/gspylib/component/BaseComponent.py +++ b/script/gspylib/component/BaseComponent.py @@ -55,6 +55,7 @@ class BaseComponent(object): self.paxos_mode = '' self.dss_mode = '' self.dss_config = '' + self.dorado_config = '' def install(self): pass diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index 846e6ab5..d3897eca 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -157,6 +157,10 @@ class DN_OLAP(Kernel): " --socketpath=\"{}\"".format( "+{},+{}".format(vgname, pri_vgname), cfg_context, inst_id, "UDS:{}/.dss_unix_d_socket".format(dss_home)) + if (self.dorado_config != "" and self.instInfo.instanceType == DefaultValue.MASTER_INSTANCE): + cmd += " -g %s" % self.dorado_config + tmpDict3 = {} + tmpDict3["xlog_lock_file_path"] = "'%s/redolog.lock'" % self.instInfo.datadir self.logger.debug("Command for initializing database " "node instance: %s" % cmd) status, output = CmdUtil.retryGetstatusoutput( @@ -167,6 +171,8 @@ class DN_OLAP(Kernel): # set ssl to DB nodes. dnGucParas = self.getDnGUCDict() self.setGucConfig(dnGucParas) + if (self.dorado_config != "" and self.instInfo.instanceType == DefaultValue.MASTER_INSTANCE): + self.setGucConfig(tmpDict3) self.copyAndModCertFiles() def getInstanceNodeName(self): diff --git a/script/impl/install/OLAP/InstallImplOLAP.py b/script/impl/install/OLAP/InstallImplOLAP.py index 5123133c..58c591fc 100644 --- a/script/impl/install/OLAP/InstallImplOLAP.py +++ b/script/impl/install/OLAP/InstallImplOLAP.py @@ -290,7 +290,7 @@ class InstallImplOLAP(InstallImpl): elif self.context.clusterInfo.enable_dss == 'on': dss_config = DssConfig.get_value_b64_handler( 'dss_nodes_list', self.context.clusterInfo.dss_config) - cmd += f" --dss_mode --dss_config={dss_config}" + cmd += f" --dss_mode --dss_config={dss_config} --dorado_config={self.context.dorado_config}" self.context.logger.debug( "Command for initializing instances: %s" % cmd) diff --git a/script/local/InitInstance.py b/script/local/InitInstance.py index 62f6bd20..d9aa6459 100644 --- a/script/local/InitInstance.py +++ b/script/local/InitInstance.py @@ -62,6 +62,7 @@ class CmdOptions(): self.paxos_mode = False self.dss_mode = False self.dss_config = "" + self.dorado_config = "" def usage(): @@ -79,7 +80,7 @@ def parseCommandLine(): try: opts, args = getopt.getopt(sys.argv[1:], "U:P:G:l:?", [ "help", "dws_mode", "vc_mode", "paxos_mode", "dss_mode", - "dss_config=" + "dss_config=", "dorado_config=" ]) except Exception as e: usage() @@ -112,6 +113,8 @@ def parseCommandLine(): g_opts.dss_mode = True elif key == "--dss_config": g_opts.dss_config = value.strip() + elif key == "--dorado_config": + g_opts.dorado_config = value.strip() Parameter.checkParaVaild(key, value) @@ -178,7 +181,8 @@ class initDbNode(LocalBaseOM): dbInitParams=None, paxos_mode=False, dss_mode=False, - dss_config=""): + dss_config="", + dorado_config = ""): """ function: init instance input : logFile, user, clusterConf, dbInitParams @@ -194,7 +198,8 @@ class initDbNode(LocalBaseOM): dbInitParams, paxos_mode, dss_mode=dss_mode, - dss_config=dss_config) + dss_config=dss_config, + dorado_config=dorado_config) if self.clusterConfig == "": # Read config from static config file self.readConfigInfo() @@ -269,7 +274,8 @@ if __name__ == '__main__': g_opts.dbInitParams, g_opts.paxos_mode, dss_mode=g_opts.dss_mode, - dss_config=g_opts.dss_config) + dss_config=g_opts.dss_config, + dorado_config=g_opts.dorado_config) dbInit.initNodeInst(g_opts.vc_mode) except Exception as e: -- Gitee From 62938471d108dfa1946ef4cb7109a82db9ce9a29 Mon Sep 17 00:00:00 2001 From: jianghongbo4 Date: Fri, 28 Apr 2023 11:27:47 +0800 Subject: [PATCH 22/96] fix I6ZCZM --- script/base_utils/os/net_util.py | 2 ++ script/gspylib/common/Common.py | 2 ++ script/gspylib/threads/SshTool.py | 1 + 3 files changed, 5 insertions(+) diff --git a/script/base_utils/os/net_util.py b/script/base_utils/os/net_util.py index fca29cbc..e28dd774 100644 --- a/script/base_utils/os/net_util.py +++ b/script/base_utils/os/net_util.py @@ -40,6 +40,8 @@ sys.path.insert(0, localDirPath + "/../../../lib") try: import psutil except ImportError as e: + if not bool(os.listdir(localDirPath + "/../../../lib")): + raise # mv psutil mode .so file by python version pythonVer = str(sys.version_info[0]) + '.' + str(sys.version_info[1]) psutilLinux = os.path.join(localDirPath, diff --git a/script/gspylib/common/Common.py b/script/gspylib/common/Common.py index 7e07779d..1ebcf333 100644 --- a/script/gspylib/common/Common.py +++ b/script/gspylib/common/Common.py @@ -63,6 +63,8 @@ sys.path.insert(0, localDirPath + "/../../../lib") try: import psutil except ImportError as e: + if not bool(os.listdir(localDirPath + "/../../../lib")): + raise # mv psutil mode .so file by python version pythonVer = str(sys.version_info[0]) + '.' + str(sys.version_info[1]) psutilLinux = os.path.join(localDirPath, diff --git a/script/gspylib/threads/SshTool.py b/script/gspylib/threads/SshTool.py index f22c01b5..584fa9dc 100644 --- a/script/gspylib/threads/SshTool.py +++ b/script/gspylib/threads/SshTool.py @@ -41,6 +41,7 @@ from gspylib.common.Constants import Constants try: import paramiko except ImportError as ex: + print(ex) try: local_path = os.path.dirname(os.path.realpath(__file__)) clib_path = os.path.realpath(os.path.join(local_path, "../../gspylib/clib/")) -- Gitee From ff321fc55d9ba46199f74729cf9075307bbf4658 Mon Sep 17 00:00:00 2001 From: totaj Date: Thu, 4 May 2023 15:31:32 +0800 Subject: [PATCH 23/96] Fix gs_om failed when upper case attribute is on. --- script/gspylib/common/DbClusterInfo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/gspylib/common/DbClusterInfo.py b/script/gspylib/common/DbClusterInfo.py index 5959f207..31e89c7c 100644 --- a/script/gspylib/common/DbClusterInfo.py +++ b/script/gspylib/common/DbClusterInfo.py @@ -1357,14 +1357,14 @@ class dbClusterInfo(): roleStatus = "Unknown" dbState = "Unknown" else: - res = re.findall(r'local_role\s*:\s*(\w+)', output) + res = re.findall(r'local_role\s*:\s*(\w+)', output, re.IGNORECASE) roleStatus = res[0] - res = re.findall(r'db_state\s*:\s*(\w+)', output) + res = re.findall(r'db_state\s*:\s*(\w+)', output, re.IGNORECASE) dbState = res[0] if (dbState == "Need"): detailInformation = re.findall( - r'detail_information\s*:\s*(\w+)', output) + r'detail_information\s*:\s*(\w+)', output, re.IGNORECASE) dbState = "Need repair(%s)" % detailInformation[0] roleStatusArray.append(roleStatus) dbStateArray.append(dbState) -- Gitee From 1694ac9ec7f2608f7318abf3881965221f846539 Mon Sep 17 00:00:00 2001 From: liuheng Date: Wed, 10 May 2023 15:32:50 +0800 Subject: [PATCH 24/96] =?UTF-8?q?om=E5=AE=89=E8=A3=85=EF=BC=8C=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0PGDATA=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/local/PreInstallUtility.py | 32 +++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 77087083..ad924fa8 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -1541,16 +1541,7 @@ Common options: self.logger.debug( "Successfully set %s user profile." % VersionInfo.PRODUCT_NAME) - def setToolEnv(self): - """ - function: set environment variables - input : NA - output: NA - """ - self.logger.debug("Setting tool ENV.") - - userProfile = self.getUserProfile() - + def clearToolEnv(self, userProfile): # clean ENV in os profile self.logger.debug("OS profile exists. Deleting crash old tool ENV.") # clean MPPRC FILE PATH @@ -1568,7 +1559,8 @@ Common options: FileUtil.deleteLine(userProfile, "^\\s*export\\s*UNPACKPATH=.*$") self.logger.debug( "Deleting crash GPHOME in user environment variables.") - + # clean PGDATA + FileUtil.deleteLine(userProfile, "^\\s*export\\s*PGDATA*") # clean LD_LIBRARY_PATH FileUtil.deleteLine(userProfile, "^\\s*export\\s*LD_LIBRARY_PATH=\\$GPHOME\\/script" @@ -1600,13 +1592,23 @@ Common options: "^\\s*export\\s*PATH=\\/root\\/gauss_om\\/%s\\" "/script:\\$PATH$" % self.user) self.logger.debug("Deleting crash PATH in user environment variables.") - # clean PYTHONPATH FileUtil.deleteLine(userProfile, "^\\s*export\\s*PYTHONPATH=\\$GPHOME\\/lib") self.logger.debug( "Deleting crash PYTHONPATH in user environment variables.") + def setToolEnv(self): + """ + function: set environment variables + input : NA + output: NA + """ + self.logger.debug("Setting tool ENV.") + userProfile = self.getUserProfile() + + self.clearToolEnv(userProfile) + # set ENV in os profile self.logger.debug( "Successfully deleted crash old tool ENV. Setting new tool ENV.") @@ -1624,6 +1626,12 @@ Common options: ["export GPHOME=%s" % self.clusterToolPath]) package_dir = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../../")) FileUtil.writeFile(userProfile, ["export UNPACKPATH=%s" % package_dir]) + # set PGDATA + hostName = NetUtil.GetHostIpOrName() + node_info = self.clusterInfo.getDbNodeByName(hostName) + datadir = node_info.datanodes[0].datadir + FileUtil.writeFile(userProfile, + ["export PGDATA=%s" % datadir]) # set PATH if userProfile is ClusterConstants.ETC_PROFILE: FileUtil.writeFile(userProfile, [ -- Gitee From 26f3fb69df01a5fb852ec0a2ef4a6603aefb9955 Mon Sep 17 00:00:00 2001 From: liuheng Date: Wed, 10 May 2023 19:14:19 +0800 Subject: [PATCH 25/96] =?UTF-8?q?=E4=BF=AE=E6=94=B9dss=20port=E5=92=8Cdms?= =?UTF-8?q?=20port=E7=BA=A6=E6=9D=9F=E9=83=A8=E5=88=86=E7=9A=84=E6=B3=A8?= =?UTF-8?q?=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/component/DSS/dss_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/gspylib/component/DSS/dss_checker.py b/script/gspylib/component/DSS/dss_checker.py index 812ad68f..4f327f55 100644 --- a/script/gspylib/component/DSS/dss_checker.py +++ b/script/gspylib/component/DSS/dss_checker.py @@ -77,7 +77,7 @@ class DssConfig(): ' The number of dns is {} and the number of dss volumes is {}'. format(len(dss_ips), len(infos[::2]))) for dp in dss_ports: - # The dms port is db port plus 10, and the dss port is db port plus 20. + # The dms port is db port plus 20, and the dss port is db port plus 10. SecurityChecker.check_port_valid( 'dataPortBase', int(dp), -- Gitee From 5be9585534fd85612db52a4a18d40ce793bc95c6 Mon Sep 17 00:00:00 2001 From: vimiix Date: Thu, 11 May 2023 16:28:34 +0800 Subject: [PATCH 26/96] fix(DefaultValue):calculate dynamic value only macro in guc value --- script/gspylib/common/Common.py | 88 ++++++++++++++++----------------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/script/gspylib/common/Common.py b/script/gspylib/common/Common.py index 1ebcf333..49e3386f 100644 --- a/script/gspylib/common/Common.py +++ b/script/gspylib/common/Common.py @@ -2153,72 +2153,68 @@ class DefaultValue(): " There are illegal characters %s " "in the content." % rac) + @staticmethod + def calculate_dynamic_size(raw_str, dynamic_para_list): + if (int(dynamic_para_list[0]) < 256): + ratioNum = 1 + elif (int(dynamic_para_list[0]) < 512): + ratioNum = 2 + else: + ratioNum = 3 + value = raw_str.replace("PHYSIC_MEMORY", dynamic_para_list[1]) \ + .replace("MAX_MASTER_DATANUM_IN_ONENODE", dynamic_para_list[2]) \ + .replace("N", str(ratioNum)) + try: + value = eval(value) + except: + raise Exception(ErrorCode.GAUSS_516["GAUSS_51632"] % + "calculate: %s " % raw_str) + return value # float, 单位 GB + @staticmethod def initGuc(gucDict, dynamicParaList, gucXml=False): """ + dynamicParaList 格式： + [主dn的个数, 集群中最小物理内存(单位GB), dn的总数] """ - for guc in gucDict: + gucDict = {k: v for k, v in gucDict.items() if v} + for guc, value in gucDict.items(): if (guc == "comm_max_datanode" and not gucXml): - if (int(dynamicParaList[0]) < 256): + primary_dn_num = int(dynamicParaList[0]) + if primary_dn_num < 256: gucDict[guc] = 256 - elif (int(dynamicParaList[0]) < 512): + elif primary_dn_num < 512: gucDict[guc] = 512 - elif (int(dynamicParaList[0]) < 1024): + elif primary_dn_num < 1024: gucDict[guc] = 1024 - elif (int(dynamicParaList[0]) < 2048): + elif primary_dn_num < 2048: gucDict[guc] = 2048 else: gucDict[guc] = 4096 continue - elif (guc == "max_process_memory"): - if (gucDict[guc] == "80GB"): + elif guc == "max_process_memory": + if "PHYSIC_MEMORY" not in value \ + and "MAX_MASTER_DATANUM_IN_ONENODE" not in value: continue - if (int(dynamicParaList[0]) < 256): - ratioNum = 1 - elif (int(dynamicParaList[0]) < 512): - ratioNum = 2 - else: - ratioNum = 3 - gucDict[guc] = gucDict[guc].replace( - "PHYSIC_MEMORY", dynamicParaList[1]) - gucDict[guc] = gucDict[guc].replace( - "MAX_MASTER_DATANUM_IN_ONENODE", dynamicParaList[2]) - gucDict[guc] = gucDict[guc].replace("N", str(ratioNum)) - try: - gucDict[guc] = eval(gucDict[guc]) - except Exception as e: - raise Exception(ErrorCode.GAUSS_516["GAUSS_51632"] % - "calculate: %s" % gucDict[guc]) - gucDict[guc] = int(gucDict[guc]) - if (gucDict[guc] >= 2 and gucDict[guc] <= 2047): - gucDict[guc] = str(gucDict[guc]) + "GB" - elif (gucDict[guc] < 2): + # 下面是依据物理内存计算 + gigabytes = DefaultValue.calculate_dynamic_size(value, dynamicParaList) + if (gigabytes >= 2 and gigabytes <= 2047): + gucDict[guc] = str(int(gigabytes)) + "GB" + elif (gigabytes < 2): gucDict[guc] = "2GB" else: gucDict[guc] = "2047GB" continue elif guc == "shared_buffers": - if (int(dynamicParaList[0]) < 256): - ratioNum = 1 - elif (int(dynamicParaList[0]) < 512): - ratioNum = 2 - else: - ratioNum = 3 - gucDict[guc] = gucDict[guc].replace( - "PHYSIC_MEMORY", dynamicParaList[1]) - gucDict[guc] = gucDict[guc].replace( - "MAX_MASTER_DATANUM_IN_ONENODE", dynamicParaList[2]) - gucDict[guc] = gucDict[guc].replace("N", str(ratioNum)) - try: - gucDict[guc] = eval(gucDict[guc]) - except Exception as e: - raise Exception(ErrorCode.GAUSS_516["GAUSS_51632"] % - "calculate: %s" % gucDict[guc]) - gucDict[guc] = int(gucDict[guc] * 1024) - if gucDict[guc] >= 1024: + if "PHYSIC_MEMORY" not in value \ + and "MAX_MASTER_DATANUM_IN_ONENODE" not in value: + continue + gigabytes = DefaultValue.calculate_dynamic_size(value, dynamicParaList) + megabytes = int(gigabytes * 1024) + if megabytes >= 1024: gucDict[guc] = "1GB" else: - gucDict[guc] = str(gucDict[guc]) + "MB" + gucDict[guc] = str(megabytes) + "MB" return gucDict @staticmethod -- Gitee From 9d9980dd6ad77a1a9784f1997f7bfe6fbe4e0fa0 Mon Sep 17 00:00:00 2001 From: "jerad.k" Date: Thu, 11 May 2023 17:24:26 +0800 Subject: [PATCH 27/96] fix(I72ALO): remain the LD_LIBRARY_PATH(gspylib/clib) in user when gs_preinstall --- script/gspylib/common/Common.py | 27 ++++++++++++++------------- script/local/PreInstallUtility.py | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/script/gspylib/common/Common.py b/script/gspylib/common/Common.py index 7e07779d..158d1b13 100644 --- a/script/gspylib/common/Common.py +++ b/script/gspylib/common/Common.py @@ -1044,7 +1044,7 @@ class DefaultValue(): @staticmethod def cleanUserEnvVariable(userProfile, cleanGAUSS_WARNING_TYPE=False, - cleanGS_CLUSTER_NAME=True): + cleanGS_CLUSTER_NAME=True, cleanLD_LIBRARY=True): """ function : Clean the user environment variable input : String,boolean @@ -1056,18 +1056,19 @@ class DefaultValue(): # clean version FileUtil.deleteLine(userProfile, "^\\s*export\\" "s*GAUSS_VERSION=.*$") - # clean lib - FileUtil.deleteLine(userProfile, - "^\\s*export\\s*LD_LIBRARY_PATH=\\" - "$GAUSSHOME\\/lib:\\$LD_LIBRARY_PATH$") - FileUtil.deleteLine(userProfile, - "^\\s*export\\s*LD_LIBRARY_PATH=\\" - "$GAUSSHOME\\/lib\\/libsimsearch:\\" - "$LD_LIBRARY_PATH$") - FileUtil.deleteLine(userProfile, - "^\\s*export\\s*LD_LIBRARY_PATH=\\$GPHOME\\" - "/script\\/gspylib\\/clib:\\" - "$LD_LIBRARY_PATH$") + if (cleanLD_LIBRARY): + # clean lib + FileUtil.deleteLine(userProfile, + "^\\s*export\\s*LD_LIBRARY_PATH=\\" + "$GAUSSHOME\\/lib:\\$LD_LIBRARY_PATH$") + FileUtil.deleteLine(userProfile, + "^\\s*export\\s*LD_LIBRARY_PATH=\\" + "$GAUSSHOME\\/lib\\/libsimsearch:\\" + "$LD_LIBRARY_PATH$") + FileUtil.deleteLine(userProfile, + "^\\s*export\\s*LD_LIBRARY_PATH=\\$GPHOME\\" + "/script\\/gspylib\\/clib:\\" + "$LD_LIBRARY_PATH$") # clean bin FileUtil.deleteLine(userProfile, "^\\s*export\\s*PATH=\\$GAUSSHOME\\" diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 77087083..32918451 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -1328,7 +1328,7 @@ Common options: # clean user's environmental variable self.logger.debug("Deleting user's environmental variable.") DefaultValue.cleanUserEnvVariable(userProfile, - cleanGS_CLUSTER_NAME=False) + cleanGS_CLUSTER_NAME=False, cleanLD_LIBRARY=False) self.logger.debug("Successfully delete user's environmental variable.") if self.mpprcFile: # import environment variable separation scene to bashrc -- Gitee From cf101d4cd92afc5fee4077b95c793ef1989546a7 Mon Sep 17 00:00:00 2001 From: liuheng Date: Fri, 12 May 2023 11:42:50 +0800 Subject: [PATCH 28/96] =?UTF-8?q?azPriority=E4=BC=98=E5=85=88=E7=BA=A7?= =?UTF-8?q?=E7=9B=B8=E5=90=8C=E5=A2=9E=E5=8A=A0=E6=8A=A5=E9=94=99=E6=8F=90?= =?UTF-8?q?=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/common/DbClusterInfo.py | 4 ++++ script/gspylib/common/ErrorCode.py | 5 +++-- script/impl/preinstall/PreinstallImpl.py | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/script/gspylib/common/DbClusterInfo.py b/script/gspylib/common/DbClusterInfo.py index 31e89c7c..91564fca 100644 --- a/script/gspylib/common/DbClusterInfo.py +++ b/script/gspylib/common/DbClusterInfo.py @@ -598,6 +598,7 @@ class instanceInfo(): self.controlPort = 0 # az name self.azName = "" + self.azPriority = 0 self.clusterName = "" # peer port etcd self.peerPort = 0 @@ -1901,6 +1902,7 @@ class dbClusterInfo(): # set DB azName for OLAP for inst in dbNode.datanodes: inst.azName = dbNode.azName + inst.azPriority = dbNode.azPriority return dbNode @@ -2616,6 +2618,7 @@ class dbClusterInfo(): db_inst.syncNum = -1 db_inst.syncNumFirst = "" db_inst.azName = db_node.azName + db_inst.azPriority = db_node.azPriority self.dbNodes[0].datanodes.append(db_inst) self.dbNodes[0].appendInstance(1, MIRROR_ID_AGENT, INSTANCE_ROLE_CMAGENT, INSTANCE_TYPE_UNDEFINED, [], None, "") @@ -3118,6 +3121,7 @@ class dbClusterInfo(): for node in self.dbNodes: for inst in node.datanodes: inst.azName = node.azName + inst.azPriority = node.azPriority self.__setNodePortForSinglePrimaryMultiStandby() def __getPeerInstance(self, dbInst): diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index 8964539d..036e26d0 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -189,7 +189,7 @@ class ErrorCode(): " outputMap: %s", 'GAUSS_50238': "[GAUSS-50238] : Check integrality of bin ", "file %s failed." - 'GAUSS_50239': "[GAUSS-50236] : %s should be set in scene config " + 'GAUSS_50239': "[GAUSS-50239] : %s should be set in scene config " "file." } @@ -630,7 +630,8 @@ class ErrorCode(): 'GAUSS_51655': "[GAUSS-51655] : There is %s on the cluster when operating on a cluster" "the %s parameter is not needed.", 'GAUSS_51656': "[GAUSS-51656] : Waiting for udev trigger to end timeout", - 'GAUSS_51657': "[GAUSS-51657] : Waiting for start %s to end timeout" + 'GAUSS_51657': "[GAUSS-51657] : Waiting for start %s to end timeout", + 'GAUSS_51658': "[GAUSS-51658] : The azName is different, and the value of azPriority must be different. " } ########################################################################### diff --git a/script/impl/preinstall/PreinstallImpl.py b/script/impl/preinstall/PreinstallImpl.py index fb868d88..3b25eb12 100644 --- a/script/impl/preinstall/PreinstallImpl.py +++ b/script/impl/preinstall/PreinstallImpl.py @@ -1462,6 +1462,26 @@ class PreinstallImpl: raise Exception(ErrorCode.GAUSS_502["GAUSS_50232"] % ( dataInst.datadir, appPath)) + def checkAzPriorityValue(self): + """ + function : Check azName and azPriority value, The azName is different, and the value of azPriority must be different. + input : None + output : None + """ + priority_map = {} + for db_node in self.context.clusterInfo.dbNodes: + for data_inst in db_node.datanodes: + az_name = data_inst.azName + priority = data_inst.azPriority + if (az_name not in priority_map): + priority_map[az_name] = priority + + result = set() + for value in priority_map.values(): + result.add(value) + if len(result) < len(priority_map.values()): + raise Exception(ErrorCode.GAUSS_516["GAUSS_51658"]) + def checkOSSoftware(self): """ function: setting the dynamic link library @@ -1537,6 +1557,8 @@ class PreinstallImpl: # Check whether the instance directory # conflicts with the application directory. self.checkInstanceDir() + # check azPriotity + self.checkAzPriorityValue() # install tools phase1 self.installToolsPhase1() # exchange user key for root user -- Gitee From 9794aaf3b0796a2f45c4b3873071b7bcdf01804d Mon Sep 17 00:00:00 2001 From: liuheng Date: Mon, 15 May 2023 15:57:05 +0800 Subject: [PATCH 29/96] =?UTF-8?q?=E4=BF=AE=E6=94=B9om=E5=AE=89=E8=A3=85ser?= =?UTF-8?q?ver=5Fencoding=E9=BB=98=E8=AE=A4=E6=98=AFutf8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index 846e6ab5..a5ab37ce 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -130,13 +130,13 @@ class DN_OLAP(Kernel): # If xlogdir is set in xmlfile, an independent xlog # path will be created. if (self.instInfo.xlogdir != ''): - cmd = "%s/gs_initdb --locale=C -D %s -X %s " \ + cmd = "%s/gs_initdb -D %s -X %s " \ "--nodename=%s %s -C %s" % ( self.binPath, self.instInfo.datadir, self.instInfo.xlogdir, nodename, " ".join(self.initParas), self.binPath) else: - cmd = "%s/gs_initdb --locale=C -D %s --nodename=%s %s -C %s" \ + cmd = "%s/gs_initdb -D %s --nodename=%s %s -C %s" \ % \ (self.binPath, self.instInfo.datadir, nodename, " ".join(self.initParas), self.binPath) -- Gitee From f46a854eabee67fe0b8190760e15428f664e7a3b Mon Sep 17 00:00:00 2001 From: liuheng Date: Fri, 19 May 2023 14:31:25 +0800 Subject: [PATCH 30/96] =?UTF-8?q?=E5=9B=9E=E9=80=80initdb=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96=E7=BC=96=E7=A0=81=E4=B8=BASQL=5FASCII?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index a5ab37ce..846e6ab5 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -130,13 +130,13 @@ class DN_OLAP(Kernel): # If xlogdir is set in xmlfile, an independent xlog # path will be created. if (self.instInfo.xlogdir != ''): - cmd = "%s/gs_initdb -D %s -X %s " \ + cmd = "%s/gs_initdb --locale=C -D %s -X %s " \ "--nodename=%s %s -C %s" % ( self.binPath, self.instInfo.datadir, self.instInfo.xlogdir, nodename, " ".join(self.initParas), self.binPath) else: - cmd = "%s/gs_initdb -D %s --nodename=%s %s -C %s" \ + cmd = "%s/gs_initdb --locale=C -D %s --nodename=%s %s -C %s" \ % \ (self.binPath, self.instInfo.datadir, nodename, " ".join(self.initParas), self.binPath) -- Gitee From 11f206a0bf49bc0dae4555d2f0b7e600120e196e Mon Sep 17 00:00:00 2001 From: jianghongbo4 Date: Tue, 16 May 2023 20:00:33 +0800 Subject: [PATCH 31/96] fix #I721ZB --- script/gs_preinstall | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/script/gs_preinstall b/script/gs_preinstall index 990e7096..aad85982 100644 --- a/script/gs_preinstall +++ b/script/gs_preinstall @@ -27,15 +27,18 @@ import grp import subprocess from gspylib.common.CheckPythonVersion import check_python_version, \ check_python_compiler_option, check_os_and_package_arch -check_os_and_package_arch() -check_python_version() -check_python_compiler_option() + +package_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = os.path.join(package_path, "..", "lib") +clib_files = os.path.join(package_path, "gspylib/clib/*.so*") +if os.listdir(lib_path): + check_os_and_package_arch() + check_python_version() + check_python_compiler_option() + from base_utils.os.file_util import FileUtil from gspylib.common.GaussLog import GaussLog if "--unused-third-party" in sys.argv: - package_path = os.path.dirname(os.path.realpath(__file__)) - lib_path = os.path.join(package_path, "..", "lib") - clib_files = os.path.join(package_path, "gspylib/clib/*.so*") FileUtil.cleanDirectoryContent(lib_path) FileUtil.removeFile(clib_files) @@ -45,6 +48,9 @@ if "--unused-third-party" in sys.argv: import cryptography import paramiko else: + check_os_and_package_arch() + check_python_version() + check_python_compiler_option() source = os.path.join(sys.path[0], '../lib/bcrypt/lib3.' + \ str(sys.version_info[1]), '_bcrypt.abi3.so') dest = os.path.join(sys.path[0], '../lib/bcrypt/') -- Gitee From df4458ecb83831849ad69c5a0499e985de4137be Mon Sep 17 00:00:00 2001 From: gentle_hu Date: Mon, 29 May 2023 09:30:54 +0800 Subject: [PATCH 32/96] support upgrade checker --- script/gs_upgradechk | 38 + script/upgrade_checker/README.md | 105 + script/upgrade_checker/__init__.py | 0 script/upgrade_checker/dev/__init__.py | 0 script/upgrade_checker/dev/developer_guide.md | 63 + script/upgrade_checker/dev/og_controller.py | 126 + .../upgrade_checker/dev/standard_vmap_gen.py | 126 + script/upgrade_checker/log.py | 189 ++ script/upgrade_checker/opengauss.py | 176 ++ script/upgrade_checker/project.py | 212 ++ script/upgrade_checker/report.py | 268 ++ script/upgrade_checker/rules/__init__.py | 0 script/upgrade_checker/rules/category.py | 75 + script/upgrade_checker/rules/meta.py | 2153 +++++++++++++++++ script/upgrade_checker/rules/rule.py | 283 +++ script/upgrade_checker/rules/rule_maker.py | 233 ++ script/upgrade_checker/rules/vmap.py | 184 ++ script/upgrade_checker/style/__init__.py | 0 script/upgrade_checker/style/markdown.py | 200 ++ script/upgrade_checker/utils/__init__.py | 0 script/upgrade_checker/utils/command.py | 67 + script/upgrade_checker/utils/exception.py | 50 + script/upgrade_checker/utils/param.py | 231 ++ script/upgrade_checker/utils/singleton.py | 33 + script/upgrade_checker/utils/version.py | 36 + script/upgrade_checker/verifier.py | 179 ++ 26 files changed, 5027 insertions(+) create mode 100644 script/gs_upgradechk create mode 100644 script/upgrade_checker/README.md create mode 100644 script/upgrade_checker/__init__.py create mode 100644 script/upgrade_checker/dev/__init__.py create mode 100644 script/upgrade_checker/dev/developer_guide.md create mode 100644 script/upgrade_checker/dev/og_controller.py create mode 100644 script/upgrade_checker/dev/standard_vmap_gen.py create mode 100644 script/upgrade_checker/log.py create mode 100644 script/upgrade_checker/opengauss.py create mode 100644 script/upgrade_checker/project.py create mode 100644 script/upgrade_checker/report.py create mode 100644 script/upgrade_checker/rules/__init__.py create mode 100644 script/upgrade_checker/rules/category.py create mode 100644 script/upgrade_checker/rules/meta.py create mode 100644 script/upgrade_checker/rules/rule.py create mode 100644 script/upgrade_checker/rules/rule_maker.py create mode 100644 script/upgrade_checker/rules/vmap.py create mode 100644 script/upgrade_checker/style/__init__.py create mode 100644 script/upgrade_checker/style/markdown.py create mode 100644 script/upgrade_checker/utils/__init__.py create mode 100644 script/upgrade_checker/utils/command.py create mode 100644 script/upgrade_checker/utils/exception.py create mode 100644 script/upgrade_checker/utils/param.py create mode 100644 script/upgrade_checker/utils/singleton.py create mode 100644 script/upgrade_checker/utils/version.py create mode 100644 script/upgrade_checker/verifier.py diff --git a/script/gs_upgradechk b/script/gs_upgradechk new file mode 100644 index 00000000..03cfa350 --- /dev/null +++ b/script/gs_upgradechk @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + + +import sys +from upgrade_checker.utils.param import Param +from upgrade_checker.project import ProjectFactory + + +if __name__ == "__main__": + param = Param(sys.path[0] + '/upgrade_checker', sys.argv) + if param.is_help(): + print(param.helper) + exit(0) + + project = ProjectFactory.produce(param) + project.init() + project.run() + project.close() + diff --git a/script/upgrade_checker/README.md b/script/upgrade_checker/README.md new file mode 100644 index 00000000..03a0fffe --- /dev/null +++ b/script/upgrade_checker/README.md @@ -0,0 +1,105 @@ +# upgrade-checker + +## 简介 +upgrade-checker是一个升级之后元数据完整性的检测工具，由python3编写，在数据库升级之后，对元数据进行检验，检查在升级过程前后，是否出现元数据的损坏。目前仅支持检测builtin、initdb的元数据，对于用户所创建的表等，暂时不会进行检测。 + +主要功能如下： +- **导出**：基于某个数据库，生成一份元数据校验地图。 +- **校验**：基于某个元数据校验地图，对数据库进行元数据的校验，并生成报告。 + + +## 如何使用 +在安装了og的环境上，使用如下命令运行此工具： +```angular2html +python3 gs_upgradechk [ action [ config-params, ... ] ] +``` +其中action支持如下参数选项： +```angular2html +export 导出 +check | verify 校验 +help | -h | -? | --help 帮助 + +默认值为 help, 不区分大小写 +``` + +config-params支持的参数选项及其默认值如下 +```config +-p | --port : 16666 +-F | --report-format : md +-M | --report-mode : summary +-v | --vmap : '' +-d | --debug +``` + +- port：数据库端口号 +- debug：会打印更多的日志，用于工具问题定位。无参数，默认关闭，指定时开启。 +- report-format：校验报告格式，当前仅支持markdown。 +- report-mode：校验报告详细程度，支持`summary`，`detail`两种模式，默认`summary`模式。此选项仅会影响生成的报告的详细程度，并不影响校验内容的多少，`detail`仅是会把所有的校验规则无论对错都整理输出，报告整理比较耗时，而`summary`则仅整理输出错误的内容。 +- vmap：指定的元数据校验地图。若指定了，则在校验时会使用指定vmap，否则会自动检测目标数据库版本并在网上下载和使用我们生成好的vmap。 + +默认标准vmap下载地址如下： +```angular2html +https://opengauss.obs.cn-south-1.myhuaweicloud.com/upgrade_checker/standard_meta_verify_map_XX_X.X.X.vmap +``` +其中`XX`表示工具版本，`X.X.X`表示openGauss版本 + +## 版本支持 +工具版本与对应所支持的openGauss版本如下： + + + + + + + + + + + + + + + + + + + + +

工具版本

版本git id

支持openGauss

最新

-

3.0.3+，5.0.X，5.1.X

5010000

-

3.0.3+，5.0.X，5.1.X

+ +其中：`X`代表通配符，即全系列产品；`N+`代表范围，即N和之后的版本。 + +在openGauss升级时，按照新数据库的版本选择本工具版本 + + +## 工具结构 +```dir introduce +upgrade-checker + |d-- utils 工具代码-通用组件 + |d-- style 工具代码-报告风格 + |d-- rules 工具代码-规则组件 + |f-- project.py 工具代码 + |f-- .. 工具代码 + |d-- dev 开发者便携脚本 + |d-- workspace 工具工作目录 + |d-- vmaps -基准校验地图存放位置 + |d-- project_name-workspace -某次检测的运行工作目录 + |f-- run.log -运行日志 + |d-- results 工具结果目录 + |f- user-created-vmap -用户自己生成的校验地图 + |f- report -某次检测的结果报告 + |f-- README.md 说明书 +|f-- main.py 入口脚本 +``` + +## 校验原理 +数据库元数据，实际就是对一张张系统表以及其数据、gaussdb二进制内硬编码的部分数据等。 +在升级过程中，切换二进制后，硬编码内容部分也随之切换。因此对于元数据的校验，仅需要校验系统表及其数据即可。 + +对于系统表和数据的校验，最直接的便是通过查询语句进行查询。 +那么本工具实际上就是一个：执行一系列查询语句，并分析结果是否符合预期，并整理成报告的工具。 + +而在上述定义中涉及到的三个点：查询语句、预期结果、报告，分别如下定义： +- **规则(Rule)**：校验某个项目的一条查询语句。这条查询语句需要的查询结果需要按照key-value进行组织，其中key的作用是可以让我们更清楚地知道查询结果的每一列都是什么，更方便的进行整理分析。例如： `select oid as key, relname as val from pg_class where oid < 10000`。同时尽可能地区分builtin、initdb、user的数据。 +- **元数据校验地图(VerifyMap)**: 规则与其预期输出的集合。 +- **报告(Report)**: 规则执行结果与预期结果的对比分析结论所形成的汇总。 diff --git a/script/upgrade_checker/__init__.py b/script/upgrade_checker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/upgrade_checker/dev/__init__.py b/script/upgrade_checker/dev/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/upgrade_checker/dev/developer_guide.md b/script/upgrade_checker/dev/developer_guide.md new file mode 100644 index 00000000..b785b7f5 --- /dev/null +++ b/script/upgrade_checker/dev/developer_guide.md @@ -0,0 +1,63 @@ +[TOC] + +# 开发者工具脚本 +## StandardVmapGen + +## OGController + +## FastCheck + + + +# 工具升级指导 + +## 什么时候工具要升级 +首先我们需要明确如下几点： +1、工具的核心数据是一批规则，规则的中心是SQL +2、VMAP的本质是工具内的SQL及其查询结果 +3、工具将自身SQL在数据库内进行查询，并将查询结果与VMAP内预期结果进行对比。 + +而当工具内的规则发生变化，便需要升级工具，并生成对应的VMAP。 + + +一些典型的场景如下： +- openGauss元数据发生变化，导致工具内有些规则内的SQL执行报错，或者不再合理时 +- 工具内的 + +需要注意的是，单纯的openGauss元数据变化，并不代表需要升级工具。 + + +## 怎么去给工具做升级 +1、按照自己的需求，修改工具内的校验规则。 + +2、将version.py中UPGRADE_CHECKER_VERSION进行修改加一。 + +3、下载version.py中FOR_OPENGAUSS所列出来的openGauss版本压缩包，进行测试。 + +4、为测试通过的版本使用`Dev/StandardVmapGen.py`来导出基准校验地图。 + +5、更新根目录下《README.MD》内版本支持列表 + +6、将新的基准校验地图上传至openGauss华为云obs存储位置。 + + +# 适配openGauss新版本 +当openGauss发布新版本时，并不代表着需要升级工具，但一定需要为新版openGauss生成一个新的VMAP。 + +首先我们需要安装运行一个openGauss新版本，并且下载本工具，之后按照如下步骤适配openGauss最新版： + +1、使用当前工具，对openGauss新版本进行VMAP导出测试。 + - 修改version.py中的FOR_OPENGAUSS, 将新的openGauss版本添加到其中。 + - 使用工具导出新版本openGauss的VMAP。可正常导出，或用`Dev/StandardVmapGen.py` + +2、修复导出测试过程中的告警、错误并重新测试导出，直到完美导出VMAP。 + +3、使用导出的vmap，对openGauss新版本进行check校验。 + +4、修复校验过程中的告警、错误，校验报告中的告警、错误，并从头重新测试，直到完美的完成校验报告。 + +5、检查上述过程中的修改是否涉及到工具升级。若涉及则需要升级工具。 + +6、使用新的VMAP进行openGauss老版本到当前新版本的升级测试。 + +6、将新的标准VMAP(还有工具升级涉及生成的新的)，上传到openGauss华为云obs存储位置。 diff --git a/script/upgrade_checker/dev/og_controller.py b/script/upgrade_checker/dev/og_controller.py new file mode 100644 index 00000000..d530eed2 --- /dev/null +++ b/script/upgrade_checker/dev/og_controller.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# +""" +快速安装openGauss的bz2 +""" + +import os +import sys +import time + +sys.path.append(sys.path[0] + '/../..') +from upgrade_checker.utils.version import FOR_OPENGAUSS +from upgrade_checker.utils.command import Shell +from upgrade_checker.utils.exception import ShellExecException + +class OGController(object): + + @staticmethod + def is_support_package(pkg): + # openGauss bz2：openGauss-5.0.0-openEuler-64bit.tar.gz + parts = os.path.basename(pkg).split('-') + + if len(parts) < 4 or parts[0] != 'openGauss' or parts[3] != '64bit.tar.bz2': + return False, '非openGauss bz2压缩安装包：' + pkg + + if parts[1] not in FOR_OPENGAUSS: + return False, '工具不支持的OG版本：' + parts[1] + + return True, '' + + def __init__(self, package): + self.package = package + + self.gausshome = '' + self.version = '' + self.dn = '' + self.port = '' + + def _generate_env(self): + env = { + 'GAUSSHOME': self.gausshome, + 'PATH': '{0}/bin:{1}'.format(self.gausshome, os.getenv('PATH')), + 'LD_LIBRARY_PATH': '{0}/lib:{1}'.format(self.gausshome, os.getenv('LD_LIBRARY_PATH')) + } + + env_text = '' + for name, val in env.items(): + os.environ[name] = val + env_text += 'export {0}={1}\n'.format(name, val) + + env_file = '{0}/env.source'.format(self.gausshome) + with open(env_file, 'w') as f: + f.write(env_text) + print('生成环境变量', env_file) + + return env, env_file + + def install(self): + pkg_path = os.path.dirname(self.package) + pkg_name = os.path.basename(self.package) + self.version = pkg_name.split('-')[1] + self.gausshome = pkg_path + '/' + self.version + + if os.access(self.gausshome, os.F_OK): + self.dn = self.gausshome + '/dn' + try: + self.stop() + except ShellExecException as e: + pass + + try: + self.uninstall() + except ShellExecException as e: + pass + + os.mkdir(self.gausshome, 0o700) + + cmd = 'tar -jxf {0} -C {1}'.format(self.package, self.gausshome) + Shell.run(cmd, print_desc='解压安装包', check=True) + + return self._generate_env() + + def initdb(self): + self.dn = self.gausshome + '/dn' + cmd = "gs_initdb -D {0} -w Test@123 --nodename='sgnode' >/dev/null 2>&1 ".format(self.dn) + Shell.run(cmd, print_desc='初始化数据库', check=True) + + def guc(self, setting): + cmd = 'gs_guc reload -D {0} -c "{1}" >/dev/null 2>&1'.format(self.dn, setting) + Shell.run(cmd, print_desc='设置GUC参数') + + def start(self): + cmd = "gs_ctl start -D {0} >/dev/null 2>&1".format(self.dn) + Shell.run(cmd, print_desc='启动数据库', check=True) + + def stop(self): + cmd = "gs_ctl stop -D {0} >/dev/null 2>&1 ".format(self.dn) + Shell.run(cmd, print_desc='关闭数据库', check=True) + + def uninstall(self): + cmd = "rm {0} {1} -fr".format(self.dn, self.gausshome) + Shell.run(cmd, print_desc='卸载数据库', check=True) + +if __name__ == "__main__": + og = OGController('/data/pkg/openGauss-5.0.0-openEuler-64bit.tar.gz') + og.install() + og.initdb() + og.start(16666) diff --git a/script/upgrade_checker/dev/standard_vmap_gen.py b/script/upgrade_checker/dev/standard_vmap_gen.py new file mode 100644 index 00000000..736d028e --- /dev/null +++ b/script/upgrade_checker/dev/standard_vmap_gen.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +""" +批量生成基准校验地图 + +用法： + 1、下载openGauss bz2安装包，保存至任意位置，可放多个包。 + - 举例 /data/pkg/openGauss-5.0.0-openEuler-64bit.tar.gz + /data/pkg/openGauss-5.1.0-openEuler-64bit.tar.gz + + 2、执行本脚本： python StandardVmapGen.py path port [ no-clean ] + - path: 安装包所在路径：/data/pkg + - port: 为提供临时安装运行openGauss所用，如 16666 + - no-clean: 不清理临时安装环境，此时port会+5以间隔 + + 基准vmap会自动生成到安装包所在路径。 +""" + +import os +import sys +import time + +sys.path.append(sys.path[0] + '/../..') +from upgrade_checker.utils.version import UPGRADE_CHECKER_VERSION +from upgrade_checker.utils.param import Param +from upgrade_checker.project import ExportProj +from upgrade_checker.rules.vmap import VerifyMap +from upgrade_checker.dev.og_controller import OGController + + +class FakeParam(Param): + """ + 运行一个导出操作所需要的参数 + """ + def __init__(self, port): + fake_root = sys.path[0] + '/..' + super(FakeParam, self).__init__(fake_root, ['', 'export', '-p', str(port)]) + + +class FakeExportProj(ExportProj): + """ + 导出工程，将导出位置、文件名称进行修改，改成我们想要的位置和名称格式 + """ + def __init__(self, port, og_version, export_dir): + super(FakeExportProj, self).__init__(FakeParam(port)) + self.export_dir = export_dir + standard_vmap_name = VerifyMap.standard_name(UPGRADE_CHECKER_VERSION, og_version) + self.vmap = '{0}/{1}'.format(self.export_dir, standard_vmap_name) + + +class VmapGen(object): + """ + Vmap生成器。批量安装openGauss包，并生成vmap + """ + def __init__(self, package_path, port, clean): + """ + :param package_path: 存放了openGauss bz2安装包的路径 + :param port: 用于临时安装运行openGauss的端口 + :param clean: 清理临时安装环境 + """ + self.package_path = package_path + self.port = port + self.clean = clean + + def run(self): + files = os.listdir(self.package_path) + vmaps = [] + for f in files: + pkg = self.package_path + '/' + f + support, msg = OGController.is_support_package(pkg) + if not support: + print('\n[跳过文件或文件夹]:', msg) + continue + + print('\n[开始执行第{}个任务]'.format(len(vmaps) + 1)) + og = OGController(pkg) + og.install() + og.initdb() + og.guc('port = {0}'.format(self.port)) + og.guc('enable_wdr_snapshot = on') + og.start() + time.sleep(2) + + proj = FakeExportProj(self.port, og.version, self.package_path) + proj.init() + proj.run() + vmap = proj.close() + vmaps.append(vmap) + + if self.clean: + og.stop() + og.uninstall() + else: + self.port += 5 + + print('第{0}个任务执行完成，输出vmap: {1}'.format(len(vmaps), vmap)) + + print('\n批量共生成{0}个vmap'.format(len(vmaps))) + print('\n'.join(vmaps)) + + +if __name__ == '__main__': + argv_pkg_path = sys.argv[1] + argv_port = int(sys.argv[2]) + argv_clean = False if len(sys.argv) > 3 and sys.argv[3].lower() == 'no-clean' else True + VmapGen(argv_pkg_path, argv_port, argv_clean).run() + diff --git a/script/upgrade_checker/log.py b/script/upgrade_checker/log.py new file mode 100644 index 00000000..050929ea --- /dev/null +++ b/script/upgrade_checker/log.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# +""" +日志模块 +""" + + +import time +import sys +import traceback +from enum import Enum +from upgrade_checker.utils.singleton import singleton + + +class LogLevel(Enum): + DEBUG = 0 # 不打印，或打印到日志 + LOG = 1 # 打印到日志 + INFO = 2 # 打印到日志和屏幕 + WARNING = 3 # 打印到日志和屏幕 + ERROR = 4 # 打印到日志和屏幕，并退出 + + +@singleton +class Logger(object): + + def __init__(self): + self._file = None + self._debug = False # debug模式 + self._in_process_bar = False # 标记当前正在打印一个进度条 + + def __del__(self): + sys.stdout.flush() + if self._file is not None: + self._file.close() + + def set_file(self, file_path): + if self._file is not None: + sys.stdout.flush() + self._file.close() + + try: + self._file = open(file_path, 'a') + except FileNotFoundError: + print('无法打开文件', file_path) + exit(1) + + self.log('Logger 日志文件设置成功： %s.' % file_path) + + def set_debug(self, state): + self._debug = state + + def _format_content(self, log_level, content, hint): + """ + 将日志内容整理成带时间、等级等的格式，用于记录或者输出。 + :param log_level: 等级 + :param content: 内容 + :param hint: 提示 + :return: + """ + s = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime()) + \ + " [{0}] ".format(log_level.name) + \ + content + \ + (("\n HINT: " + hint) if hint is not None else "") + \ + '\n' + return s + + def _write_log(self, log_content): + """ + 日志写入文件 + :param log_content: 日志内容 + :return: + """ + assert self._file is not None + self._file.write(log_content) + + def _print_log(self, log_content): + """ + 日志打印到屏幕 + :param log_content: 日志内容 + :return: + """ + if self._in_process_bar: + print('') + self._in_process_bar = False + + print(log_content, end='') + + def debug(self, content, hint=None): + """ + 定位信息，开启debug时写入文件。 + :param content: 内容 + :param hint: 提示 + :return: + """ + if not self._debug: + return + res = self._format_content(LogLevel.DEBUG, content, hint) + self._write_log(res) + + def log(self, content, hint=None): + """ + 日志信息，写入文件 + :param content: 内容 + :param hint: 提示 + :return: + """ + res = self._format_content(LogLevel.LOG, content, hint) + self._write_log(res) + + def info(self, content, hint=None): + """ + 普通信息。打印屏幕和写入文件 + :param content: 内容 + :param hint: 提示 + :return: + """ + res = self._format_content(LogLevel.INFO, content, hint) + self._write_log(res) + self._print_log(res) + + def warning(self, content, hint=None): + """ + 警告信息，打印屏幕和写入文件 + :param content: 内容 + :param hint: 提示 + :return: + """ + res = self._format_content(LogLevel.WARNING, content, hint) + self._write_log(res) + self._print_log(res) + + def err(self, content, hint=None): + """ + 错误信息，打印日志和写入文件，失败直接结束 + :param content: 内容 + :param hint: 提示 + :return: + """ + bt = ''.join(traceback.format_stack()[0:-1]) + content = content + '\n' + 'Traceback (most recent call last):\n' + bt + + res = self._format_content(LogLevel.ERROR, content, hint) + self._write_log(res) + self._print_log(res) + + sys.stdout.flush() + exit(1) + + def process_bar(self, percentage): + """ + 进度条，打印在屏幕上。 + :param percentage: [0,100] 整形参数，表示进度百分比。 + :return: + """ + assert 0 <= percentage <= 100 + + self._in_process_bar = True + print('\r进度 {}%:'.format(percentage), '[' + ('█' * percentage) + (' ' * (100 - percentage)) + ']', end='') + + def process_bar_text(self, content): + """ + 直接文本进度条，打印在屏幕上。 + :param content: 直接文本 + :return: + """ + self._in_process_bar = True + print('\r' + content, end="") + + +# 单例模式，全局一个logger +logger = Logger() diff --git a/script/upgrade_checker/opengauss.py b/script/upgrade_checker/opengauss.py new file mode 100644 index 00000000..9abd567c --- /dev/null +++ b/script/upgrade_checker/opengauss.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +""" +openGauss查询模块，用于连接og进行sql查询，并将查询结果进行包装返回。 +""" + +import subprocess +from subprocess import Popen, PIPE +from upgrade_checker.log import logger +from upgrade_checker.utils.singleton import singleton +from upgrade_checker.utils.version import FOR_OPENGAUSS +from upgrade_checker.utils.command import Shell + + +class QueryResult(object): + """ + og查询结果。用于将查询结果进行封装和处理。 + 需要注意的是，目前暂时不能处理结果行中包含换行的场景，会导致结果拆分错误，因此在设计查询SQL时应注意。 + """ + def __init__(self, sql, data, err): + self._sql = sql # sql + self.errmsg = err if err != '' else None # 错误信息 + self.attr = [] # 列名 + self.data = [] # 结果 + self._idx = 0 # 迭代输出结果的指针 + + if self.errmsg is not None: + logger.err("openGauss查询出错，\n查询命令:\n%s\n错误信息：\n%s" % (sql, err)) + + if data is None or len(data) == 0: + return + + data = data.split('\n') + self.attr = [name.strip() for name in data[0].split('|')] + for row in data[2:-3]: # 删掉最后的行数和空行 + self.data.append(tuple(col.strip() for col in row.split('|'))) + + def __getitem__(self, idx): + if idx >= len(self.data): + raise StopIteration + row = self.data[idx] + return row + + def row_count(self): + return len(self.data) + + def col_count(self): + return len(self.attr) + + def size(self): + """ + :return: 行数, 列数 + """ + return self.row_count(), self.col_count() + + def output(self): + return self.attr, self.data + + def value(self): + assert self.size() == (1, 1) + return self.data[0][0] + + def reset_iterate(self): + self._idx = 0 + + def iterate(self, format_dict=False): + """ + 迭代输出下一行，直接输出tuple，或者组装成dict进行输出 + :return: + """ + if self._idx >= len(self.data): + self._idx = 0 + return + + if format_dict: + row = dict(zip(self.attr, self.data[self._idx])) + else: + row = self.data[self._idx] + + return row + + +@singleton +class OpenGauss(object): + """ + openGauss连接模块，用于连接openGauss，执行查询、非查询语句。 + """ + def __init__(self): + self._dbname = 'postgres' + self._port = 5432 + self.version_info = '' + self.version = '' + self.nodename = '' + + def __str__(self): + return '{0} [ RUN - nodename {1} port {2} ]'.format( + self.version_info, + self.nodename, + self._port) + + def connect(self, dbname=None, port=None): + """ + 链接数据库 + :param dbname: database name + :param port: 端口号 + :return: + """ + self._dbname = dbname if dbname is not None else self._dbname + self._port = port if port is not None else self._port + + self.version_info = self.query('select version();').value() + parts = self.version_info.split(' ') + self.version = parts[1] if parts[0] == '(openGauss' else parts[3] + if self.version not in FOR_OPENGAUSS: + logger.err('暂不支持 openGauss ' + self.version) + + res = self.query('show pgxc_node_name;') + self.nodename = res.data[0][0] + + logger.log('openGauss {0} {1} (port={2} db={3}) 连接成功.'.format( + self.version, + self.nodename, + self._port, + self._dbname)) + return self + + def query(self, sql, dbname=None, port=None): + """ + 执行select，获取结果 + :param sql: 查询语句 + :param dbname: 指定的数据库名 + :param port: 指定的端口号 + :return: + """ + dbname = dbname if dbname is not None else self._dbname + port = port if port is not None else self._port + cmd = 'gsql -p {0} -d {1} -r'.format(port, dbname) + data, err = Shell.communicate(cmd, sql, check=False) + logger.debug('openGauss查询执行：{0}\n{1}'.format(cmd, sql)) + return QueryResult(sql, data, err) + + def execute(self, sql): + """ + 执行非select语句。 + :param sql:查询语句 + :return: + """ + cmd = 'gsql -d %s -p %d -c "%s"' % (self._dbname, self._port, sql) + Shell.communicate(cmd, sql, check=True) + + +# 单例模式，全局仅一个 +og = OpenGauss() + + +if __name__ == "__main__": + pass diff --git a/script/upgrade_checker/project.py b/script/upgrade_checker/project.py new file mode 100644 index 00000000..f04f9d1e --- /dev/null +++ b/script/upgrade_checker/project.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# +""" +”工程“文件，将整个程序的运行抽象为一个工程类。 +通过这个功能工程类来启动和运行工程。 +""" + +import time +import os + +from upgrade_checker.utils.param import Action, ReportFormat, ReportMode, Param +from upgrade_checker.log import logger +from upgrade_checker.opengauss import og +from upgrade_checker.report import Reporter +from upgrade_checker.rules.vmap import VerifyMap +from upgrade_checker.verifier import Collector, Analyzer, Exporter + + +class Project(object): + + def __init__(self, param): + """ + 初始化程序运行的各种数据参数，主要拼接路径，路径的创建等后续逐步进行。 + :param param: 启动参数 + """ + assert param.action != Action.HELP + self.param = param + + self.dir_root = param.root_path + self.dir_workspace = "{0}/workspace".format(self.dir_root) + self.dir_vmap = "{0}/vmap".format(self.dir_workspace) + self.dir_results = "{0}/results".format(self.dir_root) + + self.id = int(time.time()) + self.name = "{0}-{1}".format( + param.action.name, + time.strftime('%Y-%m-%d-%H_%M_%S', time.localtime(self.id)) + ) + self.workspace = "{0}/{1}".format(self.dir_workspace, self.name) + self.log = "{0}/run.log".format(self.workspace) + + def __str__(self): + info = self.param.__str__() + info += 'dir_root: ' + self.dir_root + '\n' + info += 'dir_workspace: ' + self.dir_workspace + '\n' + info += 'dir_vmap: ' + self.dir_vmap + '\n' + info += 'dir_results: ' + self.dir_results + '\n' + info += 'id: ' + str(self.id) + '\n' + info += 'name: ' + self.name + '\n' + info += 'workspace: ' + self.workspace + '\n' + info += 'log: ' + self.log + '\n' + return info + + def _prepare_workspace(self): + """ + 创建工作目录文件夹等。 + :return: + """ + if not os.access(self.dir_workspace, os.F_OK): + os.mkdir(self.dir_workspace, 0o700) + + if not os.access(self.dir_vmap, os.F_OK): + os.mkdir(self.dir_vmap, 0o700) + + if not os.access(self.dir_results, os.F_OK): + os.mkdir(self.dir_results, 0o700) + + os.mkdir(self.workspace, 0o700) + + def init(self): + self._prepare_workspace() + + logger.set_file(self.log) + logger.set_debug(self.param.debug.value) + + og.connect('postgres', self.param.port.value) + + def run(self): + pass + + def close(self): + pass + + +class ExportProj(Project): + def __init__(self, param): + assert param.action == Action.EXPORT + super(ExportProj, self).__init__(param) + self.vmap = "{0}/{1}.vmap".format(self.dir_results, self.name) + + def __str__(self): + info = 'Export Project as:' + info += super(ExportProj, self).__str__() + info += 'vmap: ' + self.vmap + '\n' + return info + + def init(self): + super(ExportProj, self).init() + + logger.log("工程初始配置完成。") + logger.debug('工程信息：\n' + self.__str__()) + + def run(self): + db_list = Collector.prepare_db_list(Action.VERIFY) + collector = Collector(db_list) + exporter = Exporter(self.vmap) + + logger.info('开始进行数据采集与记录。') + for rule in collector: + exporter.record(rule) + + percentage = collector.get_progress() + logger.process_bar(percentage) + + logger.process_bar(100) + logger.info('数据采集与记录完成，开始导出基准校验地图。') + exporter.export() + logger.info('基准校验地图导出完成:{0}'.format(self.vmap)) + + def close(self): + return self.vmap + + +class VerifyProj(Project): + def __init__(self, param): + assert param.action == Action.VERIFY + super(VerifyProj, self).__init__(param) + self.vmap = self.param.vmap.value + self.report = "{0}/{1}-report.{2}".format( + self.dir_results, + self.name, + ReportFormat.suffix(self.param.report_format.value) + ) + + def __str__(self): + info = 'Verify Project as:' + info += super(VerifyProj, self).__str__() + info += 'vmap: ' + self.vmap + '\n' + info += 'report: ' + self.report + '\n' + return info + + def init(self): + super(VerifyProj, self).init() + + # 准备基准校验文件，仅保证文件存在，是否可用将在后续加载时校验 + if self.vmap is not None: + if os.access(self.vmap, os.F_OK): + return + else: + logger.err('指定的基准校验地图文件{0}不存在。'.format(self.vmap)) + self.vmap = VerifyMap.prepare_vmap_file(self.dir_vmap) + + logger.log("工程初始配置完成。") + logger.debug('工程信息：\n' + self.__str__()) + + def run(self): + db_list = Collector.prepare_db_list(Action.VERIFY) + collector = Collector(db_list) + analyzer = Analyzer(self.vmap) + reporter = Reporter(self.report, + self.param.report_format.value, + self.param.report_mode.value) + reporter.record_info(og, analyzer.vmap) + + logger.info('开始进行数据采集与分析。') + for rule in collector: + conclusion = analyzer.analyze(rule) + reporter.record(collector.current_db(), conclusion) + + percentage = collector.get_progress() + logger.process_bar(percentage) + + logger.process_bar(100) + logger.info('数据采集与分析完成，准备整理报告。') + reporter.report() + logger.info('报告导出完成：{0}'.format(self.report)) + + def close(self): + return self.report + + +class ProjectFactory(object): + @staticmethod + def produce(param): + if param.action == Action.EXPORT: + return ExportProj(param) + elif param.action == Action.VERIFY: + return VerifyProj(param) + else: + assert False + + +if __name__ == "__main__": + pass diff --git a/script/upgrade_checker/report.py b/script/upgrade_checker/report.py new file mode 100644 index 00000000..07d3a1ca --- /dev/null +++ b/script/upgrade_checker/report.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# +""" +标题: 报告总结 +数据库信息：版本、节点名等 +标准元数据地图：xxxxxxxxx +校验结果：通过 or 失败 +内容摘要： + 共校验xx个数据库。 + 累计xx个系统元数据表，其中xx通过，xx个失败。 + 校验规则共xx条，xx条通过，xx条失败。 + Category一共xxx个,其中xx个完全通过，xx个存在失败内容 + 。。。 +失败项摘要： + 共计xxx条错误。提示：您可以通过编号在本文报告内快速搜索，查看对应的更详细的校验规则语句等内容。 + 编号 数据库 分类 校验内容 失败原因 + err1 pg 函数 pg_proc的内容，忽略的列有.. 键值（oid）为xxx的行内容错误 + err2 us1 编码 pg_encoding的内容，忽略的列有.. 键值（oid）为xxx的行内容缺失 + err3 us2 表 pg_type的列结构 键值（attname）为xxx的内容多余 + err4 us2 表 使用x函数检测pg_type的。。。 检测结果不一致 + +警告： + 存在未分类的系统元数据表: + 建议排查来源，并修改本工具，添加至分类列表中。 + +TIPS:详细校验原理与说明详见本工具的"README.md"。 + +详细内容： +数据库postgres +元数据系统表结构 + 共xx个系统表，校验项目xx条，通过xx条，失败xx条 + 表名 校验规则 规则解读 规则结果行号 规则结果([键:]内容) 预期规则结果 校验结果(失败原因) + pg_class select fun 使用x函数.. 1 fwwsf48 fgw1gr8 err1, 失败，检测结果不一致 + pg_index selec att 列结构 1 indxxx:xx15adfsa err2, 失败，列indxx缺失 + pg_index selec att 列结构 2 indxxx:xx15adfsa err3, 失败，列indxx多余 + pg_index selec att 列结构 3 indxxx:xx15adfsa indxxx:xxxxx err4, 失败，列indxx不一致 + pg_index selec att 列结构 4 indxxx:xx15adfsa indxxx:xx15adfsa 成功 + +分类内容 + 共xx个系统表，校验项目xx条，通过xx条，失败xx条 + +数据库对象 + +数据库user1db +元数据系统表结构 + .... + +""" + +from upgrade_checker.utils.param import ReportFormat, ReportMode +from upgrade_checker.log import logger +from upgrade_checker.rules.rule import StructRule, ContentRule, CommonRule, ConclusionState +from upgrade_checker.style.markdown import MarkDown + + +class Report(object): + """ + 报告 + """ + + def __init__(self): + self.db_info = '' + self.vmap_info = '' + self.pci_suc_count = 0 + self.pci_err_count = 0 + self.rule_suc_count = 0 + self.rule_err_count = 0 + self.detail_suc_count = 0 + self.detail_err_count = 0 + self.err_rules_res = [] + self.warnings = [] + self.content = {} # { "dbname": [struct rules, content rules, common rules], ... } + + def serialize(self, style): + rule_count = self.rule_suc_count + self.rule_err_count + detail_count = self.detail_suc_count + self.detail_err_count + verify_result = '成功' if self.rule_err_count == 0 else '失败' + db_summary = "共校验 %d 个数据库。" % len(self.content) + rul_summary = "校验规则累计执行%d条，%d条通过，%d条失败。" % (rule_count, self.rule_suc_count, self.rule_err_count) + detail_summary = "所有校验规则累计生成%d条内容小项，%d条通过，%d条失败。" % (detail_count, self.detail_suc_count, self.detail_err_count) + warning_summary = "警告信息%d条。" % len(self.warnings) + + report = style.title(1, "校验报告") + report += style.title_paragraph("数据库信息", self.db_info) + report += style.title_paragraph("基准元数据地图", self.vmap_info) + report += style.title_paragraph("校验结果", verify_result) + report += style.emphasize("内容摘要：") + report += style.unordered_list([db_summary, rul_summary, detail_summary, warning_summary]) + report += style.emphasize("失败项摘要：") + report += style.table(["数据库", "校验规则", "规则解读", "失败行号", "校验结果", "结果解读"], self.err_rules_res) + report += style.emphasize("警告：") + report += style.unordered_list(self.warnings) + report += style.title_paragraph("TIPS", "详细校验原理与说明参考本工具的《README.md》") + + report += style.title(1, "详细数据") + for dbname, contents in self.content.items(): + report += style.title(2, "数据库%s" % dbname) + report += style.title(3, "元数据系统表结构") + report += style.table(["表名", "校验规则", "规则解读", "行号", "校验结果", "结果解读"], + contents[0]) + report += style.title(3, "元数据系统表内容") + report += style.table(["表名", "类别", "校验规则", "规则解读", "行号", "校验结果", "结果解读"], + contents[1]) + report += style.title(3, "常规通用校验") + report += style.table(["类别", "校验规则", "规则解读", "行号", "校验结果", "结果解读"], + contents[2]) + + return report + + +class StyleFactory(object): + """ + 风格工厂 + """ + @staticmethod + def produce(fmt): + if fmt == ReportFormat.MARKDOWN: + return MarkDown() + else: + return MarkDown() + + +class Reporter(object): + """ + 报告生成器 + """ + def __init__(self, file, fmt, granularity): + self._file_path = file + + try: + self._file = open(file, 'a') # 结果文件 + except FileNotFoundError: + logger.err('无法打开文件:' + file) + + self._current_errno = 1 # 当前失败编号 + self._current_dbname = '' # 当前记录的数据库名称 + self._report = Report() # 结果 + self._style = StyleFactory.produce(fmt) # 风格 + self._granularity = granularity + + def __del__(self): + if self._file is not None: + self._file.close() + + def __str__(self): + return 'Reporter: granularity{0}, file({1})'.format(self._granularity, self._file_path) + + def record_info(self, db_info='', vmap_info=''): + self._report.db_info = db_info + self._report.vmap_info = vmap_info + + def _collect_statistic(self, old_errno, new_errno, dbname, conclusion): + """ + 整理记录校验过程中的统计信息。 + :param old_errno: + :param new_errno: + :param dbname: + :param conclusion: + :return: + """ + suc = True if old_errno == new_errno else False + if suc: + self._report.rule_suc_count += 1 + else: + self._report.rule_err_count += 1 + + detail_err_count = new_errno - old_errno + detail_suc_count = len(conclusion.details) - detail_err_count + self._report.detail_suc_count += detail_suc_count + self._report.detail_err_count += detail_err_count + + logger.debug('reporter 新增记录规则结论，结论成功状态{0}, 结论明细共计{1}条, 成功{2}，失败{3}。'.format( + suc, + len(conclusion.details), + detail_suc_count, + detail_err_count) + ) + if self._granularity == ReportMode.DETAIL: + data = self._report.content.get(dbname) + assert data is not None + logger.debug('当前数据库{0}记录结论小项分别为{1},{2},{3}'.format( + dbname, + len(data[0]), + len(data[1]), + len(data[2])) + ) + + def _transform_conclusion(self, dbname, conclusion): + """ + + :param dbname: + :param conclusion: + :return: + """ + row_num = 0 + for row in conclusion.details: + row_num += 1 + result_state = 'SUCCESS' if row['state'] == ConclusionState.SUCCESS else \ + 'ERROR(err-num-%d)' % self._current_errno + row_res_base = [ + conclusion.rule.sql, + conclusion.rule.sql_desc, + row_num, + result_state, + row['summary'] + ] + if row['state'] == ConclusionState.FAILED: + self._report.err_rules_res.append([dbname] + row_res_base) + self._current_errno += 1 + + if self._granularity == ReportMode.SUMMARY: + continue + + data = self._report.content.get(dbname) + assert data is not None + if isinstance(conclusion.rule, StructRule): + data[0].append([conclusion.rule.rel_key] + row_res_base) + elif isinstance(conclusion.rule, ContentRule): + data[1].append([conclusion.rule.rel_key, conclusion.rule.category] + row_res_base) + elif isinstance(conclusion.rule, CommonRule): + data[2].append([conclusion.rule.category] + row_res_base) + else: + assert False + + def record(self, dbname, conclusion): + """ + 记录一条结论到report内。 + """ + if dbname != self._current_dbname: + self._report.content[dbname] = [[], [], []] + self._current_dbname = dbname + + self._report.warnings += conclusion.warnings + + old_errno = self._current_errno + self._transform_conclusion(dbname, conclusion) + new_errno = self._current_errno + + self._collect_statistic(old_errno, new_errno, dbname, conclusion) + + def report(self): + """ + :return: + """ + logger.log('开始整理生成报告......') + if self._granularity == ReportMode.DETAIL: + logger.info('详细报告整理中，这可能需要较长时间......') + + self._file.write(self._report.serialize(self._style)) + logger.log('报告生成完成 %s。' % self._file_path) diff --git a/script/upgrade_checker/rules/__init__.py b/script/upgrade_checker/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/upgrade_checker/rules/category.py b/script/upgrade_checker/rules/category.py new file mode 100644 index 00000000..c2a3193c --- /dev/null +++ b/script/upgrade_checker/rules/category.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +from enum import Enum + +class Category(Enum): + """ + 聚类，用于对系统表进行分类和报告的输出。 + 值的大小目前并没有什么实际意义，但修改时仍需要升级工具。 + """ + UNKNOWN = 0 # 未知 + DATABASE = 1 # 数据库 + TABLESPACE = 2 # 表空间 + SCHEMA = 3 # 模式、名称空间 + TABLE = 4 # 表 + VIEW = 5 # 视图 + INDEX = 6 # 索引 + FUNCTION = 7 # 函数、存储过程 + TYPE = 8 # 数据类型 + OPERATOR = 9 # 操作符 + TRIGGER = 10 # 触发器 + SEQUENCE = 11 # 序列 + CONSTRAINT = 12 # 约束 + AM = 13 # 访问方法 access method + LANGUAGE = 14 # 语言 + PACKAGE = 15 # 包 + RULE = 16 # 规则 + LOCALE = 17 # 编码、排序等 + DIRECTORY = 18 # 文件夹 + SYNONYM = 19 # 同义词 + DESCRIPTION = 20 # 注释 + EXTENSION = 21 # 插件扩展 + FDW = 22 # 外部数据包装器 + TS = 23 # 文本分词搜索 + RECYCLE_BIN = 24 # 回收站 + PUB_SUB = 25 # 发布订阅 + OPTIMIZER = 26 # 优化器 + TRANSACTION = 27 # 事务 + REPLICATION = 28 # 流复制 + INFO_SCHEMA = 29 # information schema + PLDEV_SCHEMA = 30 # pl_developer schema + AI = 31 # AI + DEPEND = 32 # 依赖关系 + GUC = 33 # guc 配置 + JOB = 34 # 定时任务 + OBS = 35 # 对象存储 + LOB = 36 # 大对象 + OBJECT = 37 # 用户数据库对象 + AUTHENTICATION = 38 # 认证 + PRIVILEGE = 39 # 权限 + SECURITY = 40 # 安全 + WLM = 41 # workload manager 资源管理 + DFX = 42 # Design For X + WDR = 43 # wdr报告 + DISTRIBUTE = 44 # 分布式 + + diff --git a/script/upgrade_checker/rules/meta.py b/script/upgrade_checker/rules/meta.py new file mode 100644 index 00000000..37e2428b --- /dev/null +++ b/script/upgrade_checker/rules/meta.py @@ -0,0 +1,2153 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +""" +校验规则的元数据，根据这些数据、结构，生成校验规则。 +""" + +from enum import Enum +from upgrade_checker.rules.category import Category + + +class Accuracy(Enum): + STRICT = 0 # 严格精确 + ALLOW_MORE = 1 # 查询允许多数据，对于完全无法区分是用户数据还是内建数据的检验sql，我们只能够通过允许多数据来解决。 + ALLOW_LACK = 2 # 查询允许少数据，用于在系统表不稳定存在时，可以允许缺少这些查询数据。 + + +class Meta(object): + def __init__(self, category=Category.UNKNOWN, desc="", certain=True, content_meta=None): + self.category = category + self.desc = desc + self.certain = certain + self.content_rules_meta = content_meta if content_meta is not None else [] + + +class ContentRulesMeta(object): + def __init__(self, key=None, filters=None, ignore_col=None, oid_col=None, + complete_sql=None, complete_sql_desc=None, + key_desc="内容哈希值为%s的行", accuracy=Accuracy.STRICT): + """ + 用来生成校验一个表的内容的信息。这些入参有两种使用方式， + 1、可以使用第一行的几个入参，拼凑出来一句完成的SQL语句 + 2、在第二行的入参中，如果提供了完整的SQL，则不再去进行拼接。 + 第三行的入参为前两中方式共用的。 + 建议尽可能使用第一种方式，因为这样的话当表结构发生变化时，会具有一定的自适应能力。 + + :param key: sql的键，必须是明确的一个投影，None表示用所有需要校验的列进行字符串连接做投影。 + :param filters: where过滤条件，用与区分用户数据和builtin数据。None表示不过滤 + :param ignore_col: 需要忽略的列。None表示使用所有列。 + :param oid_col: 需要校验的列中，类型为oid的需要特殊处理列。特殊处理的话，这些会被0-9999，1W-16383的区分处理，1W以上安0。None表示没有 + :param complete_sql: 完整的SQL规则，如果提供这个选项，则不会再用前面的拼接参数去拼接组装。 + :param complete_sql_desc: 完整的SQL描述。 + :param key_desc: sql的键的描述，注意必现带一个%s + :param accuracy: 校验精度, 默认严格 + """ + self.key = key + self.key_desc = key_desc + self.filters = filters if filters is not None else 'true' + self.ignore_col = ignore_col.split(',') if ignore_col is not None else [] + self.oid_col = oid_col.split(',') if oid_col is not None else [] + self.complete_sql = complete_sql + self.complete_sql_desc = complete_sql_desc + self.accuracy = accuracy + + +META = { + 'default.default': Meta( + Category.UNKNOWN, + '未知系统表', + True, + [ + ContentRulesMeta() + ] + ), + 'pg_catalog.pg_default_acl': Meta( + Category.AUTHENTICATION, + '存储新建对象设置的初始权限', + True + ), + 'pg_catalog.pg_pltemplate': Meta( + Category.FUNCTION, + '存储过程语言的"模板"信息', + True, + [ + ContentRulesMeta( + key='tmplname', + key_desc='名称为%s的模板' + ) + ] + ), + 'pg_catalog.pg_tablespace': Meta( + Category.TABLESPACE, + '所有表空间的信息', + True, + [ + ContentRulesMeta( + key='spcname', + key_desc='名称为%s的表空间', + filters=' oid < 10000 ' + ) + ] + ), + 'pg_catalog.pg_shdepend': Meta( + Category.DEPEND, + '记录数据库对象和共享对象之间的依赖性关系', + True + # 暂时不进行校验，原因同 pg_depend + ), + 'pg_catalog.pg_type': Meta( + Category.TYPE, + '记录所有的数据类型信息', + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的数据类型', + filters=' oid < 10000 ', + ignore_col='tydefaultbin,typacl', + oid_col='typnamespace,typrelid,typelem' + ), + ContentRulesMeta( + key="(select nspname from pg_namespace n where n.oid = typnamespace) || '.' || typname", + key_desc='数据类型%s', + filters=' oid < 10000 ', + ignore_col='oid,tydefaultbin,typacl', + oid_col='typnamespace,typowner,typrelid,typelem,typarray,typbasetype,typcollation' + ) + ] + ), + 'pg_catalog.pg_attribute': Meta( + Category.TABLE, + "存储了所有表、视图的所有列的基础信息", + True, + [ + # builtin的需要严格一致 + ContentRulesMeta( + key="format('%s(%s)'," + " (select format('%s.%s', nspname, relname)" + " from pg_class c left join pg_namespace n on c.relnamespace = n.oid " + " where c.oid=attrelid)," + " attname" + ")", + key_desc='模式.关系名(列名)为%s的列', + filters=' attrelid < 10000 ' + ), + # initdb的。但忽略toast表，因为表名里也带oid，会变，无法校验。 + ContentRulesMeta( + complete_sql="select format('%s.%s(%s)', n.nspname, c.relname, a.attname)," + " md5(format('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s', " + " t.typname, a.attstattarget, a.attlen, a.attnum, a.attndims," + " a.attcacheoff, a.atttypmod, a.attbyval, a.attstorage, a.attalign," + " a.attnotnull, a.atthasdef,a.attisdropped, a.attislocal, a.attcmprmode," + " a.attinhcount, a.attcollation, a.attacl, a.attoptions, a.attfdwoptions," + " a.attinitdefval, a.attkvtype))" + "from pg_attribute a left join pg_class c on a.attrelid = c.oid " + " left join pg_namespace n on c.relnamespace = n.oid " + " left join pg_type t on a.atttypid = t.oid " + "where a.attrelid > 9999 and " + " a.attrelid < 16384 and " + " a.attisdropped = false and " + " n.nspname not in ('pg_toast')", + key_desc='模式.关系名(列名)为%s的列', + complete_sql_desc='校验initdb阶段(1W <= oid <= 16384)创建的表、索引、视图等的列信息。' + ) + ] + ), + 'pg_catalog.pg_proc': Meta( + Category.FUNCTION, + '存储了所有的函数、存储过程', + True, + [ + ContentRulesMeta(Category.FUNCTION, + complete_sql = "select 'count(*)', count(*) from pg_proc where oid < 16384", + key_desc = '数量统计方法%s', + complete_sql_desc = 'pg_proc内 oid < 16384 的系统对象(包括函数、存储过程)总数量' + ), + ContentRulesMeta( + key="oid", + key_desc='oid为%s的函数或存储过程', + filters=' oid < 10000 ', + ignore_col='proargdefaults' + ), + ContentRulesMeta( + key="format('%s.%s(%s)'," + " (select nspname from pg_namespace n where n.oid = pronamespace)," + " proname," + " pg_get_function_arguments(oid)" + ")", + key_desc='函数%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid,proargtypes,proallargtypes,proargdefaults,proacl,proargtypesext,allargtypes,allargtypesext', + oid_col='pronamespace,prolang,provariadic,prorettype,propackageid' + ), + ContentRulesMeta( + complete_sql="select format('def:%s.%s(%s)', " + " n.nspname, " + " p.proname, " + " pg_get_function_arguments(p.oid)), " + " md5(pg_get_functiondef(p.oid)::text) " + "from pg_proc p left join pg_namespace n on p.pronamespace = n.oid " + "where p.oid < 16384 and p.proisagg=false and p.proiswindow=false", + key_desc='函数%s的定义', + complete_sql_desc='通过pg_get_functiondef()来检查一般函数的定义' + ) + ] + ), + "pg_catalog.pg_class": Meta( + Category.TABLE, + '存储了所有表、索引、视图、序列等的基础信息。', + True, + [ + ContentRulesMeta( + complete_sql="select 'count(*)', count(*) from pg_class where oid < 16384", + key_desc='数量统计方法%s', + complete_sql_desc='pg_class内oid < 16384(包括表、索引、视图、序列等)总数量' + ), + ContentRulesMeta( + key="format('%s.%s',(select nspname from pg_namespace n where n.oid = relnamespace), relname)", + key_desc='名为%s的表或索引或视图等', + filters=' oid < 10000 ', + ignore_col='relfilenode,relpages,reltuples,relallvisible,relfrozenxid,relfrozenxid64,relminmxid', + oid_col='oid,reltype,reloftype', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s.%s',(select nspname from pg_namespace n where n.oid = relnamespace), relname)", + key_desc='名为%s的表或索引或视图等', + filters=" 9999 < oid and oid < 16384 and relnamespace not in (99) ", + ignore_col='oid,relfilenode,relpages,reltuples,relallvisible,reltoastrelid,reltoastidxid,' + 'relfrozenxid,relacl,relfrozenxid64,relminmxid', + oid_col='relnamespace,reltype,reloftype,relowner,reltablespace', + accuracy=Accuracy.STRICT + ), + # 表的不需要使用pg_get_tabledef检查，会在第一步的校验系统表结构的时候校验。 + # 索引等会在对应的扩展系统表校验，此处仅校验视图即可 + ContentRulesMeta( + complete_sql="select format('def:%s.%s', n.nspname, c.relname), " + " md5(pg_get_viewdef(c.oid)) " + "from pg_class c left join pg_namespace n on c.relnamespace=n.oid " + "where c.oid < 16384 and " + " c.relkind in ('v') and " + " n.nspname not in ('pg_toast', 'snapshot') ", + key_desc='视图定义%s', + complete_sql_desc='pg_get_viewdef()来检查视图的定义' + ) + ] + ), + "pg_catalog.pg_authid": Meta( + Category.AUTHENTICATION, + "存储有关数据库认证标识符（角色）的信息", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的角色', + filters=' oid = 10 ', + ignore_col='rolname,rolpassword,rolvalidbegin,rolvaliduntil,rolrespool,rolpasswordext' + ), + + # 其余角色的各种属性理论都可以通过alter修改，无需校验 + ContentRulesMeta( + complete_sql='select oid, rolname from pg_authid where oid < 10000 and oid != 10', + key_desc='oid为%s的角色', + complete_sql_desc='校验角色的信息' + ), + ContentRulesMeta( + complete_sql='select rolname, rolname from pg_authid where 9999 < oid and oid < 16384', + key_desc='角色%s', + complete_sql_desc='校验角色的信息' + ) + ] + ), + "pg_catalog.pg_auth_members": Meta( + Category.AUTHENTICATION, + "存储显示角色之间的成员关系", + True, + [ + ContentRulesMeta( + filters=' roleid < 16384 ' + ) + ] + ), + "pg_catalog.pg_database": Meta( + Category.DATABASE, + "存储database信息", + True, + [ + ContentRulesMeta( + key='datname', + key_desc='数据库%s', + filters=' oid < 10000 ', + ignore_col='encoding,datcollate,datctype,datlastsysoid,datfrozenxid,datfrozenxid64,datminmxid' + ), + ContentRulesMeta( + key='datname', + key_desc='数据库%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid,encoding,datcollate,datctype,datlastsysoid,datfrozenxid,datfrozenxid64,datminmxid' + ) + ] + ), + "pg_catalog.pg_foreign_server": Meta( + Category.FDW, + "存储外表服务器", + True, + [ + ContentRulesMeta( + key='srvname', + key_desc='外表服务器%s', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + complete_sql="select srvname, " + " md5(fw.fdwname || fs.srvtype::text || fs.srvversion::text || fs.srvoptions::text) " + "from pg_foreign_server fs left join pg_foreign_data_wrapper fw on fs.srvfdw = fw.oid " + "where 9999 < fs.oid and fs.oid < 16384 ;", + key_desc='外表服务器%s', + complete_sql_desc='校验系统表pg_foreign_server的内容' + ) + ] + ), + "pg_catalog.pg_user_mapping": Meta( + Category.FDW, + "存储外表用户映射关系", + True, + [ + ContentRulesMeta( + filters=' umuser < 16384 and umserver < 16384 ' + ) + ] + ), + "pg_catalog.pg_foreign_data_wrapper": Meta( + Category.FDW, + "存外部数据包装器", + True, + [ + ContentRulesMeta( + key='fdwname', + key_desc='外部数据包装器%s', + ignore_col='fdwacl', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + complete_sql="select fdw.fdwname, format('hander(%s),validator(%s)', p1.proname, p2.proname) " + "from pg_foreign_data_wrapper fdw left join pg_proc p1 on fdwhandler = p1.oid " + " left join pg_proc p2 on fdwvalidator=p2.oid " + "where 9999 < fdw.oid and fdw.oid < 16384 ", + key_desc='外部数据包装器%s', + complete_sql_desc='校验表pg_foreign_data_wrapper的内容' + ) + ] + ), + "pg_catalog.pg_shdescription": Meta( + Category.DESCRIPTION, + "存储共享数据库对象的注释", + True, + [ + ContentRulesMeta( + filters=' objoid < 16384 and classoid < 16384 ', + oid_col='objoid,classoid', + accuracy=Accuracy.ALLOW_MORE + ) + ] + ), + "pg_catalog.pg_aggregate": Meta( + Category.FUNCTION, + "存储聚集函数", + True, + [ + ContentRulesMeta( + key='aggfnoid::oid', + key_desc='oid为%s的聚集函数', + filters=' aggfnoid < 10000 ' + ), + ContentRulesMeta( + key="format('%s %s %s %s', aggfnoid, aggtransfn, aggcollectfn, aggfinalfn)", + key_desc='函数名、三阶段为%s的聚集函数', + filters=' 9999 < aggfnoid and aggfnoid < 16384 ' + # aggfnoid是regproc类型，自动会按照字符串打印，无需做oid特殊处理 + ) + ] + ), + "pg_catalog.pg_am": Meta( + Category.AM, + "存储数据库索引相关相关的访问方法", + True, + [ + ContentRulesMeta( + key='amname', + key_desc='访问方法%s', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + key="amname", + key_desc='访问方法%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='amkeytype' + ) + ] + ), + "pg_catalog.pg_amop": Meta( + Category.AM, + "存储有关和访问方法操作符族关联的信息", + True, + [ + ContentRulesMeta( + key="format('%s-%s-%s-%s', amopfamily, amoplefttype, amoprighttype, amopstrategy)", + key_desc='family-ltype-rtype-stg为%s的操作符访问族', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + key="format('%s-%s-%s-%s', amopfamily, amoplefttype, amoprighttype, amopstrategy)", + key_desc='family-ltype-rtype-stg为%s的操作符访问族', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='amopfamily,amoplefttype,amoprighttype,amopopr,amopmethod,amopsortfamily' + ) + ] + ), + "pg_catalog.pg_amproc": Meta( + Category.AM, + "存储有关与访问方法操作符族相关联的支持过程的信息", + True, + [ + ContentRulesMeta( + key="format('%s-%s-%s-%s', amprocfamily, amproclefttype, amprocrighttype, amprocnum)", + key_desc='family-ltype-rtype-prn为%s的函数访问族', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + key="format('%s-%s-%s-%s', amprocfamily, amproclefttype, amprocrighttype, amprocnum)", + key_desc='family-ltype-rtype-prn为%s的函数访问族', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='amprocfamily,amproclefttype,amprocrighttype' + ) + ] + ), + "pg_catalog.pg_attrdef": Meta( + Category.TABLE, + "存储列的默认值", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid编号为%s的列默认值', + filters=' oid < 10000 ', + ignore_col='adbin_on_update,adbin' + ), + ContentRulesMeta( + key="format('%s-%s', (select relname from pg_class c where c.oid=adrelid), adnum)", + key_desc='表名-列号为%s的列默认值', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid,adbin_on_update,adbin', + oid_col='adrelid' + ) + ] + ), + "pg_catalog.pg_cast": Meta( + Category.TYPE, + "存储数据类型转换方式", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid编号为%s的数据类型转换方式', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + key="format('%s-%s', (select typname from pg_type t where t.oid=castsource), " + "(select typname from pg_type t where t.oid=casttarget))", + key_desc='左右类型为%s的转换方法', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='castsource,casttarget,castfunc,castowner', + accuracy=Accuracy.ALLOW_MORE + ) + ] + ), + "pg_catalog.pg_constraint": Meta( + Category.CONSTRAINT, + "存储表上的检查约束、主键和唯一约束", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid编号为%s的约束', + filters=' oid < 10000 ', + ignore_col='conbin' + ), + ContentRulesMeta( + key="conname", + key_desc='约束%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid,conpfeqop,conppeqop,conffeqop,conexclop,conbin', + oid_col='connamespace,conrelid,contypid,conindid,confrelid' + ), + ContentRulesMeta( + complete_sql="select format('def:%s', conname), md5(pg_get_constraintdef(oid)) " + "from pg_constraint " + "where oid < 16384 ", + key_desc='约束%s的定义', + complete_sql_desc='通过pg_get_constraintdef()函数检查oid < 16384的约束的定义' + ) + ] + ), + "pg_catalog.pg_conversion": Meta( + Category.LOCALE, + "存储编码转换信息", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的编码转换方式', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + key="format('%s.%s', (select nspname from pg_namespace n where n.oid=connamespace), conname)", + key_desc='编码转换方式%s', + filters=' 9999 < oid and oid < 16384 ', + oid_col='oid,connamespace,conowner,conforencoding,contoencoding,conproc' + ) + ] + ), + "pg_catalog.pg_depend": Meta( + Category.DEPEND, + "记录数据库非共享对象之间的依赖性关系", + True + # pg_depend的内容太多且场景太复杂，暂时无法总结规律并写出SQL。 + # 在直接安装时，很多内置对象直接写入系统表，进而未配置依赖关系，例如一些函数等，但通过升级脚本创建的则可以。 + # 同时夹杂着太多的1W~16384的内容 + # 考虑此系统表的含义，仅是一个依赖的属性，对应的实体对象会在其他系统表内校验，因此暂时不进行此系统表的校验 + ), + "pg_catalog.pg_description": Meta( + Category.DESCRIPTION, + "存储数据库对象的描述", + True, + [ + ContentRulesMeta( + key="format('%s-%s-%s', objoid, classoid, objsubid)", + key_desc='obj-class-subid为%s的描述', + filters=' objoid < 10000 and classoid < 10000 and objsubid < 10000 ', + oid_col='objoid,classoid' + ), + # 1W+比较难写，仅测试数量，问题不大。 + ContentRulesMeta( + complete_sql = "select format('%s-%s-%s', " + " (case when objoid < 9999 then objoid else 0 end), " + " (case when classoid < 9999 then classoid else 0 end), " + " (case when objsubid < 9999 then objsubid else 0 end) " + " ) as key," + " count(*) as val " + "from pg_description " + "where (9999 < objoid and objoid < 16384) or " + " (9999 < classoid and classoid < 16384) or " + " (9999 < objsubid and objsubid < 16384) " + "group by key;", + key_desc = '(9999,16384)的范围内的键值为%s(1W+显示0)的注释数量', + complete_sql_desc = '统计(9999,16384)的范围内的(oid在1W+时显示0)的注释数量' + ) + ] + ), + "pg_catalog.pg_index": Meta( + Category.INDEX, + "存储索引的信息", + True, + [ + ContentRulesMeta( + key='indexrelid', + key_desc='indexrelid为%s的索引', + filters=' indexrelid < 10000 ', + ignore_col='indpred,indexprs' + ), + ContentRulesMeta( + key="format('%s-%s'," + " (select relname from pg_class c where c.oid=indrelid)," + " (select relname from pg_class c where c.oid=indexrelid)" + ")", + key_desc='表名-索引名为%s的索引', + filters=' 9999 < indexrelid and indexrelid < 16384 ', + ignore_col='indcollation,indclass', + oid_col='indexrelid,indrelid' + ), + ContentRulesMeta( + complete_sql="select format('def:%s(%s)', ct.relname, ci.relname), " + " md5(pg_get_indexdef(indexrelid)) " + "from pg_index i left join pg_class ct on i.indrelid = ct.oid " + " left join pg_class ci on i.indexrelid = ci.oid " + "where indexrelid < 16384 and ct.relnamespace not in (99, 4989)", + key_desc='索引定义%s', + complete_sql_desc='通过函数pg_get_indexdef()校验索引的定义（排除pg_toast、snapshot的索引，因为有会变OID名字等原因）' + ) + ] + ), + "pg_catalog.pg_inherits": Meta( + Category.TABLE, + "记录关于表继承层次的信息", + True, + [ + ContentRulesMeta( + key="format('%s-%s', (case when inhrelid < 10000 then inhrelid else 0 end), inhseqno)", + key_desc='表-继承号为%s的继承关系', + oid_col='inhrelid,inhparent', + accuracy=Accuracy.ALLOW_MORE + ) + ] + ), + "pg_catalog.pg_language": Meta( + Category.LANGUAGE, + "存储编程语言，用户可以用这些语言或接口写函数或者存储过程", + True, + [ + ContentRulesMeta( + key='lanname', + key_desc='oid为%s的语言', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + key='lanname', + key_desc='名称为%s的语言', + filters=' 9999 < oid and oid < 10000 ', + oid_col='oid,lanplcallfoid,laninline,lanvalidator', + ) + ] + ), + "pg_catalog.pg_largeobject": Meta( + Category.LOB, + "保存那些标记着'大对象'的数据", + True, + [ + ContentRulesMeta( + key="format('%s %s', loid, pageno)", + key_desc='loid-pageno为%s的大对象', + accuracy=Accuracy.ALLOW_MORE + ) + ] + ), + "pg_catalog.pg_namespace": Meta( + Category.SCHEMA, + "存储名称空间，即schema相关的信息", + True, + [ + ContentRulesMeta( + key='nspname', + key_desc='名称为%s的schema', + filters=' oid < 10000 ', + ignore_col='nsptimeline,in_redistribution,nspblockchain,nspcollation' + ), + ContentRulesMeta( + key='nspname', + key_desc='名称为%s的schema', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='nsptimeline,in_redistribution,nspblockchain,nspcollation', + oid_col='oid,nspowner' + ) + ] + ), + "pg_catalog.pg_opclass": Meta( + Category.OPERATOR, + "存储索引访问方法操作符类", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的操作符类', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + key="format('%s.%s(of am %s)'," + " opcnamespace," + " opcname," + " (select amname from pg_am a where a.oid=opcmethod)" + ")", + key_desc='操作符类%s', + filters=' 9999 < oid and oid < 16384 ', + oid_col='oid,opcmethod,opcnamespace,opcowner,opcfamily,opcintype,opckeytype' + ) + ] + ), + "pg_catalog.pg_operator": Meta( + Category.OPERATOR, + "存储有关操作符的信息", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的操作符', + filters=' oid < 10000 ', + ignore_col='oprnamespace,oprowner,oprleft,oprright,oprresult,oprcom,oprnegate' + ), + ContentRulesMeta( + key="format('%s.(%s %s %s)'," + " (select nspname from pg_namespace where oid=oprnamespace), " + " (select typname from pg_type t where t.oid=oprleft), " + " oprname, " + " (select typname from pg_type t where t.oid=oprright) " + ")", + key_desc='操作符 %s', + filters=' 9999 < oid and oid < 16384 ', + oid_col='oid,oprnamespace,oprowner,oprleft,oprright,oprresult,oprcom,oprnegate' + ) + ] + ), + "pg_catalog.pg_rewrite": Meta( + Category.RULE, + "存储为表和视图定义的重写规则", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的操作符', + filters=' oid < 10000 ', + ignore_col='ev_action,ev_qual' + ), + ContentRulesMeta( + key="format('%s(on %s)', rulename, (select relname from pg_class c where c.oid=ev_class))", + key_desc='重写规则%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='ev_qual,ev_action', + oid_col='oid,ev_class' + ), + ContentRulesMeta( + complete_sql="select format('def:%s.%s(%s)', n.nspname, c.relname, r.rulename), " + " md5(pg_get_ruledef(r.oid)) " + "from pg_rewrite r left join pg_class c on r.ev_class = c.oid " + " left join pg_namespace n on c.relnamespace = n.oid " + "where r.oid < 16384 and " + " n.nspname not in ('pg_toast', 'snapshot');", + key_desc='规则定义%s', + complete_sql_desc='使用函数pg_get_ruledef校验重写规则的定义' + ) + ] + ), + "pg_catalog.pg_statistic": Meta( + Category.OPTIMIZER, + "存储用于优化器行数估算的统计信息", + True + # 不用校验内容 + ), + "pg_catalog.pg_trigger": Meta( + Category.TRIGGER, + "存储触发器信息", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的触发器', + filters=' oid < 10000 ', + ignore_col='tgtime,tgqual' + ), + ContentRulesMeta( + key="format('%s(on %s)', tgname, (select relname from pg_class c where c.oid=tgrelid))", + key_desc='触发器%s', + filters=' 9999 < oid and oid < 16384 ', + oid_col='oid,tgrelid', + ignore_col='tgtime,tgqual' + ), + ContentRulesMeta( + complete_sql="select format('def:%s(%s)', c.relname, t.tgname), " + " md5(pg_get_triggerdef(t.oid)) " + "from pg_trigger t left join pg_class c on t.tgrelid=c.oid " + "where t.oid < 16384 ", + key_desc='触发器定义%s', + complete_sql_desc='通过函数pg_get_triggerdef()检查触发器的定义' + ) + ] + ), + "pg_catalog.pg_opfamily": Meta( + Category.OPERATOR, + "存储操作符族", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的触发器', + filters=' oid < 10000 ' + ), + ContentRulesMeta( + key="format('%s.%s(for %s)', " + " (select nspname from pg_namespace n where n.oid=opfnamespace), " + " opfname, " + " (select amname from pg_am a where a.oid=opfmethod) " + ")", + key_desc='操作符族%s', + filters=' 9999 < oid and oid < 16384 ', + oid_col='opfmethod,opfnamespace,opfowner' + ) + ] + ), + "pg_catalog.pg_db_role_setting": Meta( + Category.AUTHENTICATION, + "存储数据库运行时每个角色与数据绑定的配置项的默认值", + True, + [ + ContentRulesMeta( + key="format('%s(of db %s)', " + " setrole, " + " (select datname from pg_database d where d.oid=setdatabase) " + ")", + key_desc='用户%s的配置默认值', + filters=' setdatabase < 16384 and setrole < 16384 ' + ) + ] + ), + "pg_catalog.pg_largeobject_metadata": Meta( + Category.LOB, + "存储与大数据相关的元数据", + True, + [ + ContentRulesMeta( + key="lomowner", + key_desc='oid为%s的大对象元数据', + filters=' oid < 16384 ', + accuracy=Accuracy.ALLOW_MORE + ) + ] + ), + "pg_catalog.pg_extension": Meta( + Category.EXTENSION, + "存储关于所安装扩展的信息", + True, + [ + ContentRulesMeta( + key="extname", + key_desc='扩展%s', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="extname", + key_desc='扩展%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='extconfig', + oid_col='oid,extowner,extnamespace', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_foreign_table": Meta( + Category.TABLE, + "存储外部表的辅助信息", + True, + [ + ContentRulesMeta( + key="ftrelid", + key_desc='oid为%s的外表', + filters=' ftrelid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="(select relname from pg_class c where c.oid=ftrelid)", + key_desc='外表%s', + filters=' 9999 < ftrelid and ftrelid < 10000 ', + oid_col='ftrelid,ftserver', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_statistic_ext": Meta( + Category.OPTIMIZER, + "存储有关该数据库中表的扩展统计数据，包括多列统计数据和表达式统计数据", + True + # 无需校验内容 + ), + "pg_catalog.pg_rlspolicy": Meta( + Category.PRIVILEGE, + "存储行级访问控制策略", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的行级访问控制策略', + filters=' oid < 10000 ', + ignore_col='polqual', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s(on %s)', polname, (select relname from pg_class c where c.oid=polrelid))", + key_desc='行级访问控制策略%s', + filters=' 9999 < oid and oid < 10000 ', + ignore_col='polqual', + oid_col='oid,polrelid', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_resource_pool": Meta( + Category.WLM, + "提供了数据库资源池的信息", + True, + [ + ContentRulesMeta( + key='oid', + key_desc='oid为%s的资源池', + filters=' oid < 10000 ', + ignore_col='mem_percent,cpu_affinity,control_group,active_statements,max_dop,memory_limit,parentid,' + 'io_limits,io_priority,nodegroup,is_foreign,max_worker', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="respool_name", + key_desc='资源池respool_name', + filters=' 9999 < oid and oid < 10000 ', + ignore_col='mem_percent,cpu_affinity,control_group,active_statements,max_dop,memory_limit,parentid,' + 'io_limits,io_priority,nodegroup,is_foreign,max_worker', + oid_col='oid', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_workload_group": Meta( + Category.WLM, + "提供了数据库负载组的信息", + True, + [ + ContentRulesMeta( + key='workload_gpname', + key_desc='负载组%s', + filters=' oid < 16384 ', + ignore_col='act_statements', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_collation": Meta( + Category.LOCALE, + "存储了排序规则，本质上从一个SQL名称映射到操作系统本地类别", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的排序规则', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ) + # initdb阶段的不需要校验，因为这部分的是gs_initdb读取了环境locale -a之后配置到pg_collation的，因此也算是纯用户数据。 + ] + ), + "pg_catalog.pg_auth_history": Meta( + Category.AUTHENTICATION, + "存储了角色的认证历史", + True + # 无需校验内容 + ), + "pg_catalog.pg_user_status": Meta( + Category.AUTHENTICATION, + "存储了访问数据库用户的状态", + True, + # 无需校验内容 + ), + "pg_catalog.pg_app_workloadgroup_mapping": Meta( + Category.WLM, + "存储了数据库负载映射组的信息", + True, + [ + ContentRulesMeta( + key="appname", + key_desc='负载映射组%s', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_event_trigger": Meta( + Category.TRIGGER, + "存储每个事件触发器的信息", + True, + [ + ContentRulesMeta( + key="evtname", + key_desc='事件触发器%s', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_enum": Meta( + Category.TYPE, + "存储枚举类型相关信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的枚举值', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s(%s)'," + " (select typname from pg_type t where t.oid=enumtypid)," + " enumlabel" + ")", + key_desc='枚举类型值%s', + filters=' 9999 < oid and oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_set": Meta( + Category.TYPE, + "存储集合数据类型定义的元数据", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的集合值', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s(%s)'," + " (select typname from pg_type t where t.oid=settypid)," + " setlabel" + ")", + key_desc='集合类型值%s', + filters=' 9999 < oid and oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_range": Meta( + Category.TYPE, + "存储关于范围类型的信息", + True, + [ + ContentRulesMeta( + key="rngtypid", + key_desc='rngtypid为%s的范围类型', + filters=' rngtypid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_synonym": Meta( + Category.SYNONYM, + "存储同义词", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的同义词', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s.%s', (select nspname from pg_namespace n where n.oid=synnamespace), synname)", + key_desc='同义词%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='synnamespace,synowner', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_shseclabel": Meta( + Category.SECURITY, + "存储共享数据对象上的安全标签", + True + # 安全标签用户可以自行设置或取消，无需校验内容 + ), + "pg_catalog.pg_seclabel": Meta( + Category.SECURITY, + "存储数据对象上的安全标签", + True + # 安全标签用户可以自行设置或取消，无需校验内容 + ), + "pg_catalog.pg_ts_dict": Meta( + Category.TS, + "存储定义文本搜索字典的记录", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的文本搜索字典', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s.%s', (select nspname from pg_namespace n where n.oid=dictnamespace), dictname)", + key_desc='文本搜索字典%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='dictnamespace,dictowner,dicttemplate', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_ts_parser": Meta( + Category.TS, + "包含定义文本解析器的记录", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的文本解析器', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s.%s', (select nspname from pg_namespace n where n.oid=prsnamespace), prsname)", + key_desc='文本解析器%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='prsnamespace', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_ts_config": Meta( + Category.TS, + "包含表示文本搜索配置的记录", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的文本搜索配置', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s.%s', (select nspname from pg_namespace n where n.oid=cfgnamespace), cfgname)", + key_desc='文本搜索配置%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='cfgnamespace,cfgowner,cfgparser', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_ts_config_map": Meta( + Category.TS, + "包含为每个文本搜索配置的解析器的每个输出符号类型，显示哪个文本搜索字典应该被咨询、以什么顺序搜索的记录", + True, + [ + ContentRulesMeta( + key="format('%s %s %s', mapcfg, maptokentype, mapseqno)", + key_desc='键值为%s的行', + filters=' mapcfg < 10000 ', + accuracy=Accuracy.STRICT + ), + # 1W+的在common rule内 + ContentRulesMeta( + complete_sql="select format('%s %s %s', c.cfgname, cm.maptokentype, mapseqno)," + " md5(d.dictname)" + "from pg_ts_config_map cm left join pg_ts_config c on cm.mapcfg = c.oid " + " left join pg_ts_dict d on cm.mapdict = d.oid " + "where 9999 < cm.mapcfg and cm.mapcfg < 16384;", + key_desc='(9999,16384)的mapcfg范围内的键值为%s的文本搜索配置', + complete_sql_desc='(9999,16384)的mapcfg范围内的所有文本搜索配置项目', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_ts_template": Meta( + Category.TS, + "存储文本搜索模板", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的文本搜索模板', + filters=' oid < 10000 ', + accuracy=Accuracy.STRICT + ), + ContentRulesMeta( + key="format('%s.%s', (select nspname from pg_namespace n where n.oid=tmplnamespace), tmplname)", + key_desc='文本搜索模板%s', + filters=' 9999 < oid and oid < 16384 ', + ignore_col='oid', + oid_col='tmplnamespace', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_model_warehouse": Meta( + Category.AI, + "存储AI引擎训练模型，其中包含模型，训练过程的详细描述", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的训练模型', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_extension_data_source": Meta( + Category.EXTENSION, + "存储外部数据源对象的信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的外部数据源对象的信息', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_directory": Meta( + Category.DIRECTORY, + "保存用户添加的directory对象", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的文件夹', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_db_privilege": Meta( + Category.PRIVILEGE, + "记录ANY权限的授予情况，每条记录对应一条授权信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的授权记录', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_obsscaninfo": Meta( + Category.OBS, + "OBS扫描信息", + True + # 无需校验内容 + ), + "pg_catalog.gs_obsscaninfo": Meta( + Category.OBS, + "OBS扫描信息", + True + # 无需校验内容 + ), + "pg_catalog.gs_global_chain": Meta( + Category.SECURITY, + "记录用户对防篡改用户表的修改操作信息，每条记录对应一次表级修改操作", + True + # 无需校验内容 + ), + "pg_catalog.pg_subscription": Meta( + Category.PUB_SUB, + "存储所有现有的逻辑复制订阅", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的复制订阅', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_publication": Meta( + Category.PUB_SUB, + "存储所有现有的逻辑复制发布", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的复制发布', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_publication_rel": Meta( + Category.PUB_SUB, + "存储当前数据库中的表和publication之间的映射，这是一种多对多映射", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的发布表映射', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_replication_origin": Meta( + Category.REPLICATION, + "包含所有已创建的复制源，该表为全局共享表，即在每个节点上只有一份pg_replication_origin，而不是每个数据库一份", + True, + [ + ContentRulesMeta( + key="roident", + key_desc='roident为%s的复制源', + filters=' roident < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_subscription_rel": Meta( + Category.PUB_SUB, + "包含每个订阅中每个被复制表的状态，是多对多的映射关系", + True, + [ + ContentRulesMeta( + key="format('%s->%s', srsubid, srrelid)", + key_desc='从复制源%s表的赋值关系', + filters=' srsubid < 16384 or srrelid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_package": Meta( + Category.PACKAGE, + '存储PACKAGE内的信息', + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的package', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_recyclebin": Meta( + Category.RECYCLE_BIN, + '描述了回收站对象的详细信息', + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的垃圾桶', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_txn_snapshot": Meta( + Category.TRANSACTION, + '"时间戳-CSN"映射表，周期性采样，并维护适当的时间范围，用于估算范围内的时间戳对应的CSN值', + True + # 无需校验内容 + ), + "pg_catalog.gs_uid": Meta( + Category.TABLE, + '存储了数据库中使用hasuids属性表的唯一标识元信息', + True, + [ + ContentRulesMeta( + key="relid", + key_desc='relid为%s的uid', + filters=' relid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pgxc_class": Meta( + Category.DISTRIBUTE, + '存储每张表的复制或分布信息(openGauss此表无意义)', + True, + [ + ContentRulesMeta( + key="pcrelid", + key_desc='pcrelid为%s的表', + filters=' pcrelid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pgxc_group": Meta( + Category.DISTRIBUTE, + '存储集群节点组信息', + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的节点组', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pgxc_node": Meta( + Category.DISTRIBUTE, + "存储集群节点信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的节点', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_partition": Meta( + Category.TABLE, + "存储数据库内所有分区表（partitioned table）、分区（table partition）、分区上toast表和分区索引（index partition）四类对象的信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的分区或分区索引', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_job": Meta( + Category.JOB, + "存储用户创建的定时任务的任务详细信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的定时任务', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_job_proc": Meta( + Category.JOB, + "对应PG_JOB表中每个任务的作业内容（包括：PL/SQL代码块、匿名块）", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的定时任务作业内容', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pg_object": Meta( + Category.OBJECT, + "存储限定类型对象（普通表、索引、序列、视图、存储过程和函数）的创建用户、创建时间和最后修改时间", + True + # 此表无法记录builtin与initdb阶段的对象，因此无需校验内容。 + ), + "pg_catalog.pg_hashbucket": Meta( + Category.TABLE, + "存储hash bucket表的信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的bucket', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.streaming_stream": Meta( + Category.DISTRIBUTE, + "存储流相关的信息(openGauss内此表无意义)", + True + # 无需校验内容 + ), + "pg_catalog.streaming_cont_query": Meta( + Category.DISTRIBUTE, + "存储流相关的信息(openGauss内此表无意义)", + True + # 无需校验内容 + ), + "pg_catalog.streaming_reaper_status": Meta( + Category.DISTRIBUTE, + "存储流相关的信息(openGauss内此表无意义)", + True + # 无需校验内容 + ), + "pg_catalog.gs_job_attribute": Meta( + Category.JOB, + "存储定时任务的相关属性", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的定时任务属性', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.pgxc_slice": Meta( + Category.DISTRIBUTE, + "针对range范围分布和list分布创建的系统表，用来记录分布具体信息(openGauss内此表无意义)", + True + # 无需校验内容 + ), + "pg_catalog.gs_job_argument": Meta( + Category.JOB, + "存储定时任务的相关内容", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的定时任务内容', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_sql_patch": Meta( + Category.OPTIMIZER, + "存储所有SQL_PATCH的状态信息", + True + # 纯用户行为数据，无需校验内容。 + ), + "pg_catalog.gs_global_config": Meta( + Category.GUC, + "记录了数据库实例初始化时，用户指定的参数值", + True + # 纯用户行为数据，无需校验内容。 + ), + "pg_catalog.gs_policy_label": Meta( + Category.WLM, + "记录资源标签配置信息，一个资源标签对应着一条或多条记录，每条记录标记了数据库资源所属的资源标签", + True + # 纯用户行为数据，无需校验内容。 + ), + "pg_catalog.gs_auditing_policy": Meta( + Category.AUTHENTICATION, + "记录统一审计的主体信息，每条记录对应一个设计策略", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的审计策略', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_auditing_policy_access": Meta( + Category.AUTHENTICATION, + "记录与DML数据库相关操作的统一审计信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的信息', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_auditing_policy_privileges": Meta( + Category.AUTHENTICATION, + "记录统一审计DDL数据库相关操作信息，每条记录对应一个设计策略", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的设计策略', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_asp": Meta( + Category.DFX, + "存储被持久化的ACTIVE SESSION PROFILE样本", + True + # 不需要校验内容 + ), + "pg_catalog.gs_auditing_policy_filters": Meta( + Category.AUTHENTICATION, + "记录统一审计相关的过滤策略相关信息，每条记录对应一个设计策略", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的过滤策略', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_masking_policy": Meta( + Category.SECURITY, + "记录动态数据脱敏策略的主体信息，每条记录对应一个脱敏策略", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的动态数据脱敏策略', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_masking_policy_filters": Meta( + Category.SECURITY, + "记录动态数据脱敏策略对应的用户过滤条件，当用户条件满足FILTER条件时，对应的脱敏策略才会生效", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的用户过滤条件', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_masking_policy_actions": Meta( + Category.SECURITY, + "记录动态数据脱敏策略中相应的脱敏策略包含的脱敏行为，一个脱敏策略对应着该表的一行或多行记录", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的脱敏行为', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_encrypted_columns": Meta( + Category.SECURITY, + "记录密态等值特性中表的加密列相关信息，每条记录对应一条加密列信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的加密列', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_client_global_keys": Meta( + Category.SECURITY, + "记录密态等值特性中客户端加密主密钥相关信息，每条记录对应一个客户端加密主密钥", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的加密主密钥', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_column_keys": Meta( + Category.SECURITY, + "记录密态等值特性中列加密密钥相关信息，每条记录对应一个列加密密钥", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的列加密密钥', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_client_global_keys_args": Meta( + Category.SECURITY, + "记录密态等值特性中客户端加密主密钥相关元数据信息，每条记录对应客户端加密主密钥的一个键值对信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的加密主密钥键值对信息', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_column_keys_args": Meta( + Category.SECURITY, + "记录密态等值特性中客户端加密主密钥相关元数据信息，每条记录对应客户端加密主密钥的一个键值对信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的键值对信息', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_encrypted_proc": Meta( + Category.SECURITY, + "提供了密态函数/存储过程函数参数、返回值的原始数据类型，加密列等信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的加密信息', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_matview": Meta( + Category.VIEW, + "提供了关于数据库中每一个物化视图的信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的物化视图条目', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_matview_dependency": Meta( + Category.VIEW, + "提供了关于数据库中每一个增量物化视图、基表和mlog表的关联信息", + True, + [ + ContentRulesMeta( + key="oid", + key_desc='oid为%s的增量物化视图信息', + filters=' oid < 16384 ', + accuracy=Accuracy.STRICT + ) + ] + ), + "pg_catalog.gs_opt_model": Meta( + Category.AI, + "是启用AiEngine执行计划时间预测功能时的数据表，记录机器学习模型的配置、训练结果、功能、对应系统函数、训练历史等相关信息", + True + # 无需校验内容 + ), + "pg_catalog.gs_wlm_user_resource_history": Meta( + Category.WLM, + "存储与用户使用资源相关的信息", + True + # 无需校验内容 + ), + "pg_catalog.gs_wlm_instance_history": Meta( + Category.WLM, + "存储与实例（数据库主节点或数据库节点）相关的资源使用相关信息", + True + # 无需校验内容 + ), + "pg_catalog.gs_wlm_session_query_info_all": Meta( + Category.WLM, + "显示当前数据库实例执行作业结束后的负载管理记录", + True + # 无需校验内容 + ), + "pg_catalog.gs_wlm_operator_info": Meta( + Category.WLM, + "显示执行作业结束后的算子相关的记录", + True + # 无需校验内容 + ), + "pg_catalog.gs_wlm_plan_operator_info": Meta( + Category.WLM, + "显示执行作业结束后计划算子级的相关的记录", + True + # 无需校验内容 + ), + "pg_catalog.gs_wlm_plan_encoding_table": Meta( + Category.WLM, + "显示计划算子级的编码信息", + True + # 无需校验内容 + ), + "pg_catalog.gs_wlm_ec_operator_info": Meta( + Category.WLM, + "存储执行EC（Extension Connector）作业结束后的算子相关的记录", + True + # 无需校验内容 + ), + "pg_catalog.plan_table_data": Meta( + Category.OPTIMIZER, + "存储了用户通过执行EXPLAIN PLAN收集到的计划信息", + True + # 无需校验内容 + ), + "pg_catalog.statement_history": Meta( + Category.DFX, + "存储SQL语句的性能诊断信息", + True + # 不需要校验内容 + ), + "dbe_pldeveloper.gs_source": Meta( + Category.FUNCTION, + "记录PLPGSQL对象（存储过程、函数、包、包体）编译相关信息", + True + # 此表文档注明了只记录用户自定义数据，因此不需要校验内容 + + ), + "dbe_pldeveloper.gs_errors": Meta( + Category.FUNCTION, + "用于记录PLPGSQL对象（存储过程、函数、包、包体）编译过程中遇到的报错信息", + True + # 此表与gs_source一类，因此不需要校验内容 + ), + "information_schema.sql_features": Meta( + Category.INFO_SCHEMA, + "用于记录一些sql特性相关信息", + True, + ), + "information_schema.sql_implementation_info": Meta( + Category.INFO_SCHEMA, + "用于记录一些sql特性相关信息", + True, + ), + "information_schema.sql_languages": Meta( + Category.INFO_SCHEMA, + "用于记录一些语言特性相关信息", + True, + ), + "information_schema.sql_packages": Meta( + Category.INFO_SCHEMA, + "用于记录一些package特性相关信息", + True, + ), + "information_schema.sql_parts": Meta( + Category.INFO_SCHEMA, + "用于记录一些sql parts特性相关信息", + True, + ), + "information_schema.sql_sizing": Meta( + Category.INFO_SCHEMA, + "用于记录一些sql sizing特性相关信息", + True, + ), + "information_schema.sql_sizing_profiles": Meta( + Category.INFO_SCHEMA, + "用于记录一些sql profiles特性相关信息", + True, + ), + "db4ai.snapshot": Meta( + Category.AI, + "记录当前用户通过特性DB4AI.SNAPSHOT存储的快照", + True + # 无需校验内容 + ), + "snapshot.tables_snap_timestamp": Meta( + Category.WDR, + "记录所有存储的WDR snapshot中数据库、表对象、以及数据采集的开始和结束时间", + False + ), + "snapshot.snapshot": Meta( + Category.WDR, + "存储的WDR快照数据的索引信息、开始时间和结束时间", + False + ), + "snapshot.snap_global_os_runtime": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_os_threads": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_instance_time": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_workload_sql_count": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_workload_sql_elapse_time": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_workload_transaction": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_workload_transaction": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_thread_wait_status": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_memory_node_detail": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False, + ), + "snapshot.snap_global_shared_memory_detail": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_stat_db_cu": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_stat_database": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_stat_database": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_stat_database_conflicts": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_stat_database_conflicts": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_stat_bad_block": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_stat_bad_block": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_file_redo_iostat": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_file_redo_iostat": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_rel_iostat": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_rel_iostat": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_file_iostat": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_file_iostat": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_replication_slots": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_bgwriter_stat": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_replication_stat": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_transactions_running_xacts": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_transactions_running_xacts": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_transactions_prepared_xacts": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_transactions_prepared_xacts": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_statement": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_statement_count": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_statement_count": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_config_settings": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_wait_events": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_user_login": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_ckpt_status": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_double_write_status": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_pagewriter_status": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_redo_status": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_rto_status": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_recovery_status": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_threadpool_status": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_statement_responsetime_percentile": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_statio_all_indexes": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_statio_all_indexes": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_statio_all_sequences": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_statio_all_sequences": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_statio_all_tables": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_statio_all_tables": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_stat_all_indexes": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_stat_all_indexes": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_stat_user_functions": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_stat_user_functions": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_stat_all_tables": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_summary_stat_all_tables": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_class_vital_info": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ), + "snapshot.snap_global_record_reset_time": Meta( + Category.WDR, + "存储用于生成WDR报告的相关数据", + False + ) +} + + +def filter_uncertain_metas(): + uncertain_metas = [] + for key, meta in META.items(): + if not meta.certain: + uncertain_metas.append(key) + return uncertain_metas + diff --git a/script/upgrade_checker/rules/rule.py b/script/upgrade_checker/rules/rule.py new file mode 100644 index 00000000..f1aa9b30 --- /dev/null +++ b/script/upgrade_checker/rules/rule.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +from enum import Enum +from upgrade_checker.opengauss import og +from upgrade_checker.log import logger +from upgrade_checker.rules.category import Category +from upgrade_checker.rules.meta import Accuracy + + +class Rule(object): + """ + 单条检测SQL。 + 所有的检测sql都通过这个结构进行整理、检测、分析。 + 通过此数据结构来进行检测SQL的执行、结果的整理，检测内容、结果描述信息的整理生成。 + """ + + def __init__(self, sql, sql_desc="详见sql", key_desc="内容为%s的行", accuracy=Accuracy.STRICT, + callback=None): + """ + + :param sql: 校验sql + :param sql_desc: 校验sql的描述，结尾不要带句号等标点符号 + :param key_desc: 键值的描述，需要带一个%s用于格式匹配，结尾不要带句号等标点符号 + :param accuracy: 校验精度 + :param callback: 回调函数，用于对结果做一些处理 + """ + self.sql = sql + self.accuracy = accuracy + self.sql_desc = sql_desc + self.key_desc = key_desc + self.callback = callback + self.result = {} # sql res key, val + + def run(self): + """ + 执行检查、重新执行检察。 + :return: + """ + qres = og.query(self.sql) + for row in qres.data: + if self.result.get(row[0]) is not None: + logger.warning('在规则%s的查询执行中，发现重复键值%s' % (self.sql, row[0])) + + if qres.col_count() == 1: + self.result[row[0]] = row[0] + else: + assert qres.col_count() == 2 + self.result[row[0]] = row[1] + + if self.callback is not None: + self.callback.run(self) + + logger.debug('规则查询完成，共得到%d条结果，规则内容：%s' % (len(self.result), self.sql)) + return self.result + + def simplify(self): + """ + + :return: + """ + return { + 'sql': self.sql, + 'accuracy': self.accuracy.name, + 'sql_desc': self.sql_desc, + 'key_desc': self.key_desc, + 'result': self.result + } + + def analyze(self, expect): + """ + + :param expect: + :return: + """ + return Conclusion(self, expect) + + +class ConclusionState(Enum): + SUCCESS = 0 + FAILED = 1 + IGNORE = 2 + + +class Conclusion(object): + """ + 表示一个rule的分析结论。 + """ + def __init__(self, rule, expect): + """ + + :param rule: + :param expect: + """ + assert expect is None or rule.sql == expect['sql'] + self.rule = rule + self.expect = expect + + self.details = [] + self.warnings = [] + + self._analyze(rule, expect) + + def _analyze_one_row(self, key, e_key, row_desc, val, e_val, accuracy): + """ + + :param key: + :param e_key: + :param row_desc: + :param val: + :param e_val: + :param accuracy: + :return: + """ + detail = { + 'state': ConclusionState.SUCCESS, + 'summary': '成功' + } + if key is None: + detail['state'] = ConclusionState.FAILED + detail['summary'] = '%s 缺失' % row_desc + elif e_key is None: + if accuracy == Accuracy.ALLOW_MORE: + detail['state'] = ConclusionState.SUCCESS + detail['summary'] = '%s 允许冗余，校验成功' % row_desc + else: + detail['state'] = ConclusionState.FAILED + detail['summary'] = '%s 冗余' % row_desc + elif val != e_val: + detail['state'] = ConclusionState.FAILED + detail['summary'] = '%s 错误（预期%s 实际%s）' % (row_desc, e_val, val) + else: + detail['state'] = ConclusionState.SUCCESS + detail['summary'] = '%s 校验成功' % row_desc + + self.details.append(detail) + + def _analyze(self, rule, expect): + """ + + :param rule: + :param expect: + :return: + """ + # 是否能找到这条规则的预期 + if expect is None: + self.warnings = ['规则%s在校验地图内不存在。可能是由于vmap与工具不匹配、系统表结构升级不正确等原因导致，请手动排查。' % rule.sql] + return + + # 以rule为基础，对照expect进行判断 + for row_key, row_val in rule.result.items(): + row_desc = rule.key_desc % row_key + erow_val = expect['result'].get(row_key) + erow_key = row_key if erow_val is not None else None + self._analyze_one_row(row_key, erow_key, row_desc, row_val, erow_val, rule.accuracy) + + # 找到expect里有但实际没有的 + for erow_key, erow_val in expect['result'].items(): + row_val = rule.result.get(erow_key) + if row_val is not None: + continue + row_desc = expect['key_desc'] % erow_key + self._analyze_one_row(None, erow_key, row_desc, '', erow_val, rule.accuracy) + + +class StructRule(Rule): + """ + 系统表结构校验规则。 + """ + def __init__(self, rel_key, category, sql, sql_desc="详见sql", key_desc="内容为%s的行", + accuracy=Accuracy.STRICT, callback=None): + super(StructRule, self).__init__(sql, sql_desc, key_desc, accuracy, callback) + self.rel_key = rel_key + self.category = category + + def simplify(self): + data = super(StructRule, self).simplify() + data['rel_key'] = self.rel_key + data['category'] = self.category.name + return data + + def analyze(self, expect): + """ + + :param expect: + :return: + """ + conclusion = super(StructRule, self).analyze(expect) + if self.category == Category.UNKNOWN: + conclusion.warnings.append('发现未分类的系统表 {0}, 建议升级工具。'.format(self.rel_key)) + return conclusion + + +class ContentRule(Rule): + """ + 系统表内容校验规则。 + """ + def __init__(self, rel_key, category, sql, sql_desc="详见sql", key_desc="内容为%s的行", + accuracy=Accuracy.STRICT, callback=None): + super(ContentRule, self).__init__(sql, sql_desc, key_desc, accuracy, callback) + self.rel_key = rel_key + self.category = category + + def simplify(self): + data = super(ContentRule, self).simplify() + data['rel_key'] = self.rel_key + data['category'] = self.category.name + return data + + def analyze(self, expect): + """ + + :param expect: + :return: + """ + conclusion = super(ContentRule, self).analyze(expect) + return conclusion + + +class CommonRule(Rule): + """ + 常规通用校验规则。 + """ + def __init__(self, category, sql, sql_desc="详见sql", key_desc="内容为%s的行", + accuracy=Accuracy.STRICT, callback=None): + super(CommonRule, self).__init__(sql, sql_desc, key_desc, accuracy, callback) + self.category = category + + def simplify(self): + data = super(CommonRule, self).simplify() + data['category'] = self.category.name + return data + + def analyze(self, expect): + """ + + :param expect: + :return: + """ + conclusion = super(CommonRule, self).analyze(expect) + if self.category == Category.UNKNOWN: + conclusion.warnings.append('发现未分类的通用规则 {0}，请升级工具。'.format(self.sql)) + return conclusion + + +class RuleCallback(object): + def __init__(self): + self.res_keys = [] + self.exp_keys = [] + self.callback = [] + + def run(self, rule): + pass + + +COMMON_RULES = [ + # 暂时没有，放个假的临时替代。 + CommonRule(Category.TABLE, + "select 'test', 1", + '暂无规则描述', + '测试%s', + Accuracy.STRICT + ) +] + diff --git a/script/upgrade_checker/rules/rule_maker.py b/script/upgrade_checker/rules/rule_maker.py new file mode 100644 index 00000000..e5814ec2 --- /dev/null +++ b/script/upgrade_checker/rules/rule_maker.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# +""" +用于创建Rule的构造器 +""" + +import copy +from enum import Enum +from upgrade_checker.log import logger +from upgrade_checker.opengauss import og +from upgrade_checker.rules.meta import META, filter_uncertain_metas +from upgrade_checker.rules.rule import StructRule, ContentRule, COMMON_RULES + + +class RelKind(Enum): + UNKNOWN = '' + TABLE = 'r' + VIEW = 'v' + INDEX = 'i' + + +class PgClassItem(object): + @staticmethod + def construct_sql(): + """ + PgClassItem的构造SQL，将会通过这个SQL的查询结构，构造一个PgClassItem。 + :return: + """ + uncertain_metas = filter_uncertain_metas() + tables = [("'%s'" % tb) for tb in uncertain_metas] + tables = ', '.join(tables) + sql = """ + select c.oid, + c.relnamespace, + n.nspname, + c.relname, + c.relhasoids, + c.relkind, + ( select string_agg(a.attname, ',' order by a.attnum)::text + from pg_attribute a + where a.attrelid = c.oid and a.attnum > 0 and a.attisdropped != true + ) as columns, + n.nspname || '.' || c.relname as relkey + from pg_class c + left join pg_namespace n on c.relnamespace = n.oid + where c.relkind = 'r' and (c.oid < 16384 or relkey in (%s)); + """ % tables + return sql + + def __init__(self, meta1): + """ + 通过数据库查询到的结果和工具内的分类信息等，构造一个检查对象。 + :param meta1: 通过construct_sql到数据库查询来的数据。 + """ + self.oid = int(meta1[0]) + self.schema = int(meta1[1]) + self.schema_name = meta1[2] + self.name = meta1[3] + self.key = "%s.%s" % (self.schema_name, self.name) + self.has_oid = True if meta1[4] == 't' else False + self.kind = meta1[5] + self.columns = meta1[6].split(',') + if self.has_oid: + self.columns.append('oid') + + meta2 = META.get(self.key) + if meta2 is None: + meta2 = META["default.default"] + self.category = meta2.category + self.desc = meta2.desc + self.certain = meta2.certain + + logger.debug("pg_class item: %s 生成完成。" % self.key) + logger.debug(self.show_info()) + + def show_info(self): + return 'PgClassItem: {0}, {1}'.format(self.key, self.category) + + +class RuleMaker(object): + + @staticmethod + def process_oid_projection(columns): + """ + 将这一系列类型oid的列名加case when，大于1W时调整为0，并返回列表 + :param columns: 类型为oid的列。 + :return: + """ + if isinstance(columns, str): + columns = columns.split(",") + + projections = [] + for col in columns: + projections.append("(case when %s < 10000 then %s else 0 end)" % (col, col)) + return projections + + @staticmethod + def construct_text_concat_projection(columns): + """ + 将这些列，类型转换为text并进行链接 + :param columns: + :return: + """ + text_columns = ['({0}::text)'.format(col) for col in columns] + return ' || '.join(text_columns) + + @staticmethod + def construct_md5_projection(column): + """ + 传入MD5函数，构造这么一个投影 + :param column: + :return: + """ + return 'md5(' + column + ")" + + @staticmethod + def make_structure_rules(pci): + """ + 为一个系统表，构造一份校验基本表结构的方案。 + :param pci: + :return: + """ + rules = [] + # 通过函数获取综合信息。 + # pg_get_tabledef是不稳定的，当有多个索引时，索引的输出顺序根据缓存命中顺序而定，因此需要进行特殊处理。 + stable_proj = "select string_agg(x, e'\n' order by x collate \"C\")" \ + "from (select json_array_elements(" \ + " array_to_json(" \ + " string_to_array(" \ + " pg_get_tabledef('{0}'), e'\n')))::text" \ + " ) as a(x)".format(pci.key) + sql = "SELECT 'pg_get_tabledef({0})', md5(({1}))".format(pci.key, stable_proj) + sql_desc = "通过pg_get_tabledef()函数整体校验系统表%s的结构信息" % pci.name + key_desc = "系统表%s" + rules.append(StructRule(pci.key, pci.category, sql, sql_desc, key_desc)) + logger.debug('pic structure rule 生成完成。') + return rules + + @staticmethod + def make_content_rules(pci): + """ + 对一个系统表，构造一个校验内容的方案。 + :param pci: pg_class item + :return: + """ + rules = [] + + # 对于不一定存在的系统表，不校验内容。 + if not pci.certain or pci.oid > 16383: + return rules + + metas = META.get(pci.key) + if metas is None: + metas = META.get("default.default") + + for meta in metas.content_rules_meta: + # 如果提供了完整的SQL，则直接使用 + if meta.complete_sql is not None: + assert meta.complete_sql_desc is not None + rules.append( + ContentRule(pci.key, pci.category, meta.complete_sql, + meta.complete_sql_desc, meta.key_desc, meta.accuracy) + ) + continue + + # 生成检测列的投影。忽略需要忽略的列，oid类型的列处理后统一放在最前面。 + # 不可用set做删除操作，会导致顺序错乱，进而无法。 + val_columns = [] + should_ignore = meta.ignore_col + meta.oid_col + for col in pci.columns: + if col in should_ignore: + continue + val_columns.append(col) + + val_columns += RuleMaker.process_oid_projection(meta.oid_col) + val_project = RuleMaker.construct_text_concat_projection(val_columns) + val_project = RuleMaker.construct_md5_projection(val_project) + + # 处理运算键的投影, 如果没有写，则默认内容就当作键。 + key_project = meta.key if meta.key is not None else val_project + + # 合成一个sql + sql = "select {0}, {1} from {2} where {3}".format( + key_project, val_project, pci.key, meta.filters + ) + sql_desc = "校验系统表{0}内容，其功能为:{1}。校验忽略列({2})".format( + pci.key, pci.desc, ",".join(meta.ignore_col) + ) + + # 构造一个规则 + rules.append(ContentRule(pci.key, pci.category, sql, sql_desc, meta.key_desc, meta.accuracy)) + + logger.debug('pci content rule 生成完成。') + return rules + + @staticmethod + def make_common_rules(): + return copy.deepcopy(COMMON_RULES) + + @staticmethod + def make_rules(database): + """ + 连接数据库 + :param database: + :return: + """ + rules = [] + og.connect(database) + qres = og.query(PgClassItem.construct_sql()) + for row in qres: + pci = PgClassItem(row) + rules += RuleMaker.make_structure_rules(pci) + rules += RuleMaker.make_content_rules(pci) + rules += RuleMaker.make_common_rules() + return rules diff --git a/script/upgrade_checker/rules/vmap.py b/script/upgrade_checker/rules/vmap.py new file mode 100644 index 00000000..718b4029 --- /dev/null +++ b/script/upgrade_checker/rules/vmap.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +""" +vmap结构 +""" + +import json +import time +import os +from json import JSONDecodeError +from upgrade_checker.utils.version import UPGRADE_CHECKER_VERSION, FOR_OPENGAUSS +from upgrade_checker.utils.command import Download +from upgrade_checker.utils.exception import ShellExecException +from upgrade_checker.log import logger +from upgrade_checker.opengauss import og +from upgrade_checker.rules.rule import StructRule, ContentRule, CommonRule + + +class VMapHeader(object): + """ + vmap的头部信息 + """ + + @staticmethod + def check_availability(head): + """ + 校验一个vmap是否在当前工具内可用。 + """ + curr = VMapHeader() + if curr.vmap_version != head.vmap_version: + err = "vmap版本不一致，期望{0}, 实际{1}".format(curr.vmap_version, head.vmap_version) + logger.err(err) + if curr.db_version != head.db_version: + err = "vmap的数据库版本不一致。期望{0}, 实际{1}".format(curr.db_version, head.db_version) + logger.err(err) + + def __init__(self, src=None): + self.vmap_version = UPGRADE_CHECKER_VERSION + self.db_version = og.version + self.create_time = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime()) + + if src is not None: + self.vmap_version = src.get('vmap_version') + self.db_version = src.get('db_version') + self.create_time = src.get('create_time') + assert self.vmap_version is not None and \ + self.db_version is not None and \ + self.create_time is not None + + def __str__(self): + return 'version {0}, for openGauss {1}, {2}'.format( + self.vmap_version, + self.db_version, + self.create_time + ) + + def to_dict(self): + return { + 'vmap_version': self.vmap_version, + 'db_version': self.db_version, + 'create_time': self.create_time + } + + +class VerifyMap(object): + + @staticmethod + def standard_name(tool_version, db_version): + """ + :param tool_version: 工具版本 + :param db_version: openGauss 版本 + :return: 标准校验地图的名字 + """ + return "standard_meta_verify_map_{0}_{1}.vmap".format(tool_version, db_version) + + @staticmethod + def download_address(tool_version, db_version): + """ + :param tool_version: 工具版本 + :param db_version: openGauss 版本 + :return: 标准校验地图的下载位置 + """ + return "https://opengauss.obs.cn-south-1.myhuaweicloud.com/upgrade_checker/{0}".format( + VerifyMap.standard_name(tool_version, db_version)) + + @staticmethod + def prepare_vmap_file(directory): + """ + 准备或下载vmap文件, 检查可用操作在后续加载之后，通过vmap内记录的info信息进行。 + :param directory: 存放vmap的文件夹位置。 + :return: + """ + if og.version not in FOR_OPENGAUSS: + logger.err('暂不支持 openGauss %s 版本的校验' % og.version) + + standard_vmap_name = VerifyMap.standard_name(UPGRADE_CHECKER_VERSION, og.version) + + vmap_file = directory + '/' + standard_vmap_name + if os.access(vmap_file, os.F_OK): + logger.log('基准校验地图(%s)已存在。' % vmap_file) + else: + url = VerifyMap.download_address(UPGRADE_CHECKER_VERSION, og.version) + logger.info('开始下载基准校验地图: {0}'.format(url)) + try: + Download.wget(url, vmap_file) + logger.info('基准校验地图下载完成: {0}'.format(vmap_file)) + except ShellExecException as e: + msg = '基准校验地图下载失败。\n{0}'.format(e.__str__()) + logger.err(msg) + return vmap_file + + def __init__(self, file_path): + self.head = VMapHeader() + self._file_path = file_path + self._structure_map = {} # {rule.sql: rule.simplify(), ... } + self._content_map = {} + self._common_map = {} + + def __str__(self): + return 'Verify Map ({0}): {1}'.format(self.head.__str__(), self._file_path) + + def push(self, rule): + """ + 将规则加到vmap里 + """ + if self.get(rule.sql) is not None: + logger.err('正在尝试向vmap内插入重复rule.') + + if isinstance(rule, StructRule): + self._structure_map[rule.sql] = rule.simplify() + elif isinstance(rule, ContentRule): + self._content_map[rule.sql] = rule.simplify() + elif isinstance(rule, CommonRule): + self._common_map[rule.sql] = rule.simplify() + else: + assert False + + def get(self, sql): + return self._structure_map.get(sql) or \ + self._content_map.get(sql) or \ + self._common_map.get(sql) + + def load(self): + with open(self._file_path, 'r') as f: + try: + data = json.load(f) + self.head = VMapHeader(data[0]) + self._structure_map = data[1] + self._content_map = data[2] + self._common_map = data[3] + logger.log('verify map 加载成功：' + self._file_path) + except JSONDecodeError as e: + logger.err('文件格式内容错误，verify map 加载失败：' + self._file_path) + + return self + + def dump(self): + with open(self._file_path, 'w') as f: + data = [self.head.to_dict(), + self._structure_map, + self._content_map, + self._common_map] + json.dump(data, f) + logger.log('verify map 导出成功：' + self._file_path) + diff --git a/script/upgrade_checker/style/__init__.py b/script/upgrade_checker/style/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/upgrade_checker/style/markdown.py b/script/upgrade_checker/style/markdown.py new file mode 100644 index 00000000..0d2ffe5f --- /dev/null +++ b/script/upgrade_checker/style/markdown.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# +""" +MarkDown格式。 +定义了md格式的各种封装了的操作。 +""" + + +class MDUnorderedList(object): + """ + MD格式无序列表。样例格式如下： + - content1 + - content2 + """ + def __init__(self): + self.line_count = 0 + self.res = "" + + def append(self, content): + """ + + :param content: + :return: + """ + self.res += ('- ' + content + '\n') + self.line_count += 1 + + def serialize(self): + """ + + :return: + """ + if self.line_count == 0: + return '无\n\n' + + return self.res + '\n\n' + + +class MDTable(object): + """ + MD格式的表。样例格式如下: + + + + + ... + + + + + + ... + + ... + +

名称

schemaname

+ """ + + def __init__(self, title): + """ + + :param title: + """ + self.column_len = len(title) + if self.column_len == 0: + return + + self.thead = ' \n' + for col in title: + self.thead += '

{0}

\n'.format(col) + self.thead += ' \n' + + self.tbody = '' + + def append(self, row): + """ + 追加一行数据 + :param row: + :return: + """ + if self.column_len == 0: + return + self.tbody += ' \n' + for i in range(0, self.column_len): + data = row[i] if i < len(row) else ' - ' + self.tbody += '

{0}

\n'.format(data) + self.tbody += ' \n' + + def serialize(self): + """ + 序列化输出结果 + :return: + """ + if self.column_len == 0: + return '' + + if self.tbody == '': + self.append([]) + + return "\n" \ + " \n" \ + "{0}" \ + " \n" \ + " \n" \ + "{1}" \ + " \n" \ + "
".format(self.thead, self.tbody) + + +class MarkDown(object): + + @staticmethod + def title(level, title): + """ + 样例: # title + :param level: 等级 + :param title: 内容 + :return: + """ + return "#" * level + ' ' + title + '\n\n' + + @staticmethod + def title_paragraph(title, content): + """ + 样例: **title**: paragraph + :param title: + :param content: + :return: + """ + return '**%s**: %s\n\n' % (title, content) + + @staticmethod + def emphasize(content): + """ + 样例：**xxx** + :param content: + :return: + """ + return '**%s**\n\n' % content + + @staticmethod + def text(content, indent=1): + """ + + :param content: + :param indent: + :return: + """ + return ' ' * indent + content + '\n\n' + + @staticmethod + def line_wrap(): + """ + + :return: + """ + return '\n\n' + + @staticmethod + def table(title, tuples): + """ + + :param title: + :param tuples: + :return: + """ + md_table = MDTable(title) + for tup in tuples: + md_table.append(tup) + return md_table.serialize() + + @staticmethod + def unordered_list(contents): + """ + + :param contents: + :return: + """ + uo_list = MDUnorderedList() + for content in contents: + uo_list.append(content) + return uo_list.serialize() diff --git a/script/upgrade_checker/utils/__init__.py b/script/upgrade_checker/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/upgrade_checker/utils/command.py b/script/upgrade_checker/utils/command.py new file mode 100644 index 00000000..80c7150e --- /dev/null +++ b/script/upgrade_checker/utils/command.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +import subprocess +from subprocess import PIPE +from upgrade_checker.utils.exception import ShellExecException + + +class Shell(object): + + @staticmethod + def run(cmd, check=False, print_desc=None): + if print_desc is not None: + print(print_desc, cmd) + + stat, res = subprocess.getstatusoutput(cmd) + + if check and stat != 0: + raise ShellExecException(cmd, stat, res) + return stat, res + + @staticmethod + def communicate(progress, message, check=False): + conn = subprocess.Popen(progress, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, encoding="utf-8", + universal_newlines=True) + data, err = conn.communicate(message, 60) + conn.terminate() + + if check and err is not None: + raise ShellExecException('{0} < {1}'.format(progress, message), 1, err) + return data, err + + + +class Download(object): + + @staticmethod + def wget(url, output): + """ + download content of url by wget, and store it into output. + """ + cmd = 'wget {0} -O {1}'.format(url, output) + try: + Shell.run(cmd, check=True) + except ShellExecException as e: + Shell.run('rm {0} -fr'.format(output), check=False) + raise e + + diff --git a/script/upgrade_checker/utils/exception.py b/script/upgrade_checker/utils/exception.py new file mode 100644 index 00000000..10a54614 --- /dev/null +++ b/script/upgrade_checker/utils/exception.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +""" +User define Exceptions +""" + +class ParamParseException(Exception): + """ + Exception when parse params + """ + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg + +class ShellExecException(Exception): + """ + Exception when execute a shell command + """ + def __init__(self, cmd, stat, msg): + self.cmd = cmd + self.msg = msg + self.stat = stat + + def __str__(self): + return 'Command Execute Failed with err code {0}. \ncmd:\n{1}\nreason:\n{2}'.format( + self.stat, + self.cmd, + self.msg + ) diff --git a/script/upgrade_checker/utils/param.py b/script/upgrade_checker/utils/param.py new file mode 100644 index 00000000..9690f609 --- /dev/null +++ b/script/upgrade_checker/utils/param.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# +""" +参数模块 +""" + +import os +import sys +from getopt import getopt, GetoptError +from enum import Enum +from upgrade_checker.utils.exception import ParamParseException + + +class Action(Enum): + """ + 程序运行动作 + """ + HELP = 0 + VERIFY = 1 + EXPORT = 2 + + +class ReportFormat(Enum): + """ + 报告格式 + """ + MARKDOWN = 0 + + @staticmethod + def suffix(fmt): + if fmt == ReportFormat.MARKDOWN: + return 'md' + else: + assert False + + +class ReportMode(Enum): + """ + 报告粒度 + """ + SUMMARY = 0 + DETAIL = 1 + + +class Option(object): + def __init__(self, value, shortopt, longopt, assign_func=None): + """ + 一个参数选项 + :param value: 默认值 + :param shortopt: 解析时的短命令 + :param longopt: 解析时的长命令 + :param assign_func: 设置新值时的检查函数 + :return: + """ + self.value = value + self.shortopt = shortopt + self.longopt = longopt + self.assign_func = assign_func + + def assign(self, value): + self.value = value if self.assign_func is None else self.assign_func(value) + + +class Param(object): + helper = """ + 命令格式： + python3 main.py [action [params, ...] ] + + 参数 ACTION，所需执行的动作: + check | verify 校验数据库元数据 + export 导出一份元数据地图 + help | -h | -? 打印帮助 + + 参数 params： + -p | --port 数据库端口 + -F | --report_format 报告格式，支持markdown + -M | --report_mode 报告模式，detail详细，summary摘要，默认summary + -v | --vmap 指定使用某个地图，不然自己检测数据库版本并下载。默认自己检测 + -d | --debug 开启debug运行模式，将会打印更多的日志。 + + 更多详细内容参考《README.md》 + """ + + def __init__(self, root_path, argv): + self._opt_info = ['', []] # shotopts, longopts + self._opt_dict = {} + + self.root_path = root_path + self.action = Action.HELP + + self.port = self._register(16666, 'p:', 'port=', Param.assign_port) + self.report_format = self._register(ReportFormat.MARKDOWN, 'F:', 'report-format=', Param.assign_report_format) + self.report_mode = self._register(ReportMode.SUMMARY, 'M:', 'report-mode=', Param.assign_report_mode) + self.vmap = self._register(None, 'v:', 'vmap=', Param.assign_vmap) + self.debug = self._register(False, 'd', 'debug', Param.assign_debug) + + try: + self._parse(argv[1:]) + except ParamParseException as e: + self.action = Action.HELP + print('ERROR：', e) + + def __str__(self): + return 'Param as: ' + str({ + 'root_path': self.root_path, + 'action': self.action, + 'port': self.port.value, + 'report_format': self.report_format.value, + 'report_mode': self.report_mode.value, + 'vmap': self.vmap.value, + 'debug': self.debug.value + }) + '\n' + + def _register(self, value, shortopt, longopt, assign_func=None): + """ + 创建并返回一个Option，同时将长短指令添加到_opt_info中，用以解析，将Option添加到自身字典内，以长短指令为键 + :param value: 默认值 + :param shortopt: 解析时的短命令 + :param longopt: 解析时的长命令 + :param assign_func: 设置新值时的检查函数 + :return: + """ + opt = Option(value, shortopt, longopt, assign_func) + self._opt_info[0] += shortopt + self._opt_info[1].append(longopt) + + short_key = '-' + (shortopt if shortopt[-1] != ':' else shortopt[:-1]) + long_key = '--' + (longopt if longopt[-1] != '=' else longopt[:-1]) + assert self._opt_dict.get(short_key) is None + assert self._opt_dict.get(long_key) is None + self._opt_dict[short_key] = opt + self._opt_dict[long_key] = opt + + return opt + + def _parse(self, argv): + """ + 解析参数，第一个参数必须是action，后面的逐个解析 + :param argv: 参数列表 + :return: + """ + self.action = Param.assign_action(argv) + if self.is_help(): + return + + try: + opts, unused = getopt(argv[1:], self._opt_info[0], self._opt_info[1]) + except GetoptError as e: + raise ParamParseException(e.msg) + + if len(unused) > 0: + raise ParamParseException('解析出错，未知的参数{}'.format(unused[0])) + for key, value in opts: + opt = self._opt_dict.get(key) + assert opt is not None + opt.assign(value) + + def is_help(self): + return self.action == Action.HELP + + @staticmethod + def assign_action(argv): + if len(argv) == 0: + return Action.HELP + + action = argv[0] + if action in ("help", "--help", "-h", "-?"): + return Action.HELP + elif action.lower() in ["check", "verify"]: + return Action.VERIFY + elif action.lower() == "export": + return Action.EXPORT + else: + raise ParamParseException("错误的动作参数'{0}'.".format(action)) + + @staticmethod + def assign_port(port): + port = int(port) + if 0 < port < 65535: + return port + else: + raise ParamParseException("错误的端口参数port {0}, 请保持在(0, 65535)".format(port)) + + @staticmethod + def assign_report_format(fmt): + if fmt.lower() in ['md', 'markdown']: + return ReportFormat.MARKDOWN + else: + raise ParamParseException("错误的格式参数report-format {0},当前仅支持markdown格式。".format(fmt)) + + @staticmethod + def assign_report_mode(gran): + if gran.lower() == 'summary': + return ReportMode.SUMMARY + elif gran.lower == 'detail': + return ReportMode.DETAIL + else: + raise ParamParseException("错误的模式参数report-mode {0}，仅支持summary、detail。".format(gran)) + + @staticmethod + def assign_vmap(vmap): + return os.path.abspath(vmap) + + @staticmethod + def assign_debug(debug): + return True + + + +if __name__ == "__main__": + test_param = Param(sys.path[0], sys.argv) + print(test_param) + diff --git a/script/upgrade_checker/utils/singleton.py b/script/upgrade_checker/utils/singleton.py new file mode 100644 index 00000000..79405ed3 --- /dev/null +++ b/script/upgrade_checker/utils/singleton.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +""" +单例模式 +""" + +def singleton(cls): + _instance = {} + + def inner(): + if cls not in _instance: + _instance[cls] = cls() + return _instance.get(cls) + return inner diff --git a/script/upgrade_checker/utils/version.py b/script/upgrade_checker/utils/version.py new file mode 100644 index 00000000..4d0ffee3 --- /dev/null +++ b/script/upgrade_checker/utils/version.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# + +# 工具版本号 +# 格式举例：5000000 -> 05 00 00 00，用以对应openGauss版本号5.0.0 以及工具小版本的修改 +# 不必每个版本号都完整对应，一个工具版本理论上是可以上下兼容好几个og的版本的。 +UPGRADE_CHECKER_VERSION = 5010000 + +# 适用的openGauss版本 +FOR_OPENGAUSS = [ + # '2.0.0', '2.0.1', + # '2.1.0', + # '3.0.0', '3.0.1', '3.0.2' + '3.0.3', + '5.0.0', + '5.1.0' +] + diff --git a/script/upgrade_checker/verifier.py b/script/upgrade_checker/verifier.py new file mode 100644 index 00000000..e963ddb5 --- /dev/null +++ b/script/upgrade_checker/verifier.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_upgradechk is a utility to check meta data in gaussdb after upgrade. +############################################################################# +""" +校验组件模块，由三个模块进行配合，进行整体的流程。 + collector：负责数据收集。连接数据库，构造校验规则Rule，并执行Rule进行数据的收集。 + analyzer：负责数据分析。依据vmap，对输入的携带有数据的Rule进行分析，生成结论Conclusion + exporter：负责数据简化导出。将Rule的结果进行简化，记录到一个新的vmap内，最终导出生成vmap。 +""" + + +from upgrade_checker.utils.param import Action +from upgrade_checker.rules.rule_maker import RuleMaker +from upgrade_checker.rules.vmap import VMapHeader, VerifyMap +from upgrade_checker.opengauss import og +from upgrade_checker.log import logger + + +class Collector(object): + """ + 负责数据收集。连接数据库，构造校验规则Rule，并执行Rule进行数据的收集。 + """ + @staticmethod + def prepare_db_list(action): + if action == Action.VERIFY: + search_db_sql = "select datname from pg_database where datname not in ('template0', 'template1')" + qres = og.query(search_db_sql) + return [row[0] for row in qres] + elif action == Action.EXPORT: + return ['postgres'] + else: + assert False + + def __init__(self, db_list): + """ + + :param db_list: 需要收集的数据库名称列表 + """ + self._database_list = db_list + self._idx = 0 + + self._rule_buffer = [] + self._estimate = None + + logger.log('verifier.collector 数据库采集列表：[%s]。' % (', '.join(self._database_list))) + logger.log('verifier.collector 初始化完成。') + + def __str__(self): + return 'Collector: dblist [{0}]， current is {1}'.format( + ', '.join(self._database_list), + self._database_list[self._idx - 1] + ) + + def __iter__(self): + return self + + def __next__(self): + if len(self._rule_buffer) == 0 and self._collect_next_db() is None: + raise StopIteration + rule = self._rule_buffer.pop() + rule.run() + return rule + + def _estimate_workload(self): + self._estimate = len(self._rule_buffer) + + def _collect_next_db(self): + if self._idx >= len(self._database_list): + return + + self._rule_buffer = RuleMaker.make_rules(self._database_list[self._idx]) + self._estimate_workload() + logger.log('数据库%s的所有校验规则准备完毕，共计%d条，开始执行数据采集....' % (self._database_list[self._idx], len(self._rule_buffer))) + + self._idx += 1 + return self._rule_buffer + + def current_db(self): + return self._database_list[self._idx - 1] + + def check_next_rule(self): + if len(self._rule_buffer) == 0 and self._collect_next_db() is None: + return + + rule = self._rule_buffer.pop() + rule.run() + return rule + + def get_progress(self): + """ + 返回当前采集的预估进度。 + """ + finished_db_percentage = (self._idx - 1) / len(self._database_list) * 100.0 + curr_db_percentage = (1.0 - len(self._rule_buffer) / self._estimate) / len(self._database_list) * 100.0 + + percentage = int(finished_db_percentage + curr_db_percentage) + if percentage < 0: + percentage = 0 + elif percentage > 100: + percentage = 100 + + return percentage + + +class Analyzer(object): + """ + 负责数据分析。依据vmap，对输入的携带有数据的Rule进行分析，生成结论Conclusion + """ + def __init__(self, vmap): + self.vmap = VerifyMap(vmap) + self.vmap.load() + VMapHeader.check_availability(self.vmap.head) + logger.log('verifier.analyzer 初始化完成。') + + def __del__(self): + self.close() + + def __str__(self): + return 'Analyzer: using vmap ({0})'.format(self.vmap.__str__()) + + def analyze(self, rule): + """ + 在vmap内找到对应的规则预期输出，并进行分析处理。 + :param rule: + :return: + """ + expect = self.vmap.get(rule.sql) + conclusion = rule.analyze(expect) + logger.debug('分析完毕一条规则，共计{0}条小项，{1}告警，规则内容为：{2}。'.format( + len(conclusion.details), + len(conclusion.warnings), + rule.sql) + ) + return conclusion + + def close(self): + pass + + +class Exporter(object): + """ + 负责数据简化导出。将Rule的结果进行简化，记录到一个新的vmap内，最终导出生成vmap。 + """ + def __init__(self, export_vmap_file): + self.vmap = VerifyMap(export_vmap_file) + logger.log('verifier.exporter 初始化完成。') + + def __str__(self): + return 'Exporter: export to vmap ({0})'.format(self.vmap.show_info()) + + def record(self, rule): + self.vmap.push(rule) + logger.debug('Exporter完成记录一条规则：%s。' % rule.sql) + + def export(self): + self.vmap.dump() + logger.debug('Exporter导出校验地图完成。') + + def close(self): + self.vmap.dump() + + -- Gitee From 1f58959b3115ea1ad3556518a757c1f71e7db1a8 Mon Sep 17 00:00:00 2001 From: chentingting <2393940156@qq.com> Date: Fri, 2 Jun 2023 17:42:50 +0800 Subject: [PATCH 33/96] =?UTF-8?q?Revert=20"=E5=8F=96=E6=B6=88=E6=95=B4?= =?UTF-8?q?=E4=BD=93=E5=AF=B9=E7=A3=81=E7=9B=98=E8=B0=83=E5=BA=A6=E7=9A=84?= =?UTF-8?q?=E6=A3=80=E6=9F=A5"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 7a8c191aec73f5febf4224e16a62f7d5806fd71b. --- script/gs_checkos | 4 ---- 1 file changed, 4 deletions(-) diff --git a/script/gs_checkos b/script/gs_checkos index 55ab92d4..038d6950 100644 --- a/script/gs_checkos +++ b/script/gs_checkos @@ -1489,12 +1489,8 @@ def main(): if ("A" in g_opts.item_detail): itemList = CHECK_ITEMNUMLIST - if 'A10' in itemList: - itemList.remove('A10') elif ("B" in g_opts.item_detail): itemList = SET_ITEMNUMLIST - if 'B4' in itemList: - itemList.remove('B4') else: sortList = sorted(g_opts.item_detail) itemList = sortList -- Gitee From 5f81fdcda41e7d31988a0ed587740c721d3a4e73 Mon Sep 17 00:00:00 2001 From: chentingting <2393940156@qq.com> Date: Tue, 6 Jun 2023 15:38:25 +0800 Subject: [PATCH 34/96] fix upgrade cp error --- script/local/UpgradeUtility.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/script/local/UpgradeUtility.py b/script/local/UpgradeUtility.py index 039e5ed6..69fae173 100644 --- a/script/local/UpgradeUtility.py +++ b/script/local/UpgradeUtility.py @@ -2278,14 +2278,6 @@ def cleanInstallPath(): output : NA """ installPath = g_opts.appPath - commit_id = installPath[-8:] - if commit_id: - dss_app = os.path.realpath( - os.path.join(os.path.dirname(installPath), f'dss_app_{commit_id}')) - cmd = "(if [ -d '%s' ]; then rm -rf '%s'; fi)" % (dss_app, dss_app) - g_logger.log("Command for cleaning install path: %s." % cmd) - CmdExecutor.execCommandLocally(cmd) - if not os.path.exists(installPath): g_logger.debug(ErrorCode.GAUSS_502[ "GAUSS_50201"] % installPath + " No need to clean.") @@ -2314,6 +2306,15 @@ def cleanInstallPath(): # and then we will have the permission to clean # appPath under commit-upgrade # under rollback, we also need to restore the permission + + commit_id = installPath[-8:] + if commit_id: + dss_app = os.path.realpath( + os.path.join(os.path.dirname(installPath), f'dss_app_{commit_id}')) + cmd = "(if [ -d '%s' ]; then rm -rf '%s'; fi)" % (dss_app, dss_app) + g_logger.debug("Command for cleaning install path: %s." % cmd) + CmdExecutor.execCommandLocally(cmd) + pluginPath = "%s/lib/postgresql/pg_plugin" % installPath cmd = "(if [ -d '%s' ]; then chmod -R %d '%s'; fi)" % ( pluginPath, DefaultValue.KEY_DIRECTORY_MODE, pluginPath) @@ -2330,6 +2331,11 @@ def cleanInstallPath(): appBakPath) cmd += " && (if [ ! -d '%s' ]; then mkdir -p '%s'; fi)" % ( appBakPath, appBakPath) + delete_bin = os.path.join(appBakPath, f'gaussdb_{commit_id}/bin') + cmd += " && (if [ -L '{0}' ]; then unlink '{0}'; fi)".format( + os.path.join(delete_bin, 'perctrl')) + cmd += " && (if [ -L '{0}' ]; then unlink '{0}'; fi)".format( + os.path.join(delete_bin, 'cm_persist')) cmd += " && (if [ -d '%s' ]; then cp -r '%s/' '%s/to_be_delete/'; fi)" % ( installPath, installPath, tmpDir) g_logger.debug( -- Gitee From 8349cd36a5485c49280f0dfced1916d1c9b55efa Mon Sep 17 00:00:00 2001 From: jiwenke Date: Thu, 8 Jun 2023 15:18:38 +0800 Subject: [PATCH 35/96] =?UTF-8?q?au=5Fsize=E8=B0=83=E6=95=B4=E4=B8=BA?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E6=B1=A0=E5=8C=96=E5=9C=BA=E6=99=AF=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/install/OLAP/InstallImplOLAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/impl/install/OLAP/InstallImplOLAP.py b/script/impl/install/OLAP/InstallImplOLAP.py index 58c591fc..68bf7b31 100644 --- a/script/impl/install/OLAP/InstallImplOLAP.py +++ b/script/impl/install/OLAP/InstallImplOLAP.py @@ -200,7 +200,7 @@ class InstallImplOLAP(InstallImpl): self.context.clusterInfo.dss_shared_disks, self.context.clusterInfo.dss_pri_disks, self.context.user).items(): - au_size = '2048' + au_size = '4096' if dss_disk.find('shared') == -1: au_size = '65536' source_cmd = "source %s; " % self.context.mpprcFile -- Gitee From 3de833a9553c123fa269b18d7ae1b8e436c31619 Mon Sep 17 00:00:00 2001 From: chentingting <2393940156@qq.com> Date: Sat, 10 Jun 2023 15:13:46 +0800 Subject: [PATCH 36/96] check dssserver available --- script/gspylib/component/DSS/dss_checker.py | 29 +++++++++++++++++-- script/gspylib/component/DSS/dss_comp.py | 3 ++ .../component/Kernel/DN_OLAP/DN_OLAP.py | 2 -- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/script/gspylib/component/DSS/dss_checker.py b/script/gspylib/component/DSS/dss_checker.py index 4f327f55..cfb81b4b 100644 --- a/script/gspylib/component/DSS/dss_checker.py +++ b/script/gspylib/component/DSS/dss_checker.py @@ -168,6 +168,29 @@ class DssConfig(): else: return False + @staticmethod + def check_process_available(logger, process, retry=5, interval=1): + ''' + The dssserver process is not available when running. + Then check if it is available. + ''' + logger.debug(f'Start to check {process} available.' ) + if process.find('dssserver') > -1: + cmd = 'dsscmd lsvg' + for cur in range(retry): + sts, out = CmdUtil.exec_by_popen(cmd) + if not sts: + # There is '.' in the out. + logger.debug(f'The dssserver is not available. Message: {out}') + if cur >= retry - 1: + return False + else: + time.sleep(interval) + else: + logger.debug(f'The dssserver is available. The result of the lsvg: {out}') + break + return True + @staticmethod def set_cm_manual_flag(inst_id, flag, logger): gauss_home = ClusterDir.get_gauss_home() @@ -200,9 +223,11 @@ class DssConfig(): check_flag = flag logger.log(f"Start to wait for {flag} to be started.") while timeout > 0: - if not DssConfig.check_process_exist(check_flag=check_flag): + if not (DssConfig.check_process_exist(check_flag=check_flag) and + DssConfig.check_process_available(logger, flag, retry=1)): if timeout % 5 == 0: - logger.debug(f'The process {flag} if not running.') + logger.debug( + f'The process {flag} if not running or not available.') timeout -= 1 time.sleep(1) continue diff --git a/script/gspylib/component/DSS/dss_comp.py b/script/gspylib/component/DSS/dss_comp.py index 75102a7b..552d71e1 100644 --- a/script/gspylib/component/DSS/dss_comp.py +++ b/script/gspylib/component/DSS/dss_comp.py @@ -251,6 +251,9 @@ class Dss(BaseComponent): om init dss server ''' Dss.start_dss_server(self.logger, self.binPath) + if not DssConfig.check_process_available( + self.logger, 'dssserver'): + raise Exception(ErrorCode.GAUSS_512["GAUSS_51252"]) @staticmethod def catch_err(exist_so=True): diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index d3897eca..3f5f5ec3 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -144,8 +144,6 @@ class DN_OLAP(Kernel): if self.paxos_mode: cmd += " -c" elif self.dss_mode: - if not DssConfig.check_process_exist('dssserver'): - raise Exception('The dssserver process does not exist.') vgname = EnvUtil.getEnv('VGNAME') dss_home = EnvUtil.getEnv('DSS_HOME') inst_id = DssInst.get_dss_id_from_key(dss_home) -- Gitee From 386d9d60bcf26975fd2a11fcab0bff316f33cb95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E5=87=AF?= Date: Fri, 16 Jun 2023 18:12:52 +0800 Subject: [PATCH 37/96] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8DI7E53Q=20currentTim?= =?UTF-8?q?e=20-->current=5Ftime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/threads/SshTool.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/script/gspylib/threads/SshTool.py b/script/gspylib/threads/SshTool.py index 584fa9dc..f3cc4b0d 100644 --- a/script/gspylib/threads/SshTool.py +++ b/script/gspylib/threads/SshTool.py @@ -76,28 +76,28 @@ class SshTool(): self._finalizer = weakref.finalize(self, self.clenSshResultFiles) self.__sessions = {} - currentTime = str(datetime.datetime.now()).replace(" ", "_").replace( + current_time = str(datetime.datetime.now()).replace(" ", "_").replace( ".", "_") randomnum = ''.join(sample('0123456789', 3)) # can tmp path always access? if key == "": self.__hostsFile = "/tmp/gauss_hosts_file_%d_%s_%s" % ( - self.__pid, currentTime, randomnum) + self.__pid, current_time, randomnum) self.__resultFile = "/tmp/gauss_result_%d_%s_%s.log" % ( - self.__pid, currentTime, randomnum) + self.__pid, current_time, randomnum) self.__outputPath = "/tmp/gauss_output_files_%d_%s_%s" % ( - self.__pid, currentTime, randomnum) + self.__pid, current_time, randomnum) self.__errorPath = "/tmp/gauss_error_files_%d_%s_%s" % ( - self.__pid, currentTime, randomnum) + self.__pid, current_time, randomnum) else: self.__hostsFile = "/tmp/gauss_hosts_file_%d_%s_%s_%s" % ( - self.__pid, key, currentTime, randomnum) + self.__pid, key, current_time, randomnum) self.__resultFile = "/tmp/gauss_result_%d_%s_%s_%s.log" % ( - self.__pid, key, currentTime, randomnum) + self.__pid, key, current_time, randomnum) self.__outputPath = "/tmp/gauss_output_files_%d_%s_%s_%s" % ( - self.__pid, key, currentTime, randomnum) + self.__pid, key, current_time, randomnum) self.__errorPath = "/tmp/gauss_error_files_%d_%s_%s_%s" % ( - self.__pid, key, currentTime, randomnum) + self.__pid, key, current_time, randomnum) self.__resultStatus = {} if logFile is None: -- Gitee From 6854e6085e486da0e8444c04715f9e7896236256 Mon Sep 17 00:00:00 2001 From: gentle_hu Date: Sun, 11 Jun 2023 15:32:40 +0800 Subject: [PATCH 38/96] Adapt to the OM log and De-identify XXacl columns --- script/gs_upgradechk | 15 +++++++++- script/upgrade_checker/README.md | 4 +-- script/upgrade_checker/project.py | 3 ++ script/upgrade_checker/rules/meta.py | 43 +++++++++++++++++---------- script/upgrade_checker/utils/param.py | 4 +-- 5 files changed, 49 insertions(+), 20 deletions(-) diff --git a/script/gs_upgradechk b/script/gs_upgradechk index 03cfa350..28548430 100644 --- a/script/gs_upgradechk +++ b/script/gs_upgradechk @@ -21,12 +21,25 @@ import sys +import os from upgrade_checker.utils.param import Param from upgrade_checker.project import ProjectFactory +def program_workspace(): + """ + If OM exists, use the log path of OM. + """ + gauss_log = os.getenv('GAUSSLOG') + om_log = os.path.join(gauss_log, 'om') if gauss_log is not None else None + if om_log is not None and os.access(om_log, os.F_OK): + return os.path.join(om_log, 'upgrade_checker') + + return os.path.join(sys.path[0], 'upgrade_checker') + + if __name__ == "__main__": - param = Param(sys.path[0] + '/upgrade_checker', sys.argv) + param = Param(program_workspace(), sys.argv) if param.is_help(): print(param.helper) exit(0) diff --git a/script/upgrade_checker/README.md b/script/upgrade_checker/README.md index 03a0fffe..635fae07 100644 --- a/script/upgrade_checker/README.md +++ b/script/upgrade_checker/README.md @@ -31,7 +31,7 @@ config-params支持的参数选项及其默认值如下 -d | --debug ``` -- port：数据库端口号 +- port：数据库端口号。 - debug：会打印更多的日志，用于工具问题定位。无参数，默认关闭，指定时开启。 - report-format：校验报告格式，当前仅支持markdown。 - report-mode：校验报告详细程度，支持`summary`，`detail`两种模式，默认`summary`模式。此选项仅会影响生成的报告的详细程度，并不影响校验内容的多少，`detail`仅是会把所有的校验规则无论对错都整理输出，报告整理比较耗时，而`summary`则仅整理输出错误的内容。 @@ -89,7 +89,7 @@ upgrade-checker |f- user-created-vmap -用户自己生成的校验地图 |f- report -某次检测的结果报告 |f-- README.md 说明书 -|f-- main.py 入口脚本 +|f-- gs_upgradechk 入口脚本 ``` ## 校验原理 diff --git a/script/upgrade_checker/project.py b/script/upgrade_checker/project.py index f04f9d1e..c241c710 100644 --- a/script/upgrade_checker/project.py +++ b/script/upgrade_checker/project.py @@ -74,6 +74,9 @@ class Project(object): 创建工作目录文件夹等。 :return: """ + if not os.access(self.dir_root, os.F_OK): + os.mkdir(self.dir_root, 0o700) + if not os.access(self.dir_workspace, os.F_OK): os.mkdir(self.dir_workspace, 0o700) diff --git a/script/upgrade_checker/rules/meta.py b/script/upgrade_checker/rules/meta.py index 37e2428b..5048c92f 100644 --- a/script/upgrade_checker/rules/meta.py +++ b/script/upgrade_checker/rules/meta.py @@ -92,7 +92,8 @@ META = { [ ContentRulesMeta( key='tmplname', - key_desc='名称为%s的模板' + key_desc='名称为%s的模板', + ignore_col='tmplacl' ) ] ), @@ -104,7 +105,8 @@ META = { ContentRulesMeta( key='spcname', key_desc='名称为%s的表空间', - filters=' oid < 10000 ' + filters=' oid < 10000 ', + ignore_col='spcacl' ) ] ), @@ -149,16 +151,17 @@ META = { " attname" ")", key_desc='模式.关系名(列名)为%s的列', - filters=' attrelid < 10000 ' + filters=' attrelid < 10000 ', + ignore_col='attacl' ), # initdb的。但忽略toast表，因为表名里也带oid，会变，无法校验。 ContentRulesMeta( complete_sql="select format('%s.%s(%s)', n.nspname, c.relname, a.attname)," - " md5(format('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s', " + " md5(format('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s', " " t.typname, a.attstattarget, a.attlen, a.attnum, a.attndims," " a.attcacheoff, a.atttypmod, a.attbyval, a.attstorage, a.attalign," " a.attnotnull, a.atthasdef,a.attisdropped, a.attislocal, a.attcmprmode," - " a.attinhcount, a.attcollation, a.attacl, a.attoptions, a.attfdwoptions," + " a.attinhcount, a.attcollation, a.attoptions, a.attfdwoptions," " a.attinitdefval, a.attkvtype))" "from pg_attribute a left join pg_class c on a.attrelid = c.oid " " left join pg_namespace n on c.relnamespace = n.oid " @@ -186,7 +189,7 @@ META = { key="oid", key_desc='oid为%s的函数或存储过程', filters=' oid < 10000 ', - ignore_col='proargdefaults' + ignore_col='proargdefaults,proacl' ), ContentRulesMeta( key="format('%s.%s(%s)'," @@ -226,7 +229,7 @@ META = { key="format('%s.%s',(select nspname from pg_namespace n where n.oid = relnamespace), relname)", key_desc='名为%s的表或索引或视图等', filters=' oid < 10000 ', - ignore_col='relfilenode,relpages,reltuples,relallvisible,relfrozenxid,relfrozenxid64,relminmxid', + ignore_col='relfilenode,relpages,reltuples,relallvisible,relacl,relfrozenxid,relfrozenxid64,relminmxid', oid_col='oid,reltype,reloftype', accuracy=Accuracy.STRICT ), @@ -239,7 +242,7 @@ META = { oid_col='relnamespace,reltype,reloftype,relowner,reltablespace', accuracy=Accuracy.STRICT ), - # 表的不需要使用pg_get_tabledef检查，会在第一步的校验系统表结构的时候校验。 + # 表的会在第一步的校验系统表结构的时候校验pg_get_tabledef。 # 索引等会在对应的扩展系统表校验，此处仅校验视图即可 ContentRulesMeta( complete_sql="select format('def:%s.%s', n.nspname, c.relname), " @@ -297,13 +300,13 @@ META = { key='datname', key_desc='数据库%s', filters=' oid < 10000 ', - ignore_col='encoding,datcollate,datctype,datlastsysoid,datfrozenxid,datfrozenxid64,datminmxid' + ignore_col='encoding,datcollate,datctype,datlastsysoid,datfrozenxid,datacl,datfrozenxid64,datminmxid' ), ContentRulesMeta( key='datname', key_desc='数据库%s', filters=' 9999 < oid and oid < 16384 ', - ignore_col='oid,encoding,datcollate,datctype,datlastsysoid,datfrozenxid,datfrozenxid64,datminmxid' + ignore_col='oid,encoding,datcollate,datctype,datlastsysoid,datfrozenxid,datacl,datfrozenxid64,datminmxid' ) ] ), @@ -315,7 +318,8 @@ META = { ContentRulesMeta( key='srvname', key_desc='外表服务器%s', - filters=' oid < 10000 ' + filters=' oid < 10000 ', + ignore_col='srvacl' ), ContentRulesMeta( complete_sql="select srvname, " @@ -622,13 +626,15 @@ META = { ContentRulesMeta( key='lanname', key_desc='oid为%s的语言', - filters=' oid < 10000 ' + filters=' oid < 10000 ', + ignore_col='lanacl' ), ContentRulesMeta( key='lanname', key_desc='名称为%s的语言', filters=' 9999 < oid and oid < 10000 ', - oid_col='oid,lanplcallfoid,laninline,lanvalidator', + ignore_col='lanacl', + oid_col='oid,lanplcallfoid,laninline,lanvalidator' ) ] ), @@ -653,13 +659,13 @@ META = { key='nspname', key_desc='名称为%s的schema', filters=' oid < 10000 ', - ignore_col='nsptimeline,in_redistribution,nspblockchain,nspcollation' + ignore_col='nsptimeline,nspacl,in_redistribution,nspblockchain,nspcollation' ), ContentRulesMeta( key='nspname', key_desc='名称为%s的schema', filters=' 9999 < oid and oid < 16384 ', - ignore_col='nsptimeline,in_redistribution,nspblockchain,nspcollation', + ignore_col='nsptimeline,nspacl,in_redistribution,nspblockchain,nspcollation', oid_col='oid,nspowner' ) ] @@ -820,6 +826,7 @@ META = { key="lomowner", key_desc='oid为%s的大对象元数据', filters=' oid < 16384 ', + ignore_col='lomacl', accuracy=Accuracy.ALLOW_MORE ) ] @@ -1203,6 +1210,7 @@ META = { key="oid", key_desc='oid为%s的外部数据源对象的信息', filters=' oid < 16384 ', + ignore_col='srcacl', accuracy=Accuracy.STRICT ) ] @@ -1216,6 +1224,7 @@ META = { key="oid", key_desc='oid为%s的文件夹', filters=' oid < 16384 ', + ignore_col='srcacl', accuracy=Accuracy.STRICT ) ] @@ -1325,6 +1334,7 @@ META = { key="oid", key_desc='oid为%s的package', filters=' oid < 16384 ', + ignore_col='pkgacl', accuracy=Accuracy.STRICT ) ] @@ -1383,6 +1393,7 @@ META = { key="oid", key_desc='oid为%s的节点组', filters=' oid < 16384 ', + ignore_col='group_acl', accuracy=Accuracy.STRICT ) ] @@ -1645,6 +1656,7 @@ META = { key="oid", key_desc='oid为%s的加密主密钥', filters=' oid < 16384 ', + ignore_col='key_acl', accuracy=Accuracy.STRICT ) ] @@ -1658,6 +1670,7 @@ META = { key="oid", key_desc='oid为%s的列加密密钥', filters=' oid < 16384 ', + ignore_col='key_acl', accuracy=Accuracy.STRICT ) ] diff --git a/script/upgrade_checker/utils/param.py b/script/upgrade_checker/utils/param.py index 9690f609..8fe1c285 100644 --- a/script/upgrade_checker/utils/param.py +++ b/script/upgrade_checker/utils/param.py @@ -182,7 +182,7 @@ class Param(object): return Action.HELP action = argv[0] - if action in ("help", "--help", "-h", "-?"): + if action.lower() in ("help", "--help", "-h", "-?"): return Action.HELP elif action.lower() in ["check", "verify"]: return Action.VERIFY @@ -210,7 +210,7 @@ class Param(object): def assign_report_mode(gran): if gran.lower() == 'summary': return ReportMode.SUMMARY - elif gran.lower == 'detail': + elif gran.lower() == 'detail': return ReportMode.DETAIL else: raise ParamParseException("错误的模式参数report-mode {0}，仅支持summary、detail。".format(gran)) -- Gitee From ca938908380491fc8cc068eca7f1e230339f7f35 Mon Sep 17 00:00:00 2001 From: liuheng Date: Tue, 20 Jun 2023 09:04:44 +0800 Subject: [PATCH 39/96] Revert "fix(cluster_static_config): verify crc value" This reverts commit 5c3a853978c96817abffc040dd349d1fdc2a19bc. --- script/gspylib/common/DbClusterInfo.py | 104 ++++++------------------- script/gspylib/common/ErrorCode.py | 1 - 2 files changed, 23 insertions(+), 82 deletions(-) diff --git a/script/gspylib/common/DbClusterInfo.py b/script/gspylib/common/DbClusterInfo.py index 8e9096d9..91564fca 100644 --- a/script/gspylib/common/DbClusterInfo.py +++ b/script/gspylib/common/DbClusterInfo.py @@ -1804,12 +1804,6 @@ class dbClusterInfo(): info = fp.read(28) (crc, lenth, version, currenttime, nodeNum, localNodeId) = struct.unpack("=IIIqiI", info) - - wait_info = struct.pack("IIqiI",lenth, version, - currenttime, nodeNum, localNodeId) - if self.__verify_crc(crc, wait_info) == False: - raise Exception(ErrorCode.GAUSS_512["GAUSS_51258"] - % ("headInfo", staticConfigFile)) self.version = version self.installTime = currenttime self.localNodeId = localNodeId @@ -1880,43 +1874,36 @@ class dbClusterInfo(): (crc, nodeId, nodeName) = struct.unpack("=II64s", info) nodeName = nodeName.decode().strip('\x00') dbNode = dbNodeInfo(nodeId, nodeName) - wait_info = struct.pack("I64s",nodeId,nodeName.encode("utf-8")) info = fp.read(68) - wait_info += info (azName, azPriority) = struct.unpack("=64sI", info) dbNode.azName = azName.decode().strip('\x00') dbNode.azPriority = azPriority # get backIps - wait_info += self.__unPackIps(fp, dbNode.backIps) + self.__unPackIps(fp, dbNode.backIps) # get sshIps - wait_info += self.__unPackIps(fp, dbNode.sshIps) + self.__unPackIps(fp, dbNode.sshIps) if (not isLCCluster): # get cm_server information - wait_info += self.__unPackCmsInfo(fp, dbNode) + self.__unPackCmsInfo(fp, dbNode) # get cm_agent information - wait_info += self.__unpackAgentInfo(fp, dbNode) + self.__unpackAgentInfo(fp, dbNode) # get gtm information - wait_info += self.__unpackGtmInfo(fp, dbNode) + self.__unpackGtmInfo(fp, dbNode) info = fp.read(404) - wait_info += info # get cn information - wait_info += self.__unpackCooInfo(fp, dbNode) + self.__unpackCooInfo(fp, dbNode) # get DB information - wait_info += self.__unpackDataNode(fp, dbNode) + self.__unpackDataNode(fp, dbNode) if (not isLCCluster): # get etcd information - wait_info += self.__unpackEtcdInfo(fp, dbNode) + self.__unpackEtcdInfo(fp, dbNode) info = fp.read(8) - wait_info += info # set DB azName for OLAP for inst in dbNode.datanodes: inst.azName = dbNode.azName inst.azPriority = dbNode.azPriority - if self.__verify_crc(crc,wait_info) == False: - raise Exception(ErrorCode.GAUSS_512["GAUSS_51258"] - % ("nodeInfo", staticConfigFile)) return dbNode def __unpackEtcdInfo(self, fp, dbNode): @@ -1930,17 +1917,14 @@ class dbClusterInfo(): etcdInst.hostname = dbNode.name etcdInst.instanceType = INSTANCE_TYPE_UNDEFINED info = fp.read(1100) - wait_info = info (etcdNum, etcdInst.instanceId, etcdInst.mirrorId, etcdhostname, etcdInst.datadir) = struct.unpack("=IIi64s1024s", info) etcdInst.datadir = etcdInst.datadir.decode().strip('\x00') - wait_info += self.__unPackIps(fp, etcdInst.listenIps) + self.__unPackIps(fp, etcdInst.listenIps) info = fp.read(4) - wait_info += info (etcdInst.port,) = struct.unpack("=I", info) - wait_info += self.__unPackIps(fp, etcdInst.haIps) + self.__unPackIps(fp, etcdInst.haIps) info = fp.read(4) - wait_info += info (etcdInst.haPort,) = struct.unpack("=I", info) if (etcdNum == 1): dbNode.etcdNum = 1 @@ -1949,7 +1933,6 @@ class dbClusterInfo(): else: dbNode.etcdNum = 0 dbNode.etcds = [] - return wait_info def __unPackIps(self, fp, ips): """ @@ -1958,17 +1941,13 @@ class dbClusterInfo(): output : NA """ info = fp.read(4) - wait_info = info (n,) = struct.unpack("=i", info) for i in range(int(n)): info = fp.read(128) - wait_info += info (currentIp,) = struct.unpack("=128s", info) currentIp = currentIp.decode().strip('\x00') ips.append(str(currentIp.strip())) info = fp.read(128 * (MAX_IP_NUM - n)) - wait_info += info - return wait_info def __unPackCmsInfo(self, fp, dbNode): """ @@ -1980,19 +1959,16 @@ class dbClusterInfo(): cmsInst.instanceRole = INSTANCE_ROLE_CMSERVER cmsInst.hostname = dbNode.name info = fp.read(1164) - wait_info = info (cmsInst.instanceId, cmsInst.mirrorId, dbNode.cmDataDir, cmsInst.level, self.cmsFloatIp) = struct.unpack("=II1024sI128s", info) dbNode.cmDataDir = dbNode.cmDataDir.decode().strip('\x00') self.cmsFloatIp = self.cmsFloatIp.decode().strip('\x00') cmsInst.datadir = "%s/cm_server" % dbNode.cmDataDir - wait_info += self.__unPackIps(fp, cmsInst.listenIps) + self.__unPackIps(fp, cmsInst.listenIps) info = fp.read(4) - wait_info += info (cmsInst.port,) = struct.unpack("=I", info) - wait_info += self.__unPackIps(fp, cmsInst.haIps) + self.__unPackIps(fp, cmsInst.haIps) info = fp.read(8) - wait_info += info (cmsInst.haPort, cmsInst.instanceType) = struct.unpack("=II", info) if (cmsInst.instanceType == MASTER_INSTANCE): dbNode.cmsNum = 1 @@ -2002,15 +1978,13 @@ class dbClusterInfo(): raise Exception(ErrorCode.GAUSS_512["GAUSS_51204"] % ("CMServer", cmsInst.instanceType)) info = fp.read(4 + 128 * MAX_IP_NUM + 4) - wait_info += info + if (cmsInst.instanceId): dbNode.cmservers.append(cmsInst) self.cmscount += 1 else: dbNode.cmservers = [] - return wait_info - def __unpackAgentInfo(self, fp, dbNode): """ function : Unpack the info of agent. It should be called after @@ -2024,12 +1998,10 @@ class dbClusterInfo(): cmaInst.hostname = dbNode.name cmaInst.instanceType = INSTANCE_TYPE_UNDEFINED info = fp.read(8) - wait_info = info (cmaInst.instanceId, cmaInst.mirrorId) = struct.unpack("=Ii", info) - wait_info += self.__unPackIps(fp, cmaInst.listenIps) + self.__unPackIps(fp, cmaInst.listenIps) cmaInst.datadir = "%s/cm_agent" % dbNode.cmDataDir dbNode.cmagents.append(cmaInst) - return wait_info def __unpackGtmInfo(self, fp, dbNode): """ @@ -2041,13 +2013,11 @@ class dbClusterInfo(): gtmInst.instanceRole = INSTANCE_ROLE_GTM gtmInst.hostname = dbNode.name info = fp.read(1036) - wait_info = info (gtmInst.instanceId, gtmInst.mirrorId, gtmNum, gtmInst.datadir) = struct.unpack("=III1024s", info) gtmInst.datadir = gtmInst.datadir.decode().strip('\x00') - wait_info += self.__unPackIps(fp, gtmInst.listenIps) + self.__unPackIps(fp, gtmInst.listenIps) info = fp.read(8) - wait_info += info (gtmInst.port, gtmInst.instanceType) = struct.unpack("=II", info) if (gtmInst.instanceType == MASTER_INSTANCE): dbNode.gtmNum = 1 @@ -2056,20 +2026,17 @@ class dbClusterInfo(): else: raise Exception(ErrorCode.GAUSS_512["GAUSS_51204"] % ( "GTM", gtmInst.instanceType)) - wait_info += self.__unPackIps(fp, gtmInst.haIps) + self.__unPackIps(fp, gtmInst.haIps) info = fp.read(4) - wait_info += info (gtmInst.haPort,) = struct.unpack("=I", info) info = fp.read(1024 + 4 + 128 * MAX_IP_NUM + 4) - wait_info += info + if (gtmNum == 1): dbNode.gtms.append(gtmInst) self.gtmcount += 1 else: dbNode.gtms = [] - return wait_info - def __unpackCooInfo(self, fp, dbNode): """ function : Unpack the info of coordinator @@ -2081,14 +2048,12 @@ class dbClusterInfo(): cooInst.hostname = dbNode.name cooInst.instanceType = INSTANCE_TYPE_UNDEFINED info = fp.read(2060) - wait_info = info (cooInst.instanceId, cooInst.mirrorId, cooNum, cooInst.datadir, cooInst.ssdDir) = struct.unpack("=IiI1024s1024s", info) cooInst.datadir = cooInst.datadir.decode().strip('\x00') cooInst.ssdDir = cooInst.ssdDir.decode().strip('\x00') - wait_info += self.__unPackIps(fp, cooInst.listenIps) + self.__unPackIps(fp, cooInst.listenIps) info = fp.read(8) - wait_info += info (cooInst.port, cooInst.haPort) = struct.unpack("=II", info) if (cooNum == 1): dbNode.cooNum = 1 @@ -2096,7 +2061,6 @@ class dbClusterInfo(): else: dbNode.cooNum = 0 dbNode.coordinators = [] - return wait_info def __unpackDataNode(self, fp, dbNode): """ @@ -2105,7 +2069,6 @@ class dbClusterInfo(): output : NA """ info = fp.read(4) - wait_info = info (dataNodeNums,) = struct.unpack("=I", info) dbNode.dataNum = 0 @@ -2121,14 +2084,12 @@ class dbClusterInfo(): # then rollback by fp.seek(), and exchange its(xlogdir) value # with ssddir. info = fp.read(2056) - wait_info += info (dnInst.instanceId, dnInst.mirrorId, dnInst.datadir, dnInst.xlogdir) = struct.unpack("=II1024s1024s", info) dnInst.datadir = dnInst.datadir.decode().strip('\x00') dnInst.xlogdir = dnInst.xlogdir.decode().strip('\x00') info = fp.read(1024) - wait_info += info (dnInst.ssdDir) = struct.unpack("=1024s", info) dnInst.ssdDir = dnInst.ssdDir[0].decode().strip('\x00') # if notsetXlog,ssdDir should not be null.use by upgrade. @@ -2137,9 +2098,8 @@ class dbClusterInfo(): dnInst.ssdDir = dnInst.xlogdir dnInst.xlogdir = "" - wait_info += self.__unPackIps(fp, dnInst.listenIps) + self.__unPackIps(fp, dnInst.listenIps) info = fp.read(8) - wait_info += info (dnInst.port, dnInst.instanceType) = struct.unpack("=II", info) if (dnInst.instanceType == MASTER_INSTANCE): dbNode.dataNum += 1 @@ -2149,9 +2109,8 @@ class dbClusterInfo(): else: raise Exception(ErrorCode.GAUSS_512["GAUSS_51204"] % ("DN", dnInst.instanceType)) - wait_info += self.__unPackIps(fp, dnInst.haIps) + self.__unPackIps(fp, dnInst.haIps) info = fp.read(4) - wait_info += info (dnInst.haPort,) = struct.unpack("=I", info) if ( self.clusterType == @@ -2161,42 +2120,34 @@ class dbClusterInfo(): for j in range(maxStandbyCount): peerDbInst = peerInstanceInfo() info = fp.read(1024) - wait_info += info (peerDbInst.peerDataPath,) = struct.unpack("=1024s", info) peerDbInst.peerDataPath = \ peerDbInst.peerDataPath.decode().strip('\x00') - wait_info += self.__unPackIps(fp, peerDbInst.peerHAIPs) + self.__unPackIps(fp, peerDbInst.peerHAIPs) info = fp.read(8) - wait_info += info (peerDbInst.peerHAPort, peerDbInst.peerRole) = struct.unpack("=II", info) dnInst.peerInstanceInfos.append(peerDbInst) else: peerDbInst = peerInstanceInfo() info = fp.read(1024) - wait_info += info (peerDbInst.peerDataPath,) = struct.unpack("=1024s", info) peerDbInst.peerDataPath = \ peerDbInst.peerDataPath.decode().strip('\x00') - wait_info += self.__unPackIps(fp, peerDbInst.peerHAIPs) + self.__unPackIps(fp, peerDbInst.peerHAIPs) info = fp.read(8) - wait_info += info (peerDbInst.peerHAPort, peerDbInst.peerRole) = \ struct.unpack("=II", info) info = fp.read(1024) - wait_info += info (peerDbInst.peerData2Path,) = struct.unpack("=1024s", info) peerDbInst.peerData2Path = \ peerDbInst.peerDataPath.decode().strip('\x00') - wait_info += self.__unPackIps(fp, peerDbInst.peer2HAIPs) + self.__unPackIps(fp, peerDbInst.peer2HAIPs) info = fp.read(8) - wait_info += info (peerDbInst.peer2HAPort, peerDbInst.peer2Role) = \ struct.unpack("=II", info) dnInst.peerInstanceInfos.append(peerDbInst) dbNode.datanodes.append(dnInst) - return wait_info - def initFromStaticConfigWithoutUser(self, staticConfigFile): """ @@ -2221,11 +2172,6 @@ class dbClusterInfo(): info = fp.read(28) (crc, lenth, version, currenttime, nodeNum, localNodeId) = struct.unpack("=IIIqiI", info) - - wait_info = struct.pack("IIqiI",lenth, version, currenttime, nodeNum, localNodeId) - if self.__verify_crc(crc, wait_info) == False: - raise Exception(ErrorCode.GAUSS_512["GAUSS_51258"] - % ("headInfo", staticConfigFile)) if (version <= 100): raise Exception(ErrorCode.GAUSS_516["GAUSS_51637"] % ("cluster static config version[%s]" @@ -4961,7 +4907,3 @@ class dbClusterInfo(): else: raise Exception(ErrorCode.GAUSS_502["GAUSS_50204"] % \ "float IP." + " Error: \n%s" % ret_value) - - @classmethod - def __verify_crc(self, crc, info): - return crc == binascii.crc32(info) diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index ef67f471..036e26d0 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -494,7 +494,6 @@ class ErrorCode(): 'GAUSS_51255': "[GAUSS-51255] : Failed to reencrypt the password with dsscmd", 'GAUSS_51256': "[GAUSS-51256] : Failed to get the encrypted text with dsscmd", 'GAUSS_51257': "[GAUSS-51257] : There are some errors about dsscmd. ", - 'GAUSS_51258': "[GAUSS-51258] : Failed to verify the %s crc value in file: %s", } -- Gitee From a702b2fb293ec4dde86d4c445aa02f9a3c52e21c Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Wed, 21 Jun 2023 11:17:33 +0800 Subject: [PATCH 40/96] =?UTF-8?q?=E4=BF=AE=E6=94=B9gcc=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=90=8E=E7=9A=84=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 46913817..ccf89dda 100644 --- a/build.sh +++ b/build.sh @@ -1,6 +1,7 @@ #!/bin/bash declare binarylib_dir='None' +declare gcc_version='10.3' declare module_name="openGauss" declare version_number='5.1.0' declare version_Kernel='92.298' @@ -21,6 +22,7 @@ function print_help() echo "Usage: $0 [OPTION] -h|--help show help information -3rd|--binarylib_dir the parent directory of binarylibs + -cv|--gcc_version the gcc version only accepts 7.3 and 10.3 " } @@ -38,6 +40,10 @@ while [ $# -gt 0 ]; do binarylib_dir=$2 shift 2 ;; + -cv|--gcc_version) + gcc_version=$2 + shift 2 + ;; *) echo "Internal Error: option processing error: $1" 1>&2 echo "please input right paramtenter, the following command may help you" @@ -92,7 +98,6 @@ else BUILD_TOOLS_PATH="${ROOT_DIR}/binarylibs/buildtools/" fi - log() { echo "[makegaussdb] $(date +%y-%m-%d' '%T): $@" @@ -167,7 +172,7 @@ function clib_copy() { rm -rf $PKG_TMP_DIR/script/gspylib/clib mkdir -p $PKG_TMP_DIR/script/gspylib/clib - cp $BUILD_TOOLS_PATH/gcc7.3/gcc/lib64/libstdc++.so.6 $PKG_TMP_DIR/script/gspylib/clib + cp $BUILD_TOOLS_PATH/gcc${gcc_version}/gcc/lib64/libstdc++.so.6 $PKG_TMP_DIR/script/gspylib/clib cp $BINARYLIBS_PATH/openssl/comm/lib/libssl.so.1.1 $PKG_TMP_DIR/script/gspylib/clib cp $BINARYLIBS_PATH/openssl/comm/lib/libcrypto.so.1.1 $PKG_TMP_DIR/script/gspylib/clib if [ -f $BINARYLIBS_PATH_INSTALL_TOOLS/libpython3.*m.so.1.0 ] -- Gitee From ab45cbe42ed851486f3d5f137d2633275b5a5cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cross-=E7=BD=97?= <1165977584@qq.com> Date: Sun, 25 Jun 2023 13:41:49 +0000 Subject: [PATCH 41/96] =?UTF-8?q?update=20script/local/UpgradeUtility.py.?= =?UTF-8?q?=20=E6=AD=A4=E5=A4=84=E7=94=A8=E6=97=A7=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E7=9A=84libgcc.so=E5=92=8Clibstdc++.so=E8=A6=86=E7=9B=96?= =?UTF-8?q?=E4=BA=86=E6=96=B0=E7=89=88=E6=9C=AC=E7=9A=84libgcc.so=E5=92=8C?= =?UTF-8?q?libstdc++.so=EF=BC=8C=E5=AF=BC=E8=87=B4=E5=B0=B1=E5=9C=B0?= =?UTF-8?q?=E5=8D=87=E7=BA=A7=E5=90=8Egaussdb=E6=89=BE=E4=B8=8D=E5=88=B0?= =?UTF-8?q?=E5=93=8D=E5=BA=94=E7=9A=84=E6=96=B0=E7=89=88=E6=9C=AC=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E8=80=8C=E5=87=BA=E9=94=99=EF=BC=8C=E5=9C=A8=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E4=B9=8B=E5=90=8E=EF=BC=8C=E5=B0=B1=E5=9C=B0=E5=8D=87?= =?UTF-8?q?=E7=BA=A7=E5=92=8C=E7=81=B0=E5=BA=A6=E5=8D=87=E7=BA=A7=E5=9D=87?= =?UTF-8?q?=E8=83=BD=E5=A4=9F=E6=AD=A3=E5=B8=B8=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cross-罗 <1165977584@qq.com> --- script/local/UpgradeUtility.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/script/local/UpgradeUtility.py b/script/local/UpgradeUtility.py index 69fae173..77072db8 100644 --- a/script/local/UpgradeUtility.py +++ b/script/local/UpgradeUtility.py @@ -138,9 +138,7 @@ class CmdOptions(): "postgis--*.*.*.sql": "share/postgresql/extension/", "postgis.control": "share/postgresql/extension/", "pgsql2shp": "bin/", - "shp2pgsql": "bin/", - "libgcc_s.so.*": "lib/", - "libstdc++.so.*": "lib/"} + "shp2pgsql": "bin/"} self.fromFile = False self.setType = "reload" self.isSingleInst = False -- Gitee From 14ac01e96cab0c26098bca8e01bba9dc5ce75349 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Mon, 26 Jun 2023 14:31:42 +0800 Subject: [PATCH 42/96] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/install/InstallImpl.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/script/impl/install/InstallImpl.py b/script/impl/install/InstallImpl.py index f208f450..8e21db04 100644 --- a/script/impl/install/InstallImpl.py +++ b/script/impl/install/InstallImpl.py @@ -126,6 +126,7 @@ class InstallImpl: dn_instanceId = dict() dn_syncNodeHostname = dict() flag = False + for dbinfo in dbNodes: if dbinfo is None: self.context.logger.debug("the number of DN is zero") @@ -140,20 +141,26 @@ class InstallImpl: if datanodes[0].syncNumFirst: flag = True self.context.logger.debug(dn_instanceId) + for syncNode in dn_syncNodeHostname.keys(): syncNumFirst = dn_syncNodeHostname[syncNode] hostname = syncNode if syncNumFirst and syncNum != -1: - raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % "syncNode_hostname and dataNode1 cannot be exist at the same time") + raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % + "syncNode_hostname and dataNode1 cannot be exist at the same time") + if syncNumFirst and syncNum == -1: syncNumFirstRe = re.sub('[,\s]', '', syncNumFirst) - if re.match('(ANY[0-8](.*){1,8})', syncNumFirstRe) or re.match('(FIRST[0-8](.*){1,8})',syncNumFirstRe): + if re.match('(ANY[0-8](.*){1,8})', syncNumFirstRe) or re.match('(FIRST[0-8](.*){1,8})', syncNumFirstRe): self.context.logger.debug("Successfully. matching is correct") else: - raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % "syncNode_hostname, the match of syncNode_hostname is wrong.") + raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % + "syncNode_hostname, the match of syncNode_hostname is wrong. \ + any or first must be ANY or FIRST.") for sync in dn_instanceId.keys(): if syncNumFirst.count(sync) > 1: - raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % "syncNode_hostname, the node in syncNode_hostname is must be the unique.") + raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % + "syncNode_hostname, the node in syncNode_hostname is must be the unique.") elif syncNumFirst.count(sync) == 0: if sync == hostname: self.context.logger.debug("syncNode_hostname, the syncNode_hostname does not including own hostname.") @@ -161,13 +168,15 @@ class InstallImpl: self.context.logger.debug("syncNode_hostname is including non-host nodes.") elif syncNumFirst.count(sync) == 1: if sync == hostname: - raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % "syncNode_hostname, the syncNode_hostname is including own hostname.") + raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % "syncNode_hostname, \ + the syncNode_hostname is including own hostname.") elif syncNumFirst.count("ANY") and syncNumFirst.count("FIRST"): raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % "syncNode_hostname, it can be only one of 'ANY' or 'FIRST'.") else: syncNumFirst = syncNumFirst.replace(sync,'dn_%s' % (dn_instanceId[sync])) self.context.logger.debug("Check syncNode_hostname is correct.") self.context.logger.debug(syncNumFirst) + if len(syncNumFirst) == 0 and syncNum == -1 and flag: raise Exception(ErrorCode.GAUSS_530["GAUSS_53011"] % "syncNode_hostname is must be exist in every hostnode") -- Gitee From 9b16504bb5e22c3d311828a704819242f949797a Mon Sep 17 00:00:00 2001 From: liuheng Date: Thu, 29 Jun 2023 19:49:37 +0800 Subject: [PATCH 43/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8D=87=E7=BA=A7?= =?UTF-8?q?=E5=90=8E=EF=BC=8C=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/common/Common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/gspylib/common/Common.py b/script/gspylib/common/Common.py index 220cd80c..c8297f51 100644 --- a/script/gspylib/common/Common.py +++ b/script/gspylib/common/Common.py @@ -1058,11 +1058,11 @@ class DefaultValue(): # clean version FileUtil.deleteLine(userProfile, "^\\s*export\\" "s*GAUSS_VERSION=.*$") - if (cleanLD_LIBRARY): - # clean lib - FileUtil.deleteLine(userProfile, + FileUtil.deleteLine(userProfile, "^\\s*export\\s*LD_LIBRARY_PATH=\\" "$GAUSSHOME\\/lib:\\$LD_LIBRARY_PATH$") + if (cleanLD_LIBRARY): + # clean lib FileUtil.deleteLine(userProfile, "^\\s*export\\s*LD_LIBRARY_PATH=\\" "$GAUSSHOME\\/lib\\/libsimsearch:\\" -- Gitee From eb7c9cac2741fbdd41a2809d79bba47175b5c7a8 Mon Sep 17 00:00:00 2001 From: shenzheng4 Date: Thu, 15 Jun 2023 20:32:30 +0800 Subject: [PATCH 44/96] xlog store in single lun in shared storage --- script/gspylib/component/DSS/dss_checker.py | 8 ++++-- script/gspylib/component/DSS/dss_comp.py | 28 ++++++++++++++++++- .../component/Kernel/DN_OLAP/DN_OLAP.py | 9 +++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/script/gspylib/component/DSS/dss_checker.py b/script/gspylib/component/DSS/dss_checker.py index cfb81b4b..a9ae0249 100644 --- a/script/gspylib/component/DSS/dss_checker.py +++ b/script/gspylib/component/DSS/dss_checker.py @@ -70,10 +70,14 @@ class DssConfig(): inst.dss_config = str( DssConfig((dss_ids, dss_ips, dss_ports), offset=10)) infos = list(filter(None, re.split(r':|,', inst.dss_vg_info))) - if len(infos[::2]) != len(dss_ips) + 1: + + # We support two deployment method： + # 1. one dss disk for xlog of each node, and one dss disk for shared data; + # 2. one dss disk for xlogs of all nodes, and one dss disk for shared data; + if (len(infos[::2]) != len(dss_ips) + 1) and (len(infos[::2]) != 2): raise Exception( ErrorCode.GAUSS_500['GAUSS_50026'] % 'dss_vg_info' + - ' The number of volumes is one more than the number of dns.' + + ' The number of volumes is one more than the number of dns or the number of volumes is 2.' + ' The number of dns is {} and the number of dss volumes is {}'. format(len(dss_ips), len(infos[::2]))) for dp in dss_ports: diff --git a/script/gspylib/component/DSS/dss_comp.py b/script/gspylib/component/DSS/dss_comp.py index 552d71e1..9142106b 100644 --- a/script/gspylib/component/DSS/dss_comp.py +++ b/script/gspylib/component/DSS/dss_comp.py @@ -64,13 +64,39 @@ class DssInst(): else: raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % self.cfg_path) return items + + @staticmethod + def get_private_vg_num(dss_home): + ''' + Obtaining Private Volumes + ''' + + + vg_cfg = os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini') + if os.path.isfile(vg_cfg): + try: + with open(vg_cfg, "r") as fp: + context = fp.read().strip() + pris = re.findall( + '(.*):/dev/.*private_.*', context) + if pris: + return len(pris) + else: + raise Exception(ErrorCode.GAUSS_504["GAUSS_50416"] % + 'in dss_vg_conf.ini') + except Exception as eds: + raise Exception(ErrorCode.GAUSS_504["GAUSS_50414"] % eds) + else: + raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % vg_cfg) @staticmethod - def get_private_vgname_by_ini(dss_home, dss_id): + def get_private_vgname_by_ini(dss_home, dss_id, xlog_in_one_priv_vg): ''' Obtaining a Private Volume ''' + if xlog_in_one_priv_vg: + dss_id = 0 vg_cfg = os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini') if os.path.isfile(vg_cfg): try: diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index 3f5f5ec3..93cb82b4 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -150,7 +150,14 @@ class DN_OLAP(Kernel): dss_nodes_list = DssConfig.get_value_b64_handler( 'dss_nodes_list', self.dss_config, action='decode') cfg_context = DssInst.get_dms_url(dss_nodes_list) - pri_vgname = DssInst.get_private_vgname_by_ini(dss_home, inst_id) + + xlog_in_one_priv_vg = False + infos = list(filter(None, re.split(r':|:|,', dss_nodes_list))) + pri_vg_num = DssInst.get_private_vg_num(dss_home) + # when use one private vg for xlog, vgname should get from inst_id=0 + if (pri_vg_num < len(infos[::3]) and pri_vg_num == 1): + xlog_in_one_priv_vg = True + pri_vgname = DssInst.get_private_vgname_by_ini(dss_home, inst_id, xlog_in_one_priv_vg) cmd += " -n --vgname=\"{}\" --enable-dss --dms_url=\"{}\" -I {}" \ " --socketpath=\"{}\"".format( "+{},+{}".format(vgname, pri_vgname), cfg_context, inst_id, -- Gitee From 2a6b2f4d732089b7011673ecf799ae47dec54b6f Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Fri, 7 Jul 2023 19:54:40 +0800 Subject: [PATCH 45/96] =?UTF-8?q?=E6=89=A9=E5=AE=B9=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E8=AE=BE=E7=BD=AEhba=E8=AE=A4=E8=AF=81=EF=BC=8Ctrust=E6=96=B9?= =?UTF-8?q?=E5=BC=8F=E6=8C=87=E5=AE=9A=E7=AE=A1=E7=90=86=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=9B=BF=E6=8D=A2all?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/expansion/ExpansionImpl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/impl/expansion/ExpansionImpl.py b/script/impl/expansion/ExpansionImpl.py index a411a101..6e627c66 100644 --- a/script/impl/expansion/ExpansionImpl.py +++ b/script/impl/expansion/ExpansionImpl.py @@ -683,12 +683,12 @@ gs_guc set -D {dn} -c "available_zone='{azName}'" cmd = "source %s;gs_guc set -D %s" % (self.envFile, dataNode) if hostExec in self.existingHosts: for hostParam in self.context.newHostList: - cmd += " -h 'host all all %s/32 trust'" % hostParam + cmd += " -h 'host all %s %s/32 trust'" % (self.user, hostParam) cmd += self.get_add_float_ip_cmd(hostParam) else: for hostParam in allHosts: if hostExec != hostParam: - cmd += " -h 'host all all %s/32 trust'" % hostParam + cmd += " -h 'host all %s %s/32 trust'" % (self.user, hostParam) cmd += self.get_add_float_ip_cmd(hostParam) self.logger.debug("[%s] trustCmd:%s" % (hostExec, cmd)) sshTool = SshTool([hostExec]) -- Gitee From 58e7e1cc958071bafde26d6b01fb37204f950043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E5=87=AF?= Date: Wed, 12 Jul 2023 10:47:06 +0800 Subject: [PATCH 46/96] =?UTF-8?q?fix:I7KI56:=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../inspection/items/os/CheckFirewall.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/script/gspylib/inspection/items/os/CheckFirewall.py b/script/gspylib/inspection/items/os/CheckFirewall.py index dc961476..44714859 100644 --- a/script/gspylib/inspection/items/os/CheckFirewall.py +++ b/script/gspylib/inspection/items/os/CheckFirewall.py @@ -34,22 +34,19 @@ class CheckFirewall(BaseItem): def doCheck(self): (status, output) = g_service.manageOSService("firewall", "status") - if (output.find(SUSE_FLAG) > 0 or output.find( - REDHAT6_FLAG) > 0 or output.find(REDHAT7_FLAG) > 0): - firewallStatus = "disabled" - else: - firewallStatus = "enabled" - if (firewallStatus == ""): - self.result.rst = ResultStatus.OK - elif (firewallStatus != EXPECTED_VALUE): - self.result.rst = ResultStatus.NG - else: + firewallStatus = "disabled" if ( + output.find(SUSE_FLAG) > 0 or + output.find(REDHAT6_FLAG) > 0 or + output.find(REDHAT7_FLAG) > 0 + ) else "enabled" + + self.result.raw = output + self.result.val = firewallStatus + + if firewallStatus == EXPECTED_VALUE: self.result.rst = ResultStatus.OK - if (not self.result.raw): - self.result.raw = output else: - self.result.raw = output - self.result.val = firewallStatus + self.result.rst = ResultStatus.NG if firewallStatus else ResultStatus.OK def doSet(self): if g_Platform.isPlatFormEulerOSOrRHEL7X(): -- Gitee From 69cdb26ee31a98df3979f2542e5a0acc01fdc6c3 Mon Sep 17 00:00:00 2001 From: gentle_hu Date: Wed, 12 Jul 2023 14:49:29 +0800 Subject: [PATCH 47/96] remove upgrade-checker reademe --- script/upgrade_checker/README.md | 105 ------------------------------- 1 file changed, 105 deletions(-) delete mode 100644 script/upgrade_checker/README.md diff --git a/script/upgrade_checker/README.md b/script/upgrade_checker/README.md deleted file mode 100644 index 635fae07..00000000 --- a/script/upgrade_checker/README.md +++ /dev/null @@ -1,105 +0,0 @@ -# upgrade-checker - -## 简介 -upgrade-checker是一个升级之后元数据完整性的检测工具，由python3编写，在数据库升级之后，对元数据进行检验，检查在升级过程前后，是否出现元数据的损坏。目前仅支持检测builtin、initdb的元数据，对于用户所创建的表等，暂时不会进行检测。 - -主要功能如下： -- **导出**：基于某个数据库，生成一份元数据校验地图。 -- **校验**：基于某个元数据校验地图，对数据库进行元数据的校验，并生成报告。 - - -## 如何使用 -在安装了og的环境上，使用如下命令运行此工具： -```angular2html -python3 gs_upgradechk [ action [ config-params, ... ] ] -``` -其中action支持如下参数选项： -```angular2html -export 导出 -check | verify 校验 -help | -h | -? | --help 帮助 - -默认值为 help, 不区分大小写 -``` - -config-params支持的参数选项及其默认值如下 -```config --p | --port : 16666 --F | --report-format : md --M | --report-mode : summary --v | --vmap : '' --d | --debug -``` - -- port：数据库端口号。 -- debug：会打印更多的日志，用于工具问题定位。无参数，默认关闭，指定时开启。 -- report-format：校验报告格式，当前仅支持markdown。 -- report-mode：校验报告详细程度，支持`summary`，`detail`两种模式，默认`summary`模式。此选项仅会影响生成的报告的详细程度，并不影响校验内容的多少，`detail`仅是会把所有的校验规则无论对错都整理输出，报告整理比较耗时，而`summary`则仅整理输出错误的内容。 -- vmap：指定的元数据校验地图。若指定了，则在校验时会使用指定vmap，否则会自动检测目标数据库版本并在网上下载和使用我们生成好的vmap。 - -默认标准vmap下载地址如下： -```angular2html -https://opengauss.obs.cn-south-1.myhuaweicloud.com/upgrade_checker/standard_meta_verify_map_XX_X.X.X.vmap -``` -其中`XX`表示工具版本，`X.X.X`表示openGauss版本 - -## 版本支持 -工具版本与对应所支持的openGauss版本如下： - - - - - - - - - - - - - - - - - - - - -

工具版本

版本git id

支持openGauss

最新

-

3.0.3+，5.0.X，5.1.X

5010000

-

3.0.3+，5.0.X，5.1.X

- -其中：`X`代表通配符，即全系列产品；`N+`代表范围，即N和之后的版本。 - -在openGauss升级时，按照新数据库的版本选择本工具版本 - - -## 工具结构 -```dir introduce -upgrade-checker - |d-- utils 工具代码-通用组件 - |d-- style 工具代码-报告风格 - |d-- rules 工具代码-规则组件 - |f-- project.py 工具代码 - |f-- .. 工具代码 - |d-- dev 开发者便携脚本 - |d-- workspace 工具工作目录 - |d-- vmaps -基准校验地图存放位置 - |d-- project_name-workspace -某次检测的运行工作目录 - |f-- run.log -运行日志 - |d-- results 工具结果目录 - |f- user-created-vmap -用户自己生成的校验地图 - |f- report -某次检测的结果报告 - |f-- README.md 说明书 -|f-- gs_upgradechk 入口脚本 -``` - -## 校验原理 -数据库元数据，实际就是对一张张系统表以及其数据、gaussdb二进制内硬编码的部分数据等。 -在升级过程中，切换二进制后，硬编码内容部分也随之切换。因此对于元数据的校验，仅需要校验系统表及其数据即可。 - -对于系统表和数据的校验，最直接的便是通过查询语句进行查询。 -那么本工具实际上就是一个：执行一系列查询语句，并分析结果是否符合预期，并整理成报告的工具。 - -而在上述定义中涉及到的三个点：查询语句、预期结果、报告，分别如下定义： -- **规则(Rule)**：校验某个项目的一条查询语句。这条查询语句需要的查询结果需要按照key-value进行组织，其中key的作用是可以让我们更清楚地知道查询结果的每一列都是什么，更方便的进行整理分析。例如： `select oid as key, relname as val from pg_class where oid < 10000`。同时尽可能地区分builtin、initdb、user的数据。 -- **元数据校验地图(VerifyMap)**: 规则与其预期输出的集合。 -- **报告(Report)**: 规则执行结果与预期结果的对比分析结论所形成的汇总。 -- Gitee From 7f987818c8a91bd856e82e12bd0a58734995400b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E5=87=AF?= Date: Fri, 14 Jul 2023 17:17:16 +0800 Subject: [PATCH 48/96] =?UTF-8?q?fix:I7L08E=20=E5=A2=9E=E5=8A=A0=E5=88=A4?= =?UTF-8?q?=E6=96=AD=E5=8D=95=E6=9C=BA=E6=97=B6=E5=88=A4=E6=96=ADip?= =?UTF-8?q?=E6=98=AF=E5=90=A6=E4=B8=BA=E6=9C=AC=E6=9C=BAip=20=E7=84=B6?= =?UTF-8?q?=E5=90=8E=E7=BB=99localMode=E8=B5=8B=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_check | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/gs_check b/script/gs_check index da4440d5..54f19424 100644 --- a/script/gs_check +++ b/script/gs_check @@ -770,6 +770,11 @@ def parseCheckContext(): if g_context.isCached(): return g_logger.debug("Start to parse the check items config file") + + localHost = __getLocalNode(g_context.nodes) + if len(g_context.nodes)==1 and localHost == g_context.nodes[0]: + g_opts.localMode = True + items_all = [] items_oldNode = [] items_newNode = [] -- Gitee From 92fa692a1b865ce4cbf74590181e4fe2bb9c04be Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Tue, 18 Jul 2023 10:33:12 +0800 Subject: [PATCH 49/96] =?UTF-8?q?=E9=80=BB=E8=BE=91=E5=88=A4=E6=96=AD?= =?UTF-8?q?=E5=8A=A0=E5=9B=BA=EF=BC=8C=E9=81=BF=E5=85=8D=E5=87=BA=E7=8E=B0?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/local/Resetreplconninfo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/local/Resetreplconninfo.py b/script/local/Resetreplconninfo.py index 2c7f7447..b2759751 100644 --- a/script/local/Resetreplconninfo.py +++ b/script/local/Resetreplconninfo.py @@ -187,6 +187,8 @@ class Resetreplconninfo(): cmd + " Error:\n%s" % output) # get remote ip and check iscascade replinfo_all = output.split('\n')[-2].strip().split("'") + if len(replinfo_all) < 2: + continue replinfo_value = replinfo_all[1].split() for remoteip in remote_ip_dict: if remoteip in replinfo_all[1]: -- Gitee From 83d916092b43eed3fd4c8bc17c6d57763eb43330 Mon Sep 17 00:00:00 2001 From: hemny Date: Thu, 20 Jul 2023 12:27:34 +0800 Subject: [PATCH 50/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=BA=E9=99=B7-I7I5?= =?UTF-8?q?OG=EF=BC=9ANoF+=E5=8D=8F=E8=AE=AE=EF=BC=8Cgs=5Fpreinstall?= =?UTF-8?q?=E9=A2=84=E5=AE=89=E8=A3=85=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/base_utils/os/disk_util.py | 45 ++++++++++++++++++++++-- script/gspylib/common/Common.py | 1 + script/gspylib/component/DSS/dss_comp.py | 28 ++++++++++----- script/local/PreInstallUtility.py | 12 +++---- 4 files changed, 69 insertions(+), 17 deletions(-) diff --git a/script/base_utils/os/disk_util.py b/script/base_utils/os/disk_util.py index a6a146eb..722588df 100644 --- a/script/base_utils/os/disk_util.py +++ b/script/base_utils/os/disk_util.py @@ -256,8 +256,49 @@ class DiskUtil(object): return output.split('\n') @staticmethod - def get_scsi_dev_id(dev_name): - cmd = '/lib/udev/scsi_id -g -u {}'.format(dev_name) + def get_disk_dev_id(dev_name): + cmd = 'udevadm info -q symlink {}'.format(dev_name) + sts, out = CmdUtil.getstatusoutput_by_fast_popen(cmd) + if sts not in [0]: + raise Exception(ErrorCode.GAUSS_504["GAUSS_50422"] % + str(out).strip()) + + if out.find('disk/by-id/scsi-') != -1: + strtmp = out[out.index('disk/by-id/scsi-') + len('disk/by-id/scsi-'):] + if out.find(' ') != -1: + strtmp = strtmp.split()[0] + return strtmp.strip() + + if out.find('disk/by-id/ultrapath') != -1: + strtmp = out[out.index('disk/by-id/ultrapath') + len('disk/by-id/ultrapath'):] + if out.find(' ') != -1: + strtmp = strtmp.split()[0] + strtmp = strtmp.split('-', 1)[1] + return strtmp.strip() + + if out.find('disk/by-id/wwn-') != -1: + strtmp = out[out.index('disk/by-id/wwn-') + len('disk/by-id/wwn-'):] + if out.find(' ') != -1: + strtmp = strtmp.split()[0] + return strtmp.strip() + + if out.find('disk/by-id/ata-') != -1: + strtmp = out[out.index('disk/by-id/ata-') + len('disk/by-id/ata-'):] + if out.find(' ') != -1: + strtmp = strtmp.split()[0] + return strtmp.strip() + + if out.find('disk/by-id/dm-uuid-') != -1: + strtmp = out[out.index('disk/by-id/dm-uuid-') + len('disk/by-id/dm-uuid-'):] + if out.find(' ') != -1: + strtmp = strtmp.split()[0] + return strtmp.strip() + + return out.strip() + + @staticmethod + def get_disk_dev_name(dev_name): + cmd = 'udevadm info -q name {}'.format(dev_name) sts, out = CmdUtil.getstatusoutput_by_fast_popen(cmd) if sts not in [0]: raise Exception(ErrorCode.GAUSS_504["GAUSS_50422"] % diff --git a/script/gspylib/common/Common.py b/script/gspylib/common/Common.py index c8297f51..2d1016fa 100644 --- a/script/gspylib/common/Common.py +++ b/script/gspylib/common/Common.py @@ -233,6 +233,7 @@ class DefaultValue(): GREY_UPGRADE_STEP_UPGRADE_PROCESS = 3 CAP_WIO = "CAP_SYS_RAWIO" + CAP_ADM = "CAP_SYS_ADMIN" # env parameter MPPRC_FILE_ENV = "MPPDB_ENV_SEPARATE_PATH" diff --git a/script/gspylib/component/DSS/dss_comp.py b/script/gspylib/component/DSS/dss_comp.py index 552d71e1..e78d2394 100644 --- a/script/gspylib/component/DSS/dss_comp.py +++ b/script/gspylib/component/DSS/dss_comp.py @@ -376,6 +376,8 @@ class Udev(): def __init__(self, attr='', **kwargs): # uuid soft_link_name user, group + dev_id = '' + dev_name = '' if shutil.which('hot_add'): self.KERNEL = 'sd*' else: @@ -384,12 +386,19 @@ class Udev(): self.SUBSYSTEM = 'block' self.RESULT, self.SYMLINK, self.OWNER, self.GROUP = '', '', '', '', if attr: - self.RESULT, self.SYMLINK, self.OWNER, self.GROUP = attr - self.PROGRAM = ' '.join([ - '/lib/udev/scsi_id', '--whitelisted', '--replace-whitespace', - '--device=/dev/$name' - ]) + dev_id, dev_name, self.SYMLINK, self.OWNER, self.GROUP = attr + self.PROGRAM = ' '.join(['/usr/bin/udevadm', 'info', '-q', 'symlink', '/dev/$name']) + self.RESULT = '*{}*'.format(dev_id) self.MODE = '0660' + if dev_name.startswith('sd'): + self.KERNEL = 'sd*' + elif dev_name.startswith('nvme'): + self.KERNEL = 'nvme*' + elif dev_name.startswith('ultrapath'): + self.KERNEL = 'ultrapath*' + elif dev_name.startswith('dm-'): + self.KERNEL = 'dm-*' + for key, val in kwargs.items(): setattr(self, key, val) @@ -424,9 +433,10 @@ class UdevContext(): DSS_UDEV_NAME = 'zz-dss_%s.rules' DSS_UDEV_DIR = '/etc/udev/rules.d/' - def __init__(self, identity, db_info, uuid_getter=''): + def __init__(self, identity, db_info, uuid_getter='', devname_getter=''): self.user, self.group = identity self.uuid_getter = uuid_getter + self.devname_getter = devname_getter self.db_info = db_info @property @@ -489,16 +499,16 @@ class UdevContext(): if isinstance(info, dict) and key == 'dss_pri_disks': for dss_id, phy_disk in enumerate(info.values()): yield str( - Udev((self.uuid_getter(phy_disk), alias % (str(dss_id)), + Udev((self.uuid_getter(phy_disk), self.devname_getter(phy_disk), alias % (str(dss_id)), self.user, self.group))) elif isinstance(info, dict): # shared disk for phy_disk in info.values(): yield str( - Udev((self.uuid_getter(phy_disk), alias, self.user, + Udev((self.uuid_getter(phy_disk), self.devname_getter(phy_disk), alias, self.user, self.group))) elif isinstance(info, str): # the disk used by cm yield str( Udev( - (self.uuid_getter(info), alias, self.user, self.group))) + (self.uuid_getter(info), self.devname_getter(phy_disk), alias, self.user, self.group))) diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index c1f42d62..e3f96395 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -922,7 +922,7 @@ Common options: self.logger.debug("Creating dss disk link.") context = list( UdevContext((self.user, self.group), self.clusterInfo, - DiskUtil.get_scsi_dev_id)) + DiskUtil.get_disk_dev_id, DiskUtil.get_disk_dev_name)) self.logger.debug("Checking dss udev directory.") if os.path.isdir(UdevContext.DSS_UDEV_DIR): @@ -931,7 +931,7 @@ Common options: for disk in UdevContext.get_all_phy_disk(self.clusterInfo): cmd = "cd %s; grep -iro '%s' | grep -v grep | grep -v '%s'" % ( - UdevContext.DSS_UDEV_DIR, DiskUtil.get_scsi_dev_id(disk), + UdevContext.DSS_UDEV_DIR, DiskUtil.get_disk_dev_id(disk), UdevContext.DSS_UDEV_NAME % self.user) sts, out = subprocess.getstatusoutput(cmd) if sts == 1: @@ -2697,10 +2697,10 @@ Common options: FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, os.path.join(clib_app, file_)) - caps = ['perctrl', 'cm_persist'] - for file_ in caps: - FileUtil.change_caps(DefaultValue.CAP_WIO, - os.path.join(clib_app, file_)) + FileUtil.change_caps(DefaultValue.CAP_WIO, + os.path.join(clib_app, 'cm_persist')) + FileUtil.change_caps('{},{}'.format(DefaultValue.CAP_ADM, DefaultValue.CAP_WIO), + os.path.join(clib_app, 'perctrl')) self.logger.debug("Successfully modified dss cap permissions.") def fix_dss_dir_permission(self): -- Gitee From 6db3c53cd6ec7cff9019a3aa41d3f3da44b44a07 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Mon, 24 Jul 2023 21:39:04 +0800 Subject: [PATCH 51/96] =?UTF-8?q?=E4=BF=AE=E6=94=B9cm=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E6=97=B6=E5=80=99=E5=8F=AA=E6=8B=89=E8=B5=B7=E4=B8=80=E6=AC=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/component/CM/CM_OLAP/CM_OLAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/gspylib/component/CM/CM_OLAP/CM_OLAP.py b/script/gspylib/component/CM/CM_OLAP/CM_OLAP.py index ecd0707d..219288be 100644 --- a/script/gspylib/component/CM/CM_OLAP/CM_OLAP.py +++ b/script/gspylib/component/CM/CM_OLAP/CM_OLAP.py @@ -304,7 +304,7 @@ class CM_OLAP(CM): self.logger.log("======================================================================") # Call cm_ctl to start the cmd = CM_OLAP.get_start_cmd(nodeId, timeout=timeout, datadir=datadir, azName=azName) - result_set = CmdUtil.retryGetstatusoutput(cmd, retry_time=retry_times) + result_set = subprocess.getstatusoutput(cmd) # The output prompts when the failure to start if result_set[0] != 0: self.logger.error(ErrorCode.GAUSS_516["GAUSS_51607"] % start_type + -- Gitee From bc7c44b57d445b4bf1c4d2403e16e1d1a32c46c7 Mon Sep 17 00:00:00 2001 From: liuheng Date: Wed, 26 Jul 2023 10:47:28 +0800 Subject: [PATCH 52/96] =?UTF-8?q?gs=5Fpostuninstall=E5=88=A0=E9=99=A4cgrou?= =?UTF-8?q?p=E6=8E=A7=E5=88=B6=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/postuninstall/PostUninstallImpl.py | 25 ++++++++++++++++++- script/local/PreInstallUtility.py | 11 ++++++++ script/local/UnPreInstallUtility.py | 24 ++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/script/impl/postuninstall/PostUninstallImpl.py b/script/impl/postuninstall/PostUninstallImpl.py index 696cae90..326a4c51 100644 --- a/script/impl/postuninstall/PostUninstallImpl.py +++ b/script/impl/postuninstall/PostUninstallImpl.py @@ -60,7 +60,7 @@ ACTION_CLEAN_GAUSS_ENV = "clean_gauss_env" ACTION_DELETE_GROUP = "delete_group" ACTION_CLEAN_SYSLOG_CONFIG = 'clean_syslog_config' ACTION_CLEAN_DEPENDENCY = "clean_dependency" - +ACTION_DELETE_CGROUP = "delete_cgroup" class PostUninstallImpl: """ @@ -118,6 +118,8 @@ class PostUninstallImpl: try: # check uninstall self.checkUnPreInstall() + # clean cgroup + self.clean_cgroup() # clean app/log/data/temp dirs self.cleanDirectory() # clean other user @@ -170,6 +172,27 @@ class PostUninstallImpl: self.mpprcFile) self.logger.log("Successfully checked unpreinstallation.") + def clean_cgroup(self): + """ + function: clean cgroup + input : NA + output: NA + """ + self.logger.log("clean cgroup") + + cmd = "%s -t %s -u %s -l '%s' -X '%s'" % ( + OMCommand.getLocalScript("Local_UnPreInstall"), + ACTION_DELETE_CGROUP, + self.user, + self.localLog, + self.xmlFile) + self.logger.debug("Command for clean cgroup: %s" % cmd) + # check if do postuninstall in all nodes + CmdExecutor.execCommandWithMode(cmd, + self.sshTool, self.localMode, + self.mpprcFile) + self.logger.log("Successfully clean cgroup.") + def cleanDirectory(self): """ function: clean install/instance/temp dirs diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index e3f96395..0e011460 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -2586,6 +2586,17 @@ Common options: os.makedirs(dest_path) FileUtil.changeOwner("root", dest_path) + # cp cgroup to /root/gauss_om/xxx + self.logger.debug("cp cgroup to /root/gauss_om/xxx.") + root_lib_dir = os.path.join(dest_path, "lib") + root_bin_dir = os.path.join(dest_path, "bin") + FileUtil.createDirectory(root_lib_dir, mode=DefaultValue.KEY_DIRECTORY_MODE) + FileUtil.createDirectory(root_bin_dir, mode=DefaultValue.KEY_DIRECTORY_MODE) + libcgroup_dir = os.path.realpath("%s/libcgroup/lib/libcgroup.so*" % package_path) + cgroup_exe_dir = os.path.realpath("%s/libcgroup/bin/gs_cgroup" % package_path) + cp_cmd = "cp -rf %s %s; cp -rf %s %s" % (libcgroup_dir, root_lib_dir, cgroup_exe_dir, root_bin_dir) + CmdExecutor.execCommandLocally(cp_cmd) + # cp $GPHOME script lib to /root/gauss_om/xxx cmd = ("cp -rf %s/script %s/lib %s/version.cfg %s" % (self.clusterToolPath, self.clusterToolPath, diff --git a/script/local/UnPreInstallUtility.py b/script/local/UnPreInstallUtility.py index b1fafe7d..66be050d 100644 --- a/script/local/UnPreInstallUtility.py +++ b/script/local/UnPreInstallUtility.py @@ -46,6 +46,7 @@ ACTION_CLEAN_TOOL_ENV = 'clean_tool_env' ACTION_CHECK_UNPREINSTALL = "check_unpreinstall" ACTION_CLEAN_GAUSS_ENV = "clean_gauss_env" ACTION_DELETE_GROUP = "delete_group" +ACTION_DELETE_CGROUP = "delete_cgroup" # clean instance paths ACTION_CLEAN_INSTANCE_PATHS = "clean_instance_paths" # clean $GAUSS_ENV @@ -477,6 +478,27 @@ class Postuninstall(LocalBaseOM): self.logger.debug("Cleaned $GAUSS_ENV.") + def clean_cgroup(self): + """ + function: clean cgroup + input : NA + output: NA + """ + self.logger.debug("Cleaning user cgroup.") + # mkdir gauss_home dir + gsom_path = DefaultValue.ROOT_SCRIPTS_PATH + root_lib_dir = "%s/%s/lib/" % (gsom_path, self.user) + root_bin_dir = "%s/%s/bin/" % (gsom_path, self.user) + + # delete cgroup + cmd = "export LD_LIBRARY_PATH=%s:\$LD_LIBRARY_PATH && %s/gs_cgroup -d -U %s" % (root_lib_dir, root_bin_dir, self.user) + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + self.logger.logExit( + "Error: Failed to delete cgroup " + "cmd:%s. Error: \n%s" % (cmd, output)) + self.logger.debug("Successfully cleaned user cgroup.") + def cleanGroup(self): """ function: clean group @@ -603,6 +625,8 @@ class Postuninstall(LocalBaseOM): self.cleanGaussEnv() elif self.action == ACTION_DELETE_GROUP: self.cleanGroup() + elif self.action == ACTION_DELETE_CGROUP: + self.clean_cgroup() elif self.action == ACTION_CLEAN_DEPENDENCY: self.cleanScript() elif self.action == ACTION_CLEAN_ENV: -- Gitee From 7a872780aaa5c11ededbe24c8d850d9133b17da9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E5=87=AF?= Date: Thu, 27 Jul 2023 10:43:43 +0800 Subject: [PATCH 53/96] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8DI7OCXM=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E5=BB=BA=E8=AE=AE=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_check | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/gs_check b/script/gs_check index 54f19424..486a57d5 100644 --- a/script/gs_check +++ b/script/gs_check @@ -780,6 +780,7 @@ def parseCheckContext(): items_newNode = [] failedItems = [] singleSkipList = [] + rootItems = [] # generate the items from scene configuration if g_opts.scene: items_oldNode, failedItems = __parseScene(g_opts.scene) @@ -800,6 +801,7 @@ def parseCheckContext(): g_context.rootItems.append(item) if g_opts.skipRootItems and item['permission'] == 'root': items_all.remove(item) + rootItems.append(item['name']) continue if item['permission'] == 'root': g_context.rootItems.append(item) @@ -818,6 +820,8 @@ def parseCheckContext(): raise CheckException("GAUSS-53061", "The --routing is required when cluster dosen't exist") g_context.items.append(item) + if len(rootItems) != 0: + g_logger.info("Warning: These check items [%s] belong to root item" % (",".join(rootItems))) if len(singleSkipList) != 0: __printOnScreen( "The following items are skipped when the type of cluster is" -- Gitee From eda21448fb019e43ddd8579468bcc8a067484776 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Thu, 27 Jul 2023 20:27:00 +0800 Subject: [PATCH 54/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8F=96=E5=8C=85?= =?UTF-8?q?=E5=90=8D=E5=A4=A7=E5=B0=8F=E5=86=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/os/gsplatform.py | 52 ++++++++++++++++++++++++++- script/os_platform/linux_platform.py | 53 ++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/script/gspylib/os/gsplatform.py b/script/gspylib/os/gsplatform.py index 1e12c307..7e513d7a 100644 --- a/script/gspylib/os/gsplatform.py +++ b/script/gspylib/os/gsplatform.py @@ -1566,11 +1566,61 @@ class LinuxPlatform(GenericPlatform): fileName = os.path.normpath(fileName) if not os.path.exists(fileName): - raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % fileName) + try: + directory_to_search = os.path.join(dirName, "./../../../") + matched_files = self.compare_files_in_directory(directory_to_search, fileName) + + if len(matched_files) == 1: + return matched_files[0] + elif len(matched_files) > 1: + print("Multiple matching files found," + "please select one:") + + for i, file in enumerate(matched_files, 1): + print(f"{i}. {file}") + + while True: + try: + choice = int(input("Please enter the serial number" + "of the option:")) + if 1 <= choice <= len(matched_files): + return matched_files[choice - 1] + else: + print("Invalid input: Please re-enter") + except ValueError: + print("Invalid input: Please re-enter") + else: + raise Exception("No matching files found in the directory.") + except Exception as e: + raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % fileName) + if not os.path.isfile(fileName): raise Exception(ErrorCode.GAUSS_502["GAUSS_50210"] % fileName) return fileName + def compare_files_in_directory(directory, file_to_compare): + """ + function: The function is to recursively search for files in the + specified directory and compare them with the given file, + input: + directory: Directory path to traverse + file_to_compare: The file path to compare with files in the directory + output: + matched_files + """ + matched_files = [] + + for root, dirs, files in os.walk(directory): + for file in files: + file_path = os.path.join(root, file) + if os.path.isfile(file_path) and file_to_compare.lower() == file_path.lower(): + matched_files.append(file_path) + + if matched_files: + return matched_files + else: + raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % file_to_compare) + def setKeyValueInSshd(self, key, value): """ function: Set a (key, value) pair into /etc/ssh/sshd_config, diff --git a/script/os_platform/linux_platform.py b/script/os_platform/linux_platform.py index a46ed0a4..4e4c4129 100644 --- a/script/os_platform/linux_platform.py +++ b/script/os_platform/linux_platform.py @@ -243,7 +243,60 @@ class LinuxPlatform(object): package_name_list.append(package_name) if os.path.exists(file_name) and os.path.isfile(file_name): return file_name + + for file_name in file_name_list: + try: + directory_to_search = os.path.join(dir_name, "./../../../") + matched_files = self.compare_files_in_directory(directory_to_search, file_name) + + if len(matched_files) == 1: + return matched_files[0] + elif len(matched_files) > 1: + print("Multiple matching files found," + "please select one:") + + for i, file in enumerate(matched_files, 1): + print(f"{i}. {file}") + + while True: + try: + choice = int(input("Please enter the serial number" + "of the option:")) + if 1 <= choice <= len(matched_files): + return matched_files[choice - 1] + else: + print("Invalid input: Please re-enter") + except ValueError: + print("Invalid input: Please re-enter") + else: + raise Exception("No matching files found in the directory.") + except Exception as e: + raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % file_name) + raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % package_name_list) + + def compare_files_in_directory(directory, file_to_compare): + """ + function: The function is to recursively search for files in the + specified directory and compare them with the given file, + input: + directory: Directory path to traverse + file_to_compare: The file path to compare with files in the directory + output: + matched_files + """ + matched_files = [] + + for root, dirs, files in os.walk(directory): + for file in files: + file_path = os.path.join(root, file) + if os.path.isfile(file_path) and file_to_compare.lower() == file_path.lower(): + matched_files.append(file_path) + + if matched_files: + return matched_files + else: + raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % file_to_compare) def getGrepCmd(self): """ -- Gitee From 6017b661e1fcd8f4cc24ad2feea8d8bd2a0a5b72 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Fri, 28 Jul 2023 10:26:46 +0800 Subject: [PATCH 55/96] =?UTF-8?q?=E5=A2=9E=E5=8A=A0openssl=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 22222 --- script/domain_utils/security/random_value.py | 2 +- script/gspylib/common/ErrorCode.py | 39 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/script/domain_utils/security/random_value.py b/script/domain_utils/security/random_value.py index 42753996..3c33c8c1 100644 --- a/script/domain_utils/security/random_value.py +++ b/script/domain_utils/security/random_value.py @@ -60,7 +60,7 @@ class RandomValue: ranpwd += output.strip() break if count > 100: - raise Exception(ErrorCode.GAUSS_514["GAUSS_51402"] + cmd_tuple[i]) + raise Exception(ErrorCode.GAUSS_537["GAUSS_53700"] + cmd_tuple[i]) else: ranpwd += out_tuple[i].strip() return ranpwd diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index 036e26d0..7c0f0392 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -1130,6 +1130,45 @@ class ErrorCode(): 'GAUSS_53611': "[GAUSS-53611]: Error information is :\n%s", 'GAUSS_53612': "[GAUSS-53612]: Can not find any catalog in database %s" } + + ########################################################################### + # openssl error + ########################################################################### + GAUSS_537 = { + 'GAUSS_53700': "[GAUSS-53700] : Failed to execute the command: %s.\n" + "This is because the OpenSSL random number generator" + "does not work properly. This may be due to a lack of" + "sufficient entropy in the system, or an issue with " + "the configuration of OpenSSL.\n" + + "The following are commonly used inspection items:\n" + + "1. Increase system entropy pool: If the system lacks" + "sufficient entropy, the randomness of the entropy pool" + "can be increased by increasing system activity. For " + "example, several terminal windows can be opened and" + "random operations can be performed, such as moving " + "the mouse, keyboard input, etc.\n" + + "2. Check OpenSSL configuration: Ensure that the OpenSSL" + "configuration is correct. You can check the OpenSSL " + "configuration files and environment variables to ensure" + "that they point to the correct path and file.\n" + + "3. Update OpenSSL version: If the OpenSSL version is " + "older, there may be some known issues and bugs. Try" + "updating to the latest stable version.\n" + + "4. Check file permissions: Ensure that the directory" + "and file where the certificate is being generated" + "have sufficient permissions so that OpenSSL can generate" + "the certificate.\n" + + "5. Use other random number generators: If the above methods" + "do not solve the problem, you can try to use other random" + "number generators instead of the default OpenSSL random " + "number generators.\n" + } ########################################################################## # gs_expansion -- Gitee From dc2883787e9d90101a2f19edce0d79959ae114d6 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Tue, 8 Aug 2023 16:50:48 +0800 Subject: [PATCH 56/96] this is first commit --- script/gs_checkos | 65 +++++++++++++++----- script/gs_preinstall | 34 +++++++++- script/gspylib/common/ParameterParsecheck.py | 5 +- script/impl/preinstall/PreinstallImpl.py | 18 ++++-- 4 files changed, 99 insertions(+), 23 deletions(-) diff --git a/script/gs_checkos b/script/gs_checkos index 038d6950..85186372 100644 --- a/script/gs_checkos +++ b/script/gs_checkos @@ -154,6 +154,8 @@ class CmdOptions(): self.item_detail = [] self.confFile = "" self.localMode = False + self.skipOSCheck = [] + self.skip_OS_Check = {} ######################################################### @@ -187,7 +189,7 @@ gs_checkos is a utility to check and set cluster OS information. Usage: gs_checkos -? | --help gs_checkos -V | --version - gs_checkos -i ITEM [-f HOSTFILE] [-h HOSTNAME] [-X XMLFILE] [--detail] [-o OUTPUT] [-l LOGFILE] + gs_checkos -i ITEM [-f HOSTFILE] [-h HOSTNAME] [-X XMLFILE] [--detail] [-o OUTPUT] [-l LOGFILE] [--skip-os-check IGNORE] General options: -i Item number. To check all items, enter "-i A". To set all parameters, enter "-i B". @@ -196,6 +198,8 @@ General options: -h Name of the host to connect to. -X Configuration file of the cluster. --detail Show detailed information. + --skip-os-check Whether to skip OS parameter checking. + (The default value is check A1-A14.) -o Save the result to the specified file. -l Path of log file. -? --help Show help information for this utility, and exit the command line mode. @@ -271,6 +275,30 @@ def parseItemOpts(itemList): g_opts.item_detail.append(value) else: GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % 'i') + + +def parseSkipOSCheck(skipOSCheckList): + """ + parse items by value + """ + value = [] + for val in skipOSCheckList: + if (len(val.split(',')) > 1): + for i in val.split(','): + value.append(i) + else: + value.append(val) + + for val in CHECK_ITEMNUMLIST: + g_opts.skip_OS_Check[val] = True + + for val in value: + val = val.strip().upper() + if (val in CHECK_ITEMNUMLIST): + g_opts.skip_OS_Check[val] = False + else: + GaussLog.exitWithError( + ErrorCode.GAUSS_500["GAUSS_50004"] % '--skip-os-check') def parseCommandLine(): @@ -299,6 +327,8 @@ def parseCommandLine(): g_opts.itemstr = ParaDict.get("itemstr") if (ParaDict.__contains__("show_detail")): g_opts.detail = ParaDict.get("show_detail") + if (ParaDict.__contains__("skipOSCheck")): + g_opts.skipOSCheck = ParaDict.get("skipOSCheck") def readHostFile(hostfile): @@ -398,6 +428,11 @@ def checkParameter(): checkItems() parseItemOpts(g_opts.itemstr) + ########################################## + # skipOSCheck + ############################################ + parseSkipOSCheck(g_opts.skipOSCheck) + if (("B" in g_opts.item_detail)): g_opts.set = True else: @@ -429,36 +464,36 @@ def doCheckOS(itemNumber): """ do the checing ation by item """ - if (itemNumber == 'A1'): + if (itemNumber == 'A1'and g_opts.skip_OS_Check["A1"]): checkOSVersion() - elif (itemNumber == 'A2'): + elif (itemNumber == 'A2' and g_opts.skip_OS_Check["A2"]): checkKernelVersion() - elif (itemNumber == 'A3'): + elif (itemNumber == 'A3' and g_opts.skip_OS_Check["A3"]): checkUnicode() - elif (itemNumber == 'A4'): + elif (itemNumber == 'A4' and g_opts.skip_OS_Check["A4"]): checkTimeZone() - elif (itemNumber == 'A5'): + elif (itemNumber == 'A5' and g_opts.skip_OS_Check["A5"]): checkMemoryUsage() - elif (itemNumber == 'A6'): + elif (itemNumber == 'A6' and g_opts.skip_OS_Check["A6"]): checkSysCtlParameter() - elif (itemNumber == 'A7'): + elif (itemNumber == 'A7' and g_opts.skip_OS_Check["A7"]): checkFileSystemConfigure() - elif (itemNumber == 'A8'): + elif (itemNumber == 'A8' and g_opts.skip_OS_Check["A8"]): checkDiskConfigure() - elif (itemNumber == 'A9'): + elif (itemNumber == 'A9' and g_opts.skip_OS_Check["A9"]): checkBlockDevConfigure() checkLogicalBlock() - elif (itemNumber == 'A10'): + elif (itemNumber == 'A10' and g_opts.skip_OS_Check["A10"]): checkIOrequestqueue() checkMaxAsyIOrequests() checkIOConfigure() - elif (itemNumber == 'A11'): + elif (itemNumber == 'A11' and g_opts.skip_OS_Check["A11"]): checkNetworkConfigure() - elif (itemNumber == 'A12'): + elif (itemNumber == 'A12' and g_opts.skip_OS_Check["A12"]): checkTimeConsistency() - elif (itemNumber == 'A13'): + elif (itemNumber == 'A13' and g_opts.skip_OS_Check["A13"]): checkFirewallService() - elif (itemNumber == 'A14'): + elif (itemNumber == 'A14' and g_opts.skip_OS_Check["A14"]): checkTHPService() def doSetOS(itemNumber): diff --git a/script/gs_preinstall b/script/gs_preinstall index aad85982..4ea54e28 100644 --- a/script/gs_preinstall +++ b/script/gs_preinstall @@ -110,6 +110,8 @@ from base_utils.os.user_util import UserUtil # Global variables ############################################################################# userNameFirtChar = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0'] +CHECK_ITEMNUMLIST = ['A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'A8', 'A9', + 'A10', 'A11', 'A12', 'A13', 'A14'] class Preinstall(ParallelBaseOM): @@ -125,6 +127,7 @@ class Preinstall(ParallelBaseOM): self.preMode = False self.skipOSSet = False self.skipHostnameSet = False + self.skipOSCheck = [] self.passwordsec = "" self.corePath = "" self.is_new_root_path = False @@ -144,7 +147,7 @@ Usage: gs_preinstall -? | --help gs_preinstall -V | --version gs_preinstall -U USER -G GROUP -X XMLFILE - [-L] [--skip-os-set] [--env-var="ENVVAR" [...]] + [-L] [--skip-os-set] [--skip-os-check] [--env-var="ENVVAR" [...]] [--sep-env-file=ENVFILE] [--skip-hostname-set] [-l LOGFILE] [--non-interactive] [--delete-root-trust] [--unused-third-party] @@ -156,6 +159,8 @@ General options: nodes. --skip-os-set Whether to skip OS parameter setting. (The default value is set.) + --skip-os-check Whether to skip OS parameter checking. + (The default value is check A1-A14.) --env-var="ENVVAR" OS user environment variables. --sep-env-file=ENVFILE Path of the MPP environment file. --skip-hostname-set Whether to skip hostname setting. @@ -223,6 +228,9 @@ General options: # parameter --skip-os-set if (ParaDict.__contains__("skipOSSet")): self.skipOSSet = ParaDict.get("skipOSSet") + # parameter --skip-os-check + if (ParaDict.__contains__("skipOSCheck")): + self.skipOSCheck = ParaDict.get("skipOSCheck") # parameter --non-interactive if (ParaDict.__contains__("preMode")): self.preMode = ParaDict.get("preMode") @@ -315,6 +323,27 @@ General options: GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50213"] % self.logFile) UserUtil.check_path_owner(self.logFile) + + + def checkskipOS(self): + """ + parse items by value + """ + value = [] + for val in self.skipOSCheck: + if (len(val.split(',')) > 1): + for i in val.split(','): + value.append(i) + else: + value.append(val) + + for val in value: + val = val.strip().upper() + if (val in CHECK_ITEMNUMLIST): + continue + else: + GaussLog.exitWithError( + ErrorCode.GAUSS_500["GAUSS_50004"] % '--skip-os-check') def checkMpprcFile(self): @@ -390,6 +419,9 @@ General options: # check log file self.checkLogFile() + + # skip OS check + self.checkskipOS() # set LD_LIBRARY_PATH add local lib def setLibPath(self): diff --git a/script/gspylib/common/ParameterParsecheck.py b/script/gspylib/common/ParameterParsecheck.py index 6a29fc7d..e54f4b22 100644 --- a/script/gspylib/common/ParameterParsecheck.py +++ b/script/gspylib/common/ParameterParsecheck.py @@ -63,7 +63,7 @@ VALUE_CHECK_LIST = ["|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"", "{", # append '=' after long options if it required parameter # no child branch gs_preinstall = ["-?", "--help", "-V", "--version", "-U:", "-G:", "-L", - "--skip-os-set", "-X:", + "--skip-os-set", "-X:", "--skip-os-check", "--env-var=", "--sep-env-file=", "--skip-hostname-set", "-l:", "--non-interactive", "--delete-root-trust", "--unused-third-party"] gs_install = ["-?", "--help", "-V", "--version", "-X:", "-l:", @@ -93,7 +93,7 @@ gs_checkperf = ["-?", "--help", "-V", "--version", "--detail", "-o:", "-i:", "-l:", "-U:"] gs_ssh = ["-?", "--help", "-V", "--version", "-c:"] gs_checkos = ["-?", "--help", "-V", "--version", "-h:", "-f:", "-o:", - "-i:", "--detail", + "-i:", "--detail", "--skip-os-check", "-l:", "-X:"] gs_expansion = ["-?", "--help", "-V", "--version", "-U:", "-G:", "-L", "-X:", "-h:", "--sep-env-file=", "--time-out="] @@ -338,6 +338,7 @@ class Parameter(): "--dbuser": "dbuser", "--nodeId": "nodeId", "--security-mode": "security_mode", + "--skip-os-check": "skipOSCheck", "--cluster-number": "cluster_number" } parameterNeedValue_keys = parameterNeedValue.keys() diff --git a/script/impl/preinstall/PreinstallImpl.py b/script/impl/preinstall/PreinstallImpl.py index 3b25eb12..a4a5c552 100644 --- a/script/impl/preinstall/PreinstallImpl.py +++ b/script/impl/preinstall/PreinstallImpl.py @@ -1038,11 +1038,19 @@ class PreinstallImpl: self.context.logger.debug("Checking OS parameters.") try: # check the OS parameters - cmd = "%s -h %s -i A -l '%s' -X '%s'" % ( - OMCommand.getLocalScript("Gauss_CheckOS"), - namelist, - self.context.localLog, - self.context.xmlFile) + if self.context.skipOSCheck: + cmd = "%s -h %s -i A -l '%s' -X '%s --skip-os-check '%s''" % ( + OMCommand.getLocalScript("Gauss_CheckOS"), + namelist, + self.context.localLog, + self.context.xmlFile, + self.context.skipOSCheck) + else: + cmd = "%s -h %s -i A -l '%s' -X '%s'" % ( + OMCommand.getLocalScript("Gauss_CheckOS"), + namelist, + self.context.localLog, + self.context.xmlFile) (status, output) = subprocess.getstatusoutput(cmd) # if cmd failed, then raise if status != 0 and output.strip() == "": -- Gitee From 74473d5485e5be1e785b29272fde5f08c8d8152f Mon Sep 17 00:00:00 2001 From: z00793368 Date: Thu, 10 Aug 2023 15:14:59 +0800 Subject: [PATCH 57/96] check if the path between dss and datanode is nested --- script/gs_preinstall | 32 +++++++++++++++++++++++++++++- script/gspylib/common/ErrorCode.py | 5 +++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/script/gs_preinstall b/script/gs_preinstall index aad85982..d5cf1040 100644 --- a/script/gs_preinstall +++ b/script/gs_preinstall @@ -93,6 +93,8 @@ from gspylib.common.Common import DefaultValue from gspylib.common.ErrorCode import ErrorCode from gspylib.common.ParallelBaseOM import ParallelBaseOM from gspylib.common.ParameterParsecheck import Parameter +from gspylib.common.DbClusterInfo import dbNodeInfo, \ + dbClusterInfo, compareObject from impl.preinstall.OLAP.PreinstallImplOLAP import PreinstallImplOLAP from gspylib.threads.SshTool import SshTool from domain_utils.cluster_file.cluster_config_file import ClusterConfigFile @@ -544,7 +546,33 @@ General options: sys.path.insert(0, lib_path) if gsom_path == DefaultValue.ROOT_SCRIPTS_PATH: self.is_new_root_path = True - + + def checkIfNest(self): + """ + function: check syncNode + """ + clusterInfo = dbClusterInfo() + clusterInfo.initFromXml(self.xmlFile) + dbNodes = clusterInfo.dbNodes + # get the dss_home path + dss_home_path = clusterInfo.dss_home + + for dbinfo in dbNodes: + if dbinfo is None: + break + datanodes = dbinfo.datanodes + + # check if subdir_path starts with parentdir_path + for datainfo in datanodes: + parentdir_path = os.path.normpath(datainfo.datadir) + subdir_path = os.path.normpath(dss_home_path) + if not parentdir_path.endswith(os.path.sep): + parentdir_path += os.path.sep + if not subdir_path.endswith(os.path.sep): + subdir_path += os.path.sep + + if subdir_path.startswith(parentdir_path): + GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50240"] % dss_home_path) def clearHistTimeFormat(): cmd = "sed -i '/HISTTIMEFORMAT=/d' /etc/profile" @@ -571,6 +599,8 @@ if __name__ == '__main__': preinstall.parseCommandLine() # check parameters preinstall.checkParameter() + # check if the path between dss and datanode is nested + preinstall.checkIfNest() # check expect preinstall.check_expect() # init global variables diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index 7c0f0392..4ab660d2 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -190,8 +190,9 @@ class ErrorCode(): 'GAUSS_50238': "[GAUSS-50238] : Check integrality of bin ", "file %s failed." 'GAUSS_50239': "[GAUSS-50239] : %s should be set in scene config " - "file." - + "file.", + 'GAUSS_50240': "[GAUSS-50240] : %s, it not allowed that directory dss_home_path is a subset" + "of Directory datanode_path." } ########################################################################### -- Gitee From 10426e0a3ece2d797cc45ab74c6b4715ce8c8d8c Mon Sep 17 00:00:00 2001 From: shenzheng4 Date: Mon, 14 Aug 2023 21:04:25 +0800 Subject: [PATCH 58/96] add enable_ss_dorado --- script/gs_install | 3 +++ script/gspylib/common/LocalBaseOM.py | 5 ++++- script/gspylib/common/ParallelBaseOM.py | 1 + script/gspylib/common/ParameterParsecheck.py | 5 ++++- script/gspylib/component/BaseComponent.py | 1 + script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py | 2 ++ script/impl/install/OLAP/InstallImplOLAP.py | 3 ++- script/local/InitInstance.py | 11 ++++++++--- 8 files changed, 25 insertions(+), 6 deletions(-) diff --git a/script/gs_install b/script/gs_install index 17b234f2..69f48ae2 100644 --- a/script/gs_install +++ b/script/gs_install @@ -219,6 +219,9 @@ General options: # parameter --dorado-info if (ParaDict.__contains__("dorado-config")): self.dorado_config = ParaDict.get("dorado-config") + # parameter --dorado-cluster-mode + if (ParaDict.__contains__("dorado-cluster-mode")): + self.dorado_cluster_mode = ParaDict.get("dorado-cluster-mode") def checkUser(self): """ diff --git a/script/gspylib/common/LocalBaseOM.py b/script/gspylib/common/LocalBaseOM.py index 8073f861..6eb8c8a1 100644 --- a/script/gspylib/common/LocalBaseOM.py +++ b/script/gspylib/common/LocalBaseOM.py @@ -50,7 +50,8 @@ class LocalBaseOM(object): paxos_mode=False, dss_mode=False, dss_config="", - dorado_config=""): + dorado_config="", + dorado_cluster_mode=""): ''' Constructor ''' @@ -81,6 +82,7 @@ class LocalBaseOM(object): self.dss_mode = dss_mode self.dss_config = dss_config self.dorado_config = dorado_config + self.dorado_cluster_mode = dorado_cluster_mode def initComponent(self, paxos_mode=False): """ @@ -156,6 +158,7 @@ class LocalBaseOM(object): self.initComponentAttributes(component) component.initParas = self.initParas component.dorado_config = self.dorado_config + component.dorado_cluster_mode = self.dorado_cluster_mode self.dnCons.append(component) def readConfigInfo(self): diff --git a/script/gspylib/common/ParallelBaseOM.py b/script/gspylib/common/ParallelBaseOM.py index 3eac95ff..8a9a6a52 100644 --- a/script/gspylib/common/ParallelBaseOM.py +++ b/script/gspylib/common/ParallelBaseOM.py @@ -81,6 +81,7 @@ class ParallelBaseOM(object): self.dnCons = [] self.dss_cons = [] self.dorado_config = "" + self.dorado_cluster_mode = "" # localMode is same as isSingle in all OM script, expect for # gs_preinstall. # in gs_preinstall, localMode means local mode for master-standby diff --git a/script/gspylib/common/ParameterParsecheck.py b/script/gspylib/common/ParameterParsecheck.py index 6a29fc7d..139d01ec 100644 --- a/script/gspylib/common/ParameterParsecheck.py +++ b/script/gspylib/common/ParameterParsecheck.py @@ -68,7 +68,7 @@ gs_preinstall = ["-?", "--help", "-V", "--version", "-U:", "-G:", "-L", "-l:", "--non-interactive", "--delete-root-trust", "--unused-third-party"] gs_install = ["-?", "--help", "-V", "--version", "-X:", "-l:", "--gsinit-parameter=", "--dn-guc=", "--cms-guc=", - "--time-out=", "--dorado-config=", "--alarm-component="] + "--time-out=", "--dorado-config=", "--dorado-cluster-mode", "--alarm-component="] gs_uninstall = ["-?", "--help", "-V", "--version", "-l:", "-L", "--delete-data"] gs_postuninstall = ["-?", "--help", "-V", "--version", "--delete-user", @@ -303,6 +303,7 @@ class Parameter(): "--alarm-server-addr": "warningserverip", "--time-out": "time_out", "": "", "--dorado-config": "dorado-config", + "--dorado-cluster-mode" :"dorado-cluster-mode", "--alarm-component": "alarm_component", "--SSD-fault-time": "SSDFaultTime", "--begin-time": "begintime", @@ -476,6 +477,8 @@ class Parameter(): PARAMETER_VALUEDICT["upgrade-package"] = value.strip() elif key == "--dorado-config": PARAMETER_VALUEDICT["dorado-config"] = value.strip() + elif key == "--dorado-cluster-mode": + PARAMETER_VALUEDICT["dorado-cluster-mode"] = value.strip() # Only check / symbol for gs_lcct. if key in ("--name", "--nodegroup-name"): self.checkLcGroupName(key, value) diff --git a/script/gspylib/component/BaseComponent.py b/script/gspylib/component/BaseComponent.py index b2fe6f7e..52de2ede 100644 --- a/script/gspylib/component/BaseComponent.py +++ b/script/gspylib/component/BaseComponent.py @@ -56,6 +56,7 @@ class BaseComponent(object): self.dss_mode = '' self.dss_config = '' self.dorado_config = '' + self.dorado_cluster_mode = '' def install(self): pass diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index 93cb82b4..1a065b14 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -166,6 +166,8 @@ class DN_OLAP(Kernel): cmd += " -g %s" % self.dorado_config tmpDict3 = {} tmpDict3["xlog_lock_file_path"] = "'%s/redolog.lock'" % self.instInfo.datadir + if (self.dorado_cluster_mode != ""): + cmd += "--enable-ss-dorado" self.logger.debug("Command for initializing database " "node instance: %s" % cmd) status, output = CmdUtil.retryGetstatusoutput( diff --git a/script/impl/install/OLAP/InstallImplOLAP.py b/script/impl/install/OLAP/InstallImplOLAP.py index 68bf7b31..10b8ea2d 100644 --- a/script/impl/install/OLAP/InstallImplOLAP.py +++ b/script/impl/install/OLAP/InstallImplOLAP.py @@ -290,7 +290,8 @@ class InstallImplOLAP(InstallImpl): elif self.context.clusterInfo.enable_dss == 'on': dss_config = DssConfig.get_value_b64_handler( 'dss_nodes_list', self.context.clusterInfo.dss_config) - cmd += f" --dss_mode --dss_config={dss_config} --dorado_config={self.context.dorado_config}" + cmd += f" --dss_mode --dss_config={dss_config} --dorado_config={self.context.dorado_config} \ + --dorado_cluster_mode={self.context.dorado_cluster_mode}" self.context.logger.debug( "Command for initializing instances: %s" % cmd) diff --git a/script/local/InitInstance.py b/script/local/InitInstance.py index d9aa6459..fdc9a0e0 100644 --- a/script/local/InitInstance.py +++ b/script/local/InitInstance.py @@ -63,6 +63,7 @@ class CmdOptions(): self.dss_mode = False self.dss_config = "" self.dorado_config = "" + self.dorado_cluster_mode = "" def usage(): @@ -80,7 +81,7 @@ def parseCommandLine(): try: opts, args = getopt.getopt(sys.argv[1:], "U:P:G:l:?", [ "help", "dws_mode", "vc_mode", "paxos_mode", "dss_mode", - "dss_config=", "dorado_config=" + "dss_config=", "dorado_config=", "dorado_cluster_mode=" ]) except Exception as e: usage() @@ -115,6 +116,8 @@ def parseCommandLine(): g_opts.dss_config = value.strip() elif key == "--dorado_config": g_opts.dorado_config = value.strip() + elif key == "--dorado_cluster_mode": + g_opts.dorado_cluster_mode = value.strip() Parameter.checkParaVaild(key, value) @@ -182,7 +185,8 @@ class initDbNode(LocalBaseOM): paxos_mode=False, dss_mode=False, dss_config="", - dorado_config = ""): + dorado_config = "", + dorado_cluster_mode = ""): """ function: init instance input : logFile, user, clusterConf, dbInitParams @@ -275,7 +279,8 @@ if __name__ == '__main__': g_opts.paxos_mode, dss_mode=g_opts.dss_mode, dss_config=g_opts.dss_config, - dorado_config=g_opts.dorado_config) + dorado_config=g_opts.dorado_config, + dorado_cluster_mode=g_opts.dorado_cluster_mode) dbInit.initNodeInst(g_opts.vc_mode) except Exception as e: -- Gitee From 36379646348f767160e96643d591fe2d2cbdf11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E5=87=AF?= Date: Tue, 15 Aug 2023 17:01:12 +0800 Subject: [PATCH 59/96] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8DI7T0RB,=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0PGPORT=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/local/PreInstallUtility.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 0e011460..64ce2304 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -751,7 +751,7 @@ Common options: # user exists and input group not exists if userstatus == 0 and groupstatus != 0: self.logger.logExit(ErrorCode.GAUSS_503["GAUSS_50305"] - + " User:Group[%s:%s]" + + " User:Group[%s:%s]" % (self.user, self.group)) # user exists and group exists @@ -1632,6 +1632,11 @@ Common options: datadir = node_info.datanodes[0].datadir FileUtil.writeFile(userProfile, ["export PGDATA=%s" % datadir]) + + # set PGPORT + basePort = node_info.datanodes[0].port + FileUtil.writeFile(userProfile, ["export PGPORT=%d" % basePort]) + # set PATH if userProfile is ClusterConstants.ETC_PROFILE: FileUtil.writeFile(userProfile, [ @@ -1649,6 +1654,8 @@ Common options: "export LD_LIBRARY_PATH=$GPHOME/lib:$LD_LIBRARY_PATH"]) # set PYTHONPATH FileUtil.writeFile(userProfile, ["export PYTHONPATH=$GPHOME/lib"]) + + except Exception as e: self.logger.logExit(str(e)) self.logger.debug("Successfully set tool ENV.") @@ -2596,7 +2603,7 @@ Common options: cgroup_exe_dir = os.path.realpath("%s/libcgroup/bin/gs_cgroup" % package_path) cp_cmd = "cp -rf %s %s; cp -rf %s %s" % (libcgroup_dir, root_lib_dir, cgroup_exe_dir, root_bin_dir) CmdExecutor.execCommandLocally(cp_cmd) - + # cp $GPHOME script lib to /root/gauss_om/xxx cmd = ("cp -rf %s/script %s/lib %s/version.cfg %s" % (self.clusterToolPath, self.clusterToolPath, -- Gitee From 2a60e3a842ef3dba20c7782eb22a9fce312b6577 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Wed, 26 Jul 2023 15:31:35 +0800 Subject: [PATCH 60/96] this is a first commit --- script/gspylib/os/gsplatform.py | 172 ++++++++++++++++++++++++++++ script/local/LocalCheckOS.py | 39 ++++++- script/os_platform/common.py | 1 + script/os_platform/linux_distro.py | 177 +++++++++++++++++++++++++++++ script/osid.conf | 33 ++++++ 5 files changed, 419 insertions(+), 3 deletions(-) create mode 100644 script/osid.conf diff --git a/script/gspylib/os/gsplatform.py b/script/gspylib/os/gsplatform.py index 1e12c307..d496684c 100644 --- a/script/gspylib/os/gsplatform.py +++ b/script/gspylib/os/gsplatform.py @@ -34,6 +34,7 @@ import subprocess import platform import socket import time +import select sys.path.append(sys.path[0] + "/../../") from gspylib.common.ErrorCode import ErrorCode @@ -42,6 +43,10 @@ localDirPath = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, localDirPath + "/../../../lib/netifaces/") from netifaces import interfaces, ifaddresses, AF_INET, AF_INET6 +ISCONFIGURETRUE = "# isConfigure = TRUE" +ISCONFIGUREFALSE = "# isConfigure = FALSE" +SESSIONTIMEOUT = 300 + # ---------------platforms-------------------- # global variable for our platform _supported_dists = ( @@ -76,6 +81,7 @@ SUPPORT_RHEL6X_VERSION_LIST = ["6.4", "6.5", "6.6", "6.7", "6.8", "6.9", "10"] SUPPORT_RHEL7X_VERSION_LIST = ["7.0", "7.1", "7.2", "7.3", "7.4", "7.5", "7.6", "7.7", "7.8", "7.9", "10"] SUPPORT_RHEL8X_VERSION_LIST = ["8.0", "8.1", "8.2", "8.3", "8.4", "8.5"] +SUPPORT_RHEL_LEAST_VERSION = ["6.4"] SUPPORT_RHEL_SERIES_VERSION_LIST = (SUPPORT_RHEL6X_VERSION_LIST + SUPPORT_RHEL7X_VERSION_LIST + SUPPORT_RHEL8X_VERSION_LIST) @@ -149,6 +155,144 @@ def _parse_release_file(firstline): idNum = l[1] return '', version, idNum +def parse_linux_osid(filename): + """ + Tries to determine the name of the Linux OS distribution name. + + The function first looks for a distribution release file in + /etc and then reverts to _dist_try_harder() in case no + suitable files are found. + + Returns a tuple (distname,version,id) which default to the + args given as parameters. + + """ + valid_info = [] + lines_to_choose = [] + with open(filename, 'r') as file: + for line in file: + line = line.strip() + # Ignore comment lines starting with # and empty lines + if line and not line.startswith('#'): + # Separate the release name, version and bitness with spaces + distro, version = line.split() + valid_info.append({ + 'os': distro, + 'version': version + }) + lines_to_choose.append(line.strip()) + if len(lines_to_choose) == 1 or len(lines_to_choose) == 0: + return valid_info + for i, line in enumerate(lines_to_choose, 1): + print(f"{i}. {line}") + + while True: + try: + choice = int(input("Please enter the serial number of the option:")) + if (1 <= choice and choice <= len(lines_to_choose)): + chosen_line = lines_to_choose[choice - 1] + valid_info = valid_info[choice - 1] + with open(filename, 'r+') as file: + lines = file.readlines() + file.seek(0) + file.truncate() + + for line in lines: + if line.strip().startswith('#') or line.strip() == '': + file.write(line) + continue + elif chosen_line in line: + file.write(line) + else: + file.write('#' + line) + write_is_configure_true(filename, ISCONFIGURETRUE) + break + else: + print("Invalid input: Please re-enter") + except ValueError: + print("Invalid input: Please re-enter") + return valid_info + + +def write_is_configure_true(file_path, target_line): + # open the file and read all the lines + with open(file_path, 'r') as file: + lines = file.readlines() + + # Check if any row contains target content + has_target_line = any(target_line in line for line in lines) + + # If there is no target row, insert the target row before row 21 + if not has_target_line: + lines.insert(20, target_line + '\n') + + # Write the modified content back to the file + with open(file_path, 'w') as file: + file.writelines(lines) + + +def parse_linux_distributions(filename): + """ + Tries to determine the name of the Linux OS distribution name. + + The function first looks for a distribution release file in + /etc and then reverts to _dist_try_harder() in case no + + Returns a tuple (distname,version,id) which default to the + args given as parameters. + + """ + is_configure = True + valid_info = parse_linux_osid(filename) + + if len(valid_info) == 1: + write_is_configure_true(filename, ISCONFIGURETRUE) + + with open(filename, 'r') as file: + for line in file: + if ISCONFIGURETRUE in line or ISCONFIGUREFALSE in line: + is_configure = False + + # Remind the user if new content is added to the file + if len(valid_info) == 0 and is_configure: + print(f"File '{filename}' has not been configured yet," + "do you still need to configure it?" + "Enter 'yes' or 'no' to configure " + "the file: ", end='', flush=True) + rlist, _, _ = select.select([sys.stdin], [], [], SESSIONTIMEOUT) + + if rlist: + user_input = input().lower() + else: + user_input = "no" + + while True: + if user_input in ('y', 'yes'): + write_is_configure_true(filename, ISCONFIGURETRUE) + os_name = input("Please enter an operating system name:") + version = input("Please enter the version number:") + with open(filename, 'a') as file: + file.write(f"\n{os_name} {version}\n") + valid_info = parse_linux_osid(filename) + break + elif user_input in ('n', 'no'): + write_is_configure_true(filename, ISCONFIGUREFALSE) + break + else: + write_is_configure_true(filename, ISCONFIGUREFALSE) + break + return valid_info + + +def select_linux_distribution(valid_info): + # If there is no legal information, return None + if not valid_info: + return None + + # If there is only one line of legal information, return directly + if len(valid_info) == 1: + return valid_info[0] + def linux_distribution(distname='', version='', idNum='', supported_dists=_supported_dists, @@ -173,11 +317,39 @@ def linux_distribution(distname='', version='', idNum='', args given as parameters. """ + is_flag_osid = False try: etc = os.listdir('/etc') except os.error: # Probably not a Unix system return distname, version, idNum + + # Read system information from configuration file + # Call the function and pass in the filename + osid_path = os.path.realpath( + os.path.join(os.path.realpath(__file__), "../../../osid.in")) + + if os.path.exists(osid_path): + file_data = parse_linux_distributions(osid_path) + + # Output the parsed content + selected_data = select_linux_distribution(file_data) + + if selected_data: + is_flag_osid = True + + else: + print(f"The file '{osid_path}' does not exist.") + + if is_flag_osid: + if selected_data['os']: + distname = selected_data['os'] + if selected_data['version']: + version = selected_data['version'] + if selected_data['bit']: + idNum = selected_data['bit'] + return distname, version, idNum + sortEtc = sorted(etc) gFile = None for file in sortEtc: diff --git a/script/local/LocalCheckOS.py b/script/local/LocalCheckOS.py index 1fef3d32..6f39fd6f 100644 --- a/script/local/LocalCheckOS.py +++ b/script/local/LocalCheckOS.py @@ -46,7 +46,8 @@ from domain_utils.domain_common.cluster_constants import ClusterConstants from os_platform.linux_distro import LinuxDistro from os_platform.common import SUPPORT_RHEL6X_VERSION_LIST, \ SUPPORT_RHEL7X_VERSION_LIST, SUPPORT_SUSE12X_VERSION_LIST, \ - SUPPORT_SUSE11X_VERSION_LIST, SUPPORT_RHEL8X_VERSION_LIST + SUPPORT_SUSE11X_VERSION_LIST, SUPPORT_RHEL8X_VERSION_LIST, \ + SUPPORT_RHEL_LEAST_VERSION sys.path.insert(0, localDirPath + "/../../lib") import psutil @@ -1786,6 +1787,10 @@ def CheckPlatformInfo(): mixed_type = "%s8" % data.distname platform_str = "%s_%s_%s" % (data.distname, data.version, data.bits) + elif int(data.version[0:3]) >= int(SUPPORT_RHEL_LEAST_VERSION[0]): + mixed_type = "%s" % data.distname + platform_str = "%s_%s_%s" % (data.distname, data.version, + data.bits) else: platform_str = "%s_%s_%s" % (data.distname, data.version, data.bits) @@ -1798,14 +1803,42 @@ def CheckPlatformInfo(): mixed_type = "%s" % data.distname platform_str = "%s_%s_%s" % (data.distname, data.version, data.bits) else: - platform_str = "%s_%s_%s" % (data.distname, data.version, data.bits) - g_logger.log("False unknown %s" % platform_str) + g_logger.log("Warning reason: %s version is not the official version" + "supported by OM, but you can still deploy and install it" % + platform_str) + if ask_to_continue(): + mixed_type = "%s" % data.distname + platform_str = "%s_%s_%s" % (data.distname, data.version, data.bits) + g_logger.log("True %s %s" % (mixed_type, platform_str)) + else: + g_logger.log("False unknown %s" % platform_str) return g_logger.log("True %s %s" % (mixed_type, platform_str)) return +############################################################################# +def ask_to_continue(): + """ + function : Check proceed with the installation + input : NA + output : NA + """ + while True: + response = input("Do you wish to proceed with the installation?" + "(yes/no): ").strip().lower() + if response == 'yes': + print("Executing the operation.") + return True + elif response == 'no': + print("Operation canceled.") + return False + else: + print("Invalid input. Please enter 'yes' or 'no'.") + + + ############################################################################# def CheckUname(): """ diff --git a/script/os_platform/common.py b/script/os_platform/common.py index fa4eaa87..933177ca 100644 --- a/script/os_platform/common.py +++ b/script/os_platform/common.py @@ -40,6 +40,7 @@ SUPPORT_RHEL6X_VERSION_LIST = ["6.4", "6.5", "6.6", "6.7", "6.8", "6.9", "10"] SUPPORT_RHEL7X_VERSION_LIST = ["7.0", "7.1", "7.2", "7.3", "7.4", "7.5", "7.6", "7.7", "7.8", "7.9", "10"] SUPPORT_RHEL8X_VERSION_LIST = ["8.0", "8.1", "8.2", "8.3", "8.4", "8.5"] +SUPPORT_RHEL_LEAST_VERSION = ["6.4"] SUPPORT_RHEL_SERIES_VERSION_LIST = (SUPPORT_RHEL6X_VERSION_LIST + SUPPORT_RHEL7X_VERSION_LIST + SUPPORT_RHEL8X_VERSION_LIST) diff --git a/script/os_platform/linux_distro.py b/script/os_platform/linux_distro.py index c434796c..559a9aa3 100644 --- a/script/os_platform/linux_distro.py +++ b/script/os_platform/linux_distro.py @@ -21,9 +21,14 @@ import os import re +import select +import sys from os_platform.common import _supported_dists +ISCONFIGURETRUE = "# isConfigure = TRUE" +ISCONFIGUREFALSE = "# isConfigure = FALSE" +SESSIONTIMEOUT = 300 class LinuxDistro(object): """ @@ -64,6 +69,148 @@ class LinuxDistro(object): if len(line) > 1: id_num = line[1] return '', version, id_num + + @staticmethod + def parse_linux_osid(filename): + """ + Tries to determine the name of the Linux OS distribution name. + + The function first looks for a distribution release file in + /etc and then reverts to _dist_try_harder() in case no + suitable files are found. + + Returns a tuple (distname,version,id) which default to the + args given as parameters. + + """ + valid_info = [] + lines_to_choose = [] + with open(filename, 'r') as file: + for line in file: + line = line.strip() + # Ignore comment lines starting with # and empty lines + if line and not line.startswith('#'): + # Separate the release name, version with spaces + distro, version = line.split() + valid_info.append({ + 'os': distro, + 'version': version + }) + lines_to_choose.append(line.strip()) + if len(lines_to_choose) == 1 or len(lines_to_choose) == 0: + return valid_info + for i, line in enumerate(lines_to_choose, 1): + print(f"{i}. {line}") + + while True: + try: + choice = int(input("Please enter the serial number of the option:")) + if (1 <= choice and choice <= len(lines_to_choose)): + chosen_line = lines_to_choose[choice - 1] + valid_info = valid_info[choice - 1] + with open(filename, 'r+') as file: + lines = file.readlines() + file.seek(0) + file.truncate() + + for line in lines: + if line.strip().startswith('#') or line.strip() == '': + file.write(line) + continue + elif chosen_line in line: + file.write(line) + else: + file.write('#' + line) + LinuxDistro.write_is_configure_true(filename, ISCONFIGURETRUE) + break + else: + print("Invalid input: Please re-enter") + except ValueError: + print("Invalid input: Please re-enter") + return valid_info + + + @staticmethod + def write_is_configure_true(file_path, target_line): + # open the file and read all the lines + with open(file_path, 'r') as file: + lines = file.readlines() + + # Check if any row contains target content + has_target_line = any(target_line in line for line in lines) + + # If there is no target row, insert the target row before row 21 + if not has_target_line: + lines.insert(20, target_line + '\n') + + # Write the modified content back to the file + with open(file_path, 'w') as file: + file.writelines(lines) + + + @staticmethod + def parse_linux_distributions(filename): + """ + Tries to determine the name of the Linux OS distribution name. + + The function first looks for a distribution release file in + /etc and then reverts to _dist_try_harder() in case no + + Returns a tuple (distname,version,id) which default to the + args given as parameters. + + """ + is_configure = True + valid_info = LinuxDistro.parse_linux_osid(filename) + + if len(valid_info) == 1: + LinuxDistro.write_is_configure_true(filename, ISCONFIGURETRUE) + + with open(filename, 'r') as file: + for line in file: + if ISCONFIGURETRUE in line or ISCONFIGUREFALSE in line: + is_configure = False + + # Remind the user if new content is added to the file + if len(valid_info) == 0 and is_configure: + print(f"File '{filename}' has not been configured yet," + "do you still need to configure it?" + "Enter 'yes' or 'no' to configure " + "the file: ", end='', flush=True) + rlist, _, _ = select.select([sys.stdin], [], [], SESSIONTIMEOUT) + + if rlist: + user_input = input().lower() + else: + user_input = "no" + + while True: + if user_input in ('y', 'yes'): + LinuxDistro.write_is_configure_true(filename, ISCONFIGURETRUE) + os_name = input("Please enter an operating system name:") + version = input("Please enter the version number:") + with open(filename, 'a') as file: + file.write(f"\n{os_name} {version}\n") + valid_info = LinuxDistro.parse_linux_osid(filename) + break + elif user_input in ('n', 'no'): + LinuxDistro.write_is_configure_true(filename, ISCONFIGUREFALSE) + break + else: + LinuxDistro.write_is_configure_true(filename, ISCONFIGUREFALSE) + break + return valid_info + + + @staticmethod + def select_linux_distribution(valid_info): + # If there is no legal information, return None + if not valid_info: + return None + + # If there is only one line of legal information, return directly + if len(valid_info) == 1: + return valid_info[0] @staticmethod def linux_distribution(distname='', version='', idNum='', @@ -89,11 +236,41 @@ class LinuxDistro(object): args given as parameters. """ + is_flag_osid = False try: etc_dir = os.listdir('/etc') except os.error: # Probably not a Unix system return distname, version, idNum + + # Read system information from configuration file + # Call the function and pass in the filename + osid_path = os.path.realpath( + os.path.join(os.path.realpath(__file__), "../../osid.in")) + + if os.path.exists(osid_path): + file_data = LinuxDistro.parse_linux_distributions(osid_path) + + # Output the parsed content + selected_data = LinuxDistro.select_linux_distribution(file_data) + + if selected_data: + is_flag_osid = True + + else: + print(f"The file '{osid_path}' does not exist.") + + if is_flag_osid: + if selected_data['os']: + distname = selected_data['os'] + if selected_data['version']: + version = selected_data['version'] + if selected_data['bit']: + idNum = selected_data['bit'] + return distname, version, idNum + # else: + # g_logger.debug("Start to distributing the check context dump file") + etc_dir.sort() gFile = None _release_filename = re.compile(r'(\w+)[-_](release|version)') diff --git a/script/osid.conf b/script/osid.conf new file mode 100644 index 00000000..b7ce1e4e --- /dev/null +++ b/script/osid.conf @@ -0,0 +1,33 @@ +# ---------------now we support this platform--------------- + # RHEL/CentOS "6.4", "6.5", "6.6", "6.7", "6.8", "6.9", + # "7.0", "7.1", "7.2", "7.3", "7.4", "7.5 "64bit + # EulerOS "2.0", "2.3" 64bit + # SuSE11 sp1/2/3/4 64bit + # SuSE12 sp0/1/2/3 64bit + # Kylin "10" 64bit + # Ubuntu "18.04" 64bit + + +# -------------------why to configure------------------------------- +# OM工具对OS版本是强依赖，目前各大厂商基于openEuler、centos等开源 +# 操作系统做了相关适配，当利用OM对openGauss安装时，checkos等OS检查项 +# 会报错，所以我们设置一个osid.configure文件，在该文件中对openEuler、 +# centos做相关设置，可以使用户顺利做相关校验，安装成功。 + +# -------------------how to configure------------------------------- +# 如果我们在script/osid.configure文件中配置了OS、version等信息，则OM +# 管理工具在gs_preintsall和gs_install过程中，其中读取OS系统的项中，会 +# 首先从该配置文件中读取。 + +# 如果该文件用户没有配置，gs_preinstall预安装过程中，会提示用户该文件 +# 没有配置，是否配置，用户可以根据自己的选择在此时是否需要配置。如果配置： +# 只需要配置OS、version、bit这三个选项。并且将配置内容写入该文件中。 +# 如果用户在该文件中直接进行配置，在————define-platform————行下边也是配 +# 置OS、version、bit这三个选项，如第29行所示，并且将行首的#号取消。 + +# ---------------define-platform------------------------------ +# centos 7.5 +# openEuler 20.03 +# openEuler 22.03 +# redhat 6.4 +# RHEL 7.6 \ No newline at end of file -- Gitee From 6eb0c0fea0778bf1487ec634e49e2d601e14dbc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E5=87=AF?= Date: Wed, 16 Aug 2023 11:24:30 +0800 Subject: [PATCH 61/96] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8DI7T0RB,=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E5=8D=B8=E8=BD=BD=E6=97=B6=E7=9A=84=E6=B8=85=E9=99=A4?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/postuninstall/PostUninstallImpl.py | 10 +++++++++- script/local/PreInstallUtility.py | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/script/impl/postuninstall/PostUninstallImpl.py b/script/impl/postuninstall/PostUninstallImpl.py index 326a4c51..ccc512e1 100644 --- a/script/impl/postuninstall/PostUninstallImpl.py +++ b/script/impl/postuninstall/PostUninstallImpl.py @@ -191,7 +191,7 @@ class PostUninstallImpl: CmdExecutor.execCommandWithMode(cmd, self.sshTool, self.localMode, self.mpprcFile) - self.logger.log("Successfully clean cgroup.") + self.logger.log("Successfully clean cgroup.") def cleanDirectory(self): """ @@ -471,6 +471,11 @@ class PostUninstallImpl: self.logger.debug( "Deleting environmental software of local nodes.") + hostName = NetUtil.GetHostIpOrName() + node_info = self.clusterInfo.getDbNodeByName(hostName) + datadir = node_info.datanodes[0].datadir + datadir_escaped = datadir.replace("/", "\\/") + basePort = node_info.datanodes[0].port # clean local node environment variable cmd = "(if [ -s '%s' ]; then " % PROFILE_FILE cmd += "sed -i -e '/^export PATH=\$PATH:\/root\/gauss_om\/%s\/" \ @@ -481,7 +486,10 @@ class PostUninstallImpl: "\$LD_LIBRARY_PATH$/d' %s " % PROFILE_FILE cmd += "-e '/^export LD_LIBRARY_PATH=\$GPHOME\/lib:" \ "\$LD_LIBRARY_PATH$/d' %s " % PROFILE_FILE + cmd += "-e '/^export PGPORT=%d/d' %s " %(basePort,PROFILE_FILE) + cmd += "-e '/^export PGDATA=%s/d' %s " %(datadir_escaped,PROFILE_FILE) cmd +="-e '/^export PYTHONPATH=\$GPHOME\/lib$/d' %s; fi) " % PROFILE_FILE + self.logger.debug( "Command for deleting environment variable: %s" % cmd) (status, output) = subprocess.getstatusoutput(cmd) diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 64ce2304..5c032f5b 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -1561,6 +1561,8 @@ Common options: "Deleting crash GPHOME in user environment variables.") # clean PGDATA FileUtil.deleteLine(userProfile, "^\\s*export\\s*PGDATA*") + # clean PGPORT + FileUtil.deleteLine(userProfile, "^\\s*export\\s*PGPORT*") # clean LD_LIBRARY_PATH FileUtil.deleteLine(userProfile, "^\\s*export\\s*LD_LIBRARY_PATH=\\$GPHOME\\/script" -- Gitee From 0f770e8c09e0af996f80c9244012e1fb6ae0064b Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Fri, 18 Aug 2023 08:54:01 +0800 Subject: [PATCH 62/96] =?UTF-8?q?om817=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_install | 3 --- script/gspylib/common/LocalBaseOM.py | 3 --- script/gspylib/common/ParallelBaseOM.py | 1 - script/gspylib/common/ParameterParsecheck.py | 5 +---- script/gspylib/component/BaseComponent.py | 1 - script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py | 8 +------- script/impl/install/OLAP/InstallImplOLAP.py | 5 +++-- script/local/InitInstance.py | 9 ++------- 8 files changed, 7 insertions(+), 28 deletions(-) diff --git a/script/gs_install b/script/gs_install index 69f48ae2..e4ec0c82 100644 --- a/script/gs_install +++ b/script/gs_install @@ -216,9 +216,6 @@ General options: # parameter --time-out if (ParaDict.__contains__("time_out")): self.time_out = ParaDict.get("time_out") - # parameter --dorado-info - if (ParaDict.__contains__("dorado-config")): - self.dorado_config = ParaDict.get("dorado-config") # parameter --dorado-cluster-mode if (ParaDict.__contains__("dorado-cluster-mode")): self.dorado_cluster_mode = ParaDict.get("dorado-cluster-mode") diff --git a/script/gspylib/common/LocalBaseOM.py b/script/gspylib/common/LocalBaseOM.py index 6eb8c8a1..71f1a546 100644 --- a/script/gspylib/common/LocalBaseOM.py +++ b/script/gspylib/common/LocalBaseOM.py @@ -50,7 +50,6 @@ class LocalBaseOM(object): paxos_mode=False, dss_mode=False, dss_config="", - dorado_config="", dorado_cluster_mode=""): ''' Constructor @@ -81,7 +80,6 @@ class LocalBaseOM(object): self.paxos_mode = paxos_mode self.dss_mode = dss_mode self.dss_config = dss_config - self.dorado_config = dorado_config self.dorado_cluster_mode = dorado_cluster_mode def initComponent(self, paxos_mode=False): @@ -157,7 +155,6 @@ class LocalBaseOM(object): component.paxos_mode = paxos_mode self.initComponentAttributes(component) component.initParas = self.initParas - component.dorado_config = self.dorado_config component.dorado_cluster_mode = self.dorado_cluster_mode self.dnCons.append(component) diff --git a/script/gspylib/common/ParallelBaseOM.py b/script/gspylib/common/ParallelBaseOM.py index 8a9a6a52..4f2a9539 100644 --- a/script/gspylib/common/ParallelBaseOM.py +++ b/script/gspylib/common/ParallelBaseOM.py @@ -80,7 +80,6 @@ class ParallelBaseOM(object): self.cnCons = [] self.dnCons = [] self.dss_cons = [] - self.dorado_config = "" self.dorado_cluster_mode = "" # localMode is same as isSingle in all OM script, expect for # gs_preinstall. diff --git a/script/gspylib/common/ParameterParsecheck.py b/script/gspylib/common/ParameterParsecheck.py index 7c3c9539..99752f1e 100644 --- a/script/gspylib/common/ParameterParsecheck.py +++ b/script/gspylib/common/ParameterParsecheck.py @@ -68,7 +68,7 @@ gs_preinstall = ["-?", "--help", "-V", "--version", "-U:", "-G:", "-L", "-l:", "--non-interactive", "--delete-root-trust", "--unused-third-party"] gs_install = ["-?", "--help", "-V", "--version", "-X:", "-l:", "--gsinit-parameter=", "--dn-guc=", "--cms-guc=", - "--time-out=", "--dorado-config=", "--dorado-cluster-mode", "--alarm-component="] + "--time-out=", "--dorado-cluster-mode=", "--alarm-component="] gs_uninstall = ["-?", "--help", "-V", "--version", "-l:", "-L", "--delete-data"] gs_postuninstall = ["-?", "--help", "-V", "--version", "--delete-user", @@ -302,7 +302,6 @@ class Parameter(): "--alarm-type": "warningType", "--alarm-server-addr": "warningserverip", "--time-out": "time_out", "": "", - "--dorado-config": "dorado-config", "--dorado-cluster-mode" :"dorado-cluster-mode", "--alarm-component": "alarm_component", "--SSD-fault-time": "SSDFaultTime", @@ -476,8 +475,6 @@ class Parameter(): PARAMETER_VALUEDICT['new_values'] = value.strip().split(",") elif key == "--upgrade-package": PARAMETER_VALUEDICT["upgrade-package"] = value.strip() - elif key == "--dorado-config": - PARAMETER_VALUEDICT["dorado-config"] = value.strip() elif key == "--dorado-cluster-mode": PARAMETER_VALUEDICT["dorado-cluster-mode"] = value.strip() # Only check / symbol for gs_lcct. diff --git a/script/gspylib/component/BaseComponent.py b/script/gspylib/component/BaseComponent.py index 52de2ede..134bd021 100644 --- a/script/gspylib/component/BaseComponent.py +++ b/script/gspylib/component/BaseComponent.py @@ -55,7 +55,6 @@ class BaseComponent(object): self.paxos_mode = '' self.dss_mode = '' self.dss_config = '' - self.dorado_config = '' self.dorado_cluster_mode = '' def install(self): diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index 1a065b14..5a94e885 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -162,12 +162,8 @@ class DN_OLAP(Kernel): " --socketpath=\"{}\"".format( "+{},+{}".format(vgname, pri_vgname), cfg_context, inst_id, "UDS:{}/.dss_unix_d_socket".format(dss_home)) - if (self.dorado_config != "" and self.instInfo.instanceType == DefaultValue.MASTER_INSTANCE): - cmd += " -g %s" % self.dorado_config - tmpDict3 = {} - tmpDict3["xlog_lock_file_path"] = "'%s/redolog.lock'" % self.instInfo.datadir if (self.dorado_cluster_mode != ""): - cmd += "--enable-ss-dorado" + cmd += " --enable-ss-dorado" self.logger.debug("Command for initializing database " "node instance: %s" % cmd) status, output = CmdUtil.retryGetstatusoutput( @@ -178,8 +174,6 @@ class DN_OLAP(Kernel): # set ssl to DB nodes. dnGucParas = self.getDnGUCDict() self.setGucConfig(dnGucParas) - if (self.dorado_config != "" and self.instInfo.instanceType == DefaultValue.MASTER_INSTANCE): - self.setGucConfig(tmpDict3) self.copyAndModCertFiles() def getInstanceNodeName(self): diff --git a/script/impl/install/OLAP/InstallImplOLAP.py b/script/impl/install/OLAP/InstallImplOLAP.py index 10b8ea2d..cb1810fc 100644 --- a/script/impl/install/OLAP/InstallImplOLAP.py +++ b/script/impl/install/OLAP/InstallImplOLAP.py @@ -290,8 +290,9 @@ class InstallImplOLAP(InstallImpl): elif self.context.clusterInfo.enable_dss == 'on': dss_config = DssConfig.get_value_b64_handler( 'dss_nodes_list', self.context.clusterInfo.dss_config) - cmd += f" --dss_mode --dss_config={dss_config} --dorado_config={self.context.dorado_config} \ - --dorado_cluster_mode={self.context.dorado_cluster_mode}" + cmd += f" --dss_mode --dss_config={dss_config}" + if self.context.dorado_cluster_mode != "": + cmd += f" --dorado_cluster_mode={self.context.dorado_cluster_mode}" self.context.logger.debug( "Command for initializing instances: %s" % cmd) diff --git a/script/local/InitInstance.py b/script/local/InitInstance.py index fdc9a0e0..edbda69f 100644 --- a/script/local/InitInstance.py +++ b/script/local/InitInstance.py @@ -62,7 +62,6 @@ class CmdOptions(): self.paxos_mode = False self.dss_mode = False self.dss_config = "" - self.dorado_config = "" self.dorado_cluster_mode = "" @@ -81,7 +80,7 @@ def parseCommandLine(): try: opts, args = getopt.getopt(sys.argv[1:], "U:P:G:l:?", [ "help", "dws_mode", "vc_mode", "paxos_mode", "dss_mode", - "dss_config=", "dorado_config=", "dorado_cluster_mode=" + "dss_config=", "dorado_cluster_mode=" ]) except Exception as e: usage() @@ -114,8 +113,6 @@ def parseCommandLine(): g_opts.dss_mode = True elif key == "--dss_config": g_opts.dss_config = value.strip() - elif key == "--dorado_config": - g_opts.dorado_config = value.strip() elif key == "--dorado_cluster_mode": g_opts.dorado_cluster_mode = value.strip() Parameter.checkParaVaild(key, value) @@ -185,7 +182,6 @@ class initDbNode(LocalBaseOM): paxos_mode=False, dss_mode=False, dss_config="", - dorado_config = "", dorado_cluster_mode = ""): """ function: init instance @@ -203,7 +199,7 @@ class initDbNode(LocalBaseOM): paxos_mode, dss_mode=dss_mode, dss_config=dss_config, - dorado_config=dorado_config) + dorado_cluster_mode=dorado_cluster_mode) if self.clusterConfig == "": # Read config from static config file self.readConfigInfo() @@ -279,7 +275,6 @@ if __name__ == '__main__': g_opts.paxos_mode, dss_mode=g_opts.dss_mode, dss_config=g_opts.dss_config, - dorado_config=g_opts.dorado_config, dorado_cluster_mode=g_opts.dorado_cluster_mode) dbInit.initNodeInst(g_opts.vc_mode) -- Gitee From bb34b284be881ac5702478a62bc9a70f388c786f Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Tue, 18 Jul 2023 15:42:30 +0800 Subject: [PATCH 63/96] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=8D=87=E7=BA=A7?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E6=97=B6=E5=80=99=E7=9A=84=E7=AD=89=E5=BE=85?= =?UTF-8?q?=E8=B6=85=E6=97=B6=E6=97=B6=E9=97=B4=EF=BC=8C=E9=80=82=E5=BA=94?= =?UTF-8?q?=E9=83=A8=E5=88=86=E7=8E=AF=E5=A2=83=E4=B8=8B=E5=A4=87=E6=9C=BA?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E5=9B=9E=E6=94=BE=E6=85=A2=E7=9A=84=E6=83=85?= =?UTF-8?q?=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/upgrade/UpgradeImpl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index 18d78873..8be6f5ab 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -1945,7 +1945,7 @@ class UpgradeImpl: self.context.oldClusterAppPath, "bin/upgrade_version") if os.path.exists(versionFile): _, number, _ = VersionInfo.get_version_info(versionFile) - cmd = "gs_om -t start --cluster-number='%s'" % (number) + cmd = "gs_om -t start --cluster-number='%s' --time-out=600" % (number) else: cmd = "gs_om -t start" (status, output) = subprocess.getstatusoutput(cmd) -- Gitee From e2f741c65f03445b6b53b4a1e7de69b3ed9307c8 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Mon, 31 Jul 2023 16:34:40 +0800 Subject: [PATCH 64/96] =?UTF-8?q?=E8=A7=A3=E5=86=B3dropnode=E5=90=8E?= =?UTF-8?q?=E5=BB=BA=E7=AB=8B=E5=AE=B9=E7=81=BE=E9=9B=86=E7=BE=A4=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/streaming_disaster_recovery/streaming_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/impl/streaming_disaster_recovery/streaming_base.py b/script/impl/streaming_disaster_recovery/streaming_base.py index acbc1a57..38afb16a 100644 --- a/script/impl/streaming_disaster_recovery/streaming_base.py +++ b/script/impl/streaming_disaster_recovery/streaming_base.py @@ -935,6 +935,8 @@ class StreamingBase(object): if output.count("=NULL") > 2 or "iscrossregion=true" in output.lower(): self.logger.debug("InstanceID:%s, Index:%s" % (dn_inst.instanceId, idx)) return idx, orignal_ports + if output.count(f"replconninfo{idx}=''") >= 2: + continue ret = re.search( r"replconninfo%s='localhost=(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})" r" localport=(\d{4,5}) localheartbeatport=(\d{4,5}) " -- Gitee From 39b7dda233fab29a90ca017695cc7807e039d009 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Thu, 10 Aug 2023 21:09:18 +0800 Subject: [PATCH 65/96] =?UTF-8?q?=E5=88=A0=E9=99=A4stop=E5=90=8E=E7=BD=AE?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=EF=BC=8C=E5=A4=9A=E4=BD=99=E4=B8=94=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=87=BA=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/component/Kernel/Kernel.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/script/gspylib/component/Kernel/Kernel.py b/script/gspylib/component/Kernel/Kernel.py index 93618d93..7cbb7253 100644 --- a/script/gspylib/component/Kernel/Kernel.py +++ b/script/gspylib/component/Kernel/Kernel.py @@ -127,15 +127,6 @@ class Kernel(BaseComponent): if status != 0: raise Exception(ErrorCode.GAUSS_516["GAUSS_51610"] % "instance" + " Error: \n%s." % output) - if output.find("No such process") > 0: - cmd = "ps c -eo pid,euid,cmd | grep gaussdb | grep -v grep | " \ - "awk '{if($2 == curuid && $1!=\"-n\") " \ - "print \"/proc/\"$1\"/cwd\"}' curuid=`id -u`|" \ - " xargs ls -l |awk '{if ($NF==\"%s\") print $(NF-2)}' | " \ - "awk -F/ '{print $3 }'" % (self.instInfo.datadir) - (status, rightpid) = subprocess.getstatusoutput(cmd) - if rightpid or status != 0: - GaussLog.exitWithError(output) def isPidFileExist(self): pidFile = "%s/postmaster.pid" % self.instInfo.datadir -- Gitee From a0631842fa57ee69736e6af12a5a4d2053e317c8 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Thu, 17 Aug 2023 20:44:19 +0800 Subject: [PATCH 66/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcm=E4=B8=8B=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=AF=BC=E8=87=B4application=5Fname=E5=92=8Cpgxc=5Fno?= =?UTF-8?q?de=5Fname=E5=BC=82=E5=B8=B8=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/expansion/expansion_impl_with_cm.py | 7 ++++--- script/impl/expansion/expansion_impl_with_cm_local.py | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/script/impl/expansion/expansion_impl_with_cm.py b/script/impl/expansion/expansion_impl_with_cm.py index e7695eae..8f7f0089 100644 --- a/script/impl/expansion/expansion_impl_with_cm.py +++ b/script/impl/expansion/expansion_impl_with_cm.py @@ -76,6 +76,7 @@ class ExpansionImplWithCm(ExpansionImpl): self.xml_cluster_info = dbClusterInfo() self.ssh_tool = None self.new_nodes = list() + self.app_names = list() self._init_global() def _init_global(self): @@ -297,10 +298,9 @@ class ExpansionImplWithCm(ExpansionImpl): self.logger.debug("Start to set other guc parameters.") # set port|application_name|log_directory|audit_directory on new nodes - app_names = self.getIncreaseAppNames(len(self.new_nodes)) log_path = ClusterDir.getUserLogDirWithUser(self.user) new_nodes_para_list = [] - for node,appname in zip(self.new_nodes, app_names): + for node,appname in zip(self.new_nodes, self.app_names): if node.datanodes: datains = node.datanodes[0] log_dir = "%s/pg_log/dn_%d" % (log_path, appname) @@ -552,9 +552,10 @@ class ExpansionImplWithCm(ExpansionImpl): self.expansion_check() self.do_preinstall() self.logger.debug("[preinstall end] new nodes success: %s" % self.expansionSuccess) + self.app_names = self.getIncreaseAppNames(len(self.new_nodes)) + self.logger.debug("get increase application names %s" % self.app_names) change_user_executor(self.do_install) change_user_executor(self.do_config) - self._set_pgxc_node_name() change_user_executor(self.do_start) self.check_new_node_state(True) diff --git a/script/impl/expansion/expansion_impl_with_cm_local.py b/script/impl/expansion/expansion_impl_with_cm_local.py index fda26275..fdb8b3a2 100644 --- a/script/impl/expansion/expansion_impl_with_cm_local.py +++ b/script/impl/expansion/expansion_impl_with_cm_local.py @@ -151,6 +151,5 @@ class ExpansionImplWithCmLocal(ExpansionImplWithCm): self.expansion_check() self._set_expansion_success() change_user_executor(self.do_config) - self._set_pgxc_node_name() change_user_executor(self.do_start) self.check_new_node_state(True) -- Gitee From 2ca631f73375127a92a791175e45756505149cd2 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Sun, 20 Aug 2023 10:46:26 +0800 Subject: [PATCH 67/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dosid.in=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=89=BE=E4=B8=8D=E5=88=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/os/gsplatform.py | 63 +++++++++++++++-------------- script/os_platform/linux_distro.py | 64 +++++++++++++++--------------- 2 files changed, 66 insertions(+), 61 deletions(-) diff --git a/script/gspylib/os/gsplatform.py b/script/gspylib/os/gsplatform.py index d496684c..c0c9a247 100644 --- a/script/gspylib/os/gsplatform.py +++ b/script/gspylib/os/gsplatform.py @@ -318,38 +318,12 @@ def linux_distribution(distname='', version='', idNum='', """ is_flag_osid = False + is_flag_oscorrect = True try: etc = os.listdir('/etc') except os.error: # Probably not a Unix system - return distname, version, idNum - - # Read system information from configuration file - # Call the function and pass in the filename - osid_path = os.path.realpath( - os.path.join(os.path.realpath(__file__), "../../../osid.in")) - - if os.path.exists(osid_path): - file_data = parse_linux_distributions(osid_path) - - # Output the parsed content - selected_data = select_linux_distribution(file_data) - - if selected_data: - is_flag_osid = True - - else: - print(f"The file '{osid_path}' does not exist.") - - if is_flag_osid: - if selected_data['os']: - distname = selected_data['os'] - if selected_data['version']: - version = selected_data['version'] - if selected_data['bit']: - idNum = selected_data['bit'] - return distname, version, idNum - + return distname, version, idNum sortEtc = sorted(etc) gFile = None for file in sortEtc: @@ -372,12 +346,41 @@ def linux_distribution(distname='', version='', idNum='', if _distname and full_distribution_name: distname = _distname + for dist in SUPPORT_WHOLE_PLATFORM_LIST: + if dist.lower == _distname.lower: + is_flag_oscorrect = True if _version: version = _version if _id: idNum = _id - return distname, version, idNum - + if is_flag_oscorrect == True: + return distname, version, idNum + elif is_flag_oscorrect == False: + # Read system information from configuration file + # Call the function and pass in the filename + osid_path = os.path.realpath( + os.path.join(os.path.realpath(__file__), "../../../osid.conf")) + + if os.path.exists(osid_path): + file_data = parse_linux_distributions(osid_path) + + # Output the parsed content + selected_data = select_linux_distribution(file_data) + + if selected_data: + is_flag_osid = True + + else: + print(f"The file '{osid_path}' does not exist.") + + if is_flag_osid: + if selected_data['os']: + distname = selected_data['os'] + if selected_data['version']: + version = selected_data['version'] + if selected_data['bit']: + idNum = selected_data['bit'] + return distname, version, idNum def dist(supported_dists=_supported_dists): """ Tries to determine the name of the Linux OS distribution name. diff --git a/script/os_platform/linux_distro.py b/script/os_platform/linux_distro.py index 559a9aa3..fc961eb3 100644 --- a/script/os_platform/linux_distro.py +++ b/script/os_platform/linux_distro.py @@ -24,7 +24,7 @@ import re import select import sys -from os_platform.common import _supported_dists +from os_platform.common import _supported_dists,SUPPORT_WHOLE_PLATFORM_LIST ISCONFIGURETRUE = "# isConfigure = TRUE" ISCONFIGUREFALSE = "# isConfigure = FALSE" @@ -237,40 +237,12 @@ class LinuxDistro(object): """ is_flag_osid = False + is_flag_oscorrect = True try: etc_dir = os.listdir('/etc') except os.error: # Probably not a Unix system return distname, version, idNum - - # Read system information from configuration file - # Call the function and pass in the filename - osid_path = os.path.realpath( - os.path.join(os.path.realpath(__file__), "../../osid.in")) - - if os.path.exists(osid_path): - file_data = LinuxDistro.parse_linux_distributions(osid_path) - - # Output the parsed content - selected_data = LinuxDistro.select_linux_distribution(file_data) - - if selected_data: - is_flag_osid = True - - else: - print(f"The file '{osid_path}' does not exist.") - - if is_flag_osid: - if selected_data['os']: - distname = selected_data['os'] - if selected_data['version']: - version = selected_data['version'] - if selected_data['bit']: - idNum = selected_data['bit'] - return distname, version, idNum - # else: - # g_logger.debug("Start to distributing the check context dump file") - etc_dir.sort() gFile = None _release_filename = re.compile(r'(\w+)[-_](release|version)') @@ -294,8 +266,38 @@ class LinuxDistro(object): if _distname and full_distribution_name: distname = _distname + for dist in SUPPORT_WHOLE_PLATFORM_LIST: + if dist.lower == _distname.lower: + is_flag_oscorrect = True if _version: version = _version if _id: idNum = _id - return distname, version, idNum + if is_flag_oscorrect == True: + return distname, version, idNum + elif is_flag_oscorrect == False: + # Read system information from configuration file + # Call the function and pass in the filename + osid_path = os.path.realpath( + os.path.join(os.path.realpath(__file__), "../../osid.conf")) + + if os.path.exists(osid_path): + file_data = LinuxDistro.parse_linux_distributions(osid_path) + + # Output the parsed content + selected_data = LinuxDistro.select_linux_distribution(file_data) + + if selected_data: + is_flag_osid = True + + else: + print(f"The file '{osid_path}' does not exist.") + + if is_flag_osid: + if selected_data['os']: + distname = selected_data['os'] + if selected_data['version']: + version = selected_data['version'] + if selected_data['bit']: + idNum = selected_data['bit'] + return distname, version, idNum -- Gitee From 500b456383f1772de9df6af070299162485046ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E5=87=AF?= Date: Mon, 21 Aug 2023 15:18:38 +0800 Subject: [PATCH 68/96] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8DI7TMVG,=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0PGDATABSE=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/postuninstall/PostUninstallImpl.py | 1 + script/local/PreInstallUtility.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/script/impl/postuninstall/PostUninstallImpl.py b/script/impl/postuninstall/PostUninstallImpl.py index ccc512e1..38daaf61 100644 --- a/script/impl/postuninstall/PostUninstallImpl.py +++ b/script/impl/postuninstall/PostUninstallImpl.py @@ -486,6 +486,7 @@ class PostUninstallImpl: "\$LD_LIBRARY_PATH$/d' %s " % PROFILE_FILE cmd += "-e '/^export LD_LIBRARY_PATH=\$GPHOME\/lib:" \ "\$LD_LIBRARY_PATH$/d' %s " % PROFILE_FILE + cmd += "-e '/^export PGDATABASE=postgres/d' %s " % PROFILE_FILE cmd += "-e '/^export PGPORT=%d/d' %s " %(basePort,PROFILE_FILE) cmd += "-e '/^export PGDATA=%s/d' %s " %(datadir_escaped,PROFILE_FILE) cmd +="-e '/^export PYTHONPATH=\$GPHOME\/lib$/d' %s; fi) " % PROFILE_FILE diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 5c032f5b..95b87c16 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -1563,6 +1563,8 @@ Common options: FileUtil.deleteLine(userProfile, "^\\s*export\\s*PGDATA*") # clean PGPORT FileUtil.deleteLine(userProfile, "^\\s*export\\s*PGPORT*") + # clean PGDATABASE + FileUtil.deleteLine(userProfile, "^\\s*export\\s*PGDATABASE*") # clean LD_LIBRARY_PATH FileUtil.deleteLine(userProfile, "^\\s*export\\s*LD_LIBRARY_PATH=\\$GPHOME\\/script" @@ -1634,6 +1636,8 @@ Common options: datadir = node_info.datanodes[0].datadir FileUtil.writeFile(userProfile, ["export PGDATA=%s" % datadir]) + # set PGDATABASE + FileUtil.writeFile(userProfile, ["export PGDATABASE=%s" % "postgres"]) # set PGPORT basePort = node_info.datanodes[0].port -- Gitee From c487c53286d1f822a57e2a3a731f3dce0542cf94 Mon Sep 17 00:00:00 2001 From: gentle_hu Date: Tue, 22 Aug 2023 09:42:29 +0800 Subject: [PATCH 69/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dverify=E4=B8=8Echeck?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E4=B8=8D=E5=AF=B9=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/upgrade_checker/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/upgrade_checker/project.py b/script/upgrade_checker/project.py index c241c710..1e7483a3 100644 --- a/script/upgrade_checker/project.py +++ b/script/upgrade_checker/project.py @@ -122,7 +122,7 @@ class ExportProj(Project): logger.debug('工程信息：\n' + self.__str__()) def run(self): - db_list = Collector.prepare_db_list(Action.VERIFY) + db_list = Collector.prepare_db_list(self.param.action) collector = Collector(db_list) exporter = Exporter(self.vmap) @@ -175,7 +175,7 @@ class VerifyProj(Project): logger.debug('工程信息：\n' + self.__str__()) def run(self): - db_list = Collector.prepare_db_list(Action.VERIFY) + db_list = Collector.prepare_db_list(self.param.action) collector = Collector(db_list) analyzer = Analyzer(self.vmap) reporter = Reporter(self.report, -- Gitee From 48433d58587894648890af70b6e9b3394c2ac3cf Mon Sep 17 00:00:00 2001 From: gentle_hu Date: Tue, 22 Aug 2023 14:42:41 +0800 Subject: [PATCH 70/96] add pid suffix for upgradechk project workspace dir --- script/upgrade_checker/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/upgrade_checker/project.py b/script/upgrade_checker/project.py index 1e7483a3..858cfb99 100644 --- a/script/upgrade_checker/project.py +++ b/script/upgrade_checker/project.py @@ -54,7 +54,7 @@ class Project(object): param.action.name, time.strftime('%Y-%m-%d-%H_%M_%S', time.localtime(self.id)) ) - self.workspace = "{0}/{1}".format(self.dir_workspace, self.name) + self.workspace = "{0}/{1}_{2}".format(self.dir_workspace, self.name, os.getpid()) self.log = "{0}/run.log".format(self.workspace) def __str__(self): -- Gitee From 13c45bcfe0ec22d49d5cef5dfece910bcfdc91cd Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Wed, 23 Aug 2023 16:05:46 +0800 Subject: [PATCH 71/96] copy upgrade script of dolphin from old appPath to new one --- script/impl/upgrade/UpgradeImpl.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index 18d78873..9f49117c 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -1529,6 +1529,9 @@ class UpgradeImpl: # prepared in the preinstall self.installNewBin() + # copy dolphin upgrade script from old appPath to new one + self.cpDolphinUpgradeScript() + # decompress the catalog upgrade_sql.tar.gz to temp dir, # include upgrade sql file and guc set self.prepareUpgradeSqlFolder() @@ -6763,6 +6766,23 @@ class UpgradeImpl: self.context.logger.debug("Failed to install new binary files.") raise Exception(str(e)) + def cpDolphinUpgradeScript(self): + """ + function: copy upgrade script of dolphin from old appPath to new one + input: none + output: none + """ + self.context.logger.debug("Start to copy dolphin upgrade script.") + + try: + cmd = "if ls %s/share/postgresql/extension/ | grep -qE \"dolphin--(.*)--(.*)sql\" ; " \ + "then cp -f %s/share/postgresql/extension/dolphin--*--*sql %s/share/postgresql/extension/; fi" % \ + (self.context.oldClusterAppPath, self.context.oldClusterAppPath, self.context.newClusterAppPath) + CmdExecutor.execCommandLocally(cmd) + self.context.logger.debug("Successfully copy dolphin upgrade script.") + except Exception as e: + raise Exception("Failed to copy dolphin upgrade script.") + def backupHotpatch(self): """ function: backup hotpatch config file patch.info in xxx/data/hotpatch -- Gitee From d96ffe9612b3284e1b255233beab73b3dcc3826e Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Wed, 23 Aug 2023 20:25:03 +0800 Subject: [PATCH 72/96] =?UTF-8?q?=E5=AF=B9=E5=8F=91=E9=80=81=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=A4=B1=E8=B4=A5=E5=A2=9E=E5=8A=A0=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/threads/SshTool.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/script/gspylib/threads/SshTool.py b/script/gspylib/threads/SshTool.py index f3cc4b0d..3ee09583 100644 --- a/script/gspylib/threads/SshTool.py +++ b/script/gspylib/threads/SshTool.py @@ -787,11 +787,13 @@ class SshTool(): targetDir, self.__resultFile) (status, output) = subprocess.getstatusoutput(scpCmd) - # If sending the file fails, we retry after 3s to avoid the - # failure caused by intermittent network disconnection. + # If sending the file fails, we retry 3 * 10s to avoid the + # failure caused by intermittent network disconnection. such as Broken pipe. # If the fails is caused by timeout. no need to retry. - if status != 0 and output.find("Timed out") < 0: - time.sleep(3) + max_retry_times = 3 + while(max_retry_times > 0 and status != 0 and output.find("Timed out") < 0): + max_retry_times -= 1 + time.sleep(10) (status, output) = subprocess.getstatusoutput(scpCmd) if status != 0: -- Gitee From 583b3492b58b1fa16ba487a3b456cfd8c0857f3b Mon Sep 17 00:00:00 2001 From: z00793368 Date: Fri, 25 Aug 2023 10:07:25 +0800 Subject: [PATCH 73/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B1=A0=E5=8C=96=E5=AE=89=E8=A3=85=E6=94=AF=E6=8C=81DM-multip?= =?UTF-8?q?ath=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/component/DSS/dss_comp.py | 2 +- script/local/PreInstallUtility.py | 31 ++++++++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/script/gspylib/component/DSS/dss_comp.py b/script/gspylib/component/DSS/dss_comp.py index 2329f75d..acd807fa 100644 --- a/script/gspylib/component/DSS/dss_comp.py +++ b/script/gspylib/component/DSS/dss_comp.py @@ -103,7 +103,7 @@ class DssInst(): with open(vg_cfg, "r") as fp: context = fp.read().strip() pris = re.findall( - '(.*):/dev/.*private_{}'.format(str(dss_id)), context) + '(.*):/dev/.*'.format(str(dss_id)), context) if pris: return pris[0].strip() else: diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 0e011460..505bde0b 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -888,13 +888,33 @@ Common options: Preparing the VG Configuration File ''' + dss_dict = {} dss_vg_ini = os.path.realpath( os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini')) - lun_map = UdevContext.get_all_vgname_disk_pair( - self.clusterInfo.dss_shared_disks, self.clusterInfo.dss_pri_disks, - self.user) + + lun_map = self.clusterInfo.dss_vg_info + lun_dss_vgname = self.clusterInfo.dss_vgname + + # dss_vg_info checker + infos = list(filter(None, re.split(r':|,', lun_map))) + + # The volume name must correspond to the disk. + if (lun_map.count(':') != lun_map.count(',') + 1) or ( + not infos) or (infos and len(infos) % 2 != 0): + raise Exception(ErrorCode.GAUSS_504["GAUSS_50414"] % + "The volume name must correspond to the disk.") + + # The shared volume must be in vg_config. + if lun_dss_vgname not in infos[::2]: + raise Exception(ErrorCode.GAUSS_504["GAUSS_50419"] % + (lun_dss_vgname, lun_map)) + + for i in range(0, len(lun_map), 2): + key = lun_map[i] + value = lun_map[i + 1] + dss_dict[key] = value - context = [':'.join([k, v]) for k, v in lun_map.items()] + context = [':'.join([k, v]) for k, v in dss_dict.items()] FileUtil.write_custom_context( dss_vg_ini, context, authority=DefaultValue.KEY_FILE_MODE_IN_OS) @@ -910,7 +930,8 @@ Common options: self.prepareGivenPath(dss_cfg, False) self.prepareGivenPath(dss_log, False) self.prepare_dss_inst_ini(dss_home, dss_id) - self.prepare_dss_soft_link() + # this function does not need to be detected under multipath + # self.prepare_dss_soft_link() self.prepare_dss_vg_ini(dss_home) -- Gitee From c2df2e3e1f5a4ddfeb616d9bbdf566b727fcfebb Mon Sep 17 00:00:00 2001 From: xue_meng_en <1836611252@qq.com> Date: Fri, 25 Aug 2023 15:26:48 +0800 Subject: [PATCH 74/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B8=A6CM=E5=B0=B1?= =?UTF-8?q?=E5=9C=B0=E5=8D=87=E7=BA=A7=E5=8D=A1=E4=BD=8F=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/upgrade/UpgradeImpl.py | 54 +++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index b18aa188..0540a7f6 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -3726,6 +3726,58 @@ class UpgradeImpl: "Output: %s" % output) self.context.logger.debug("Start cluster with gs_om successfully.") + def cm_start_cluster(self): + """ + Start Cluster with cm + """ + self.context.logger.debug("Starting cluster with cm.") + gauss_home = EnvUtil.getEnv("GAUSSHOME") + gauss_log = EnvUtil.getEnv("GAUSSLOG") + # check whether om_monitor started + check_monitor_cmd = "gs_ssh -c 'ps x | grep -v grep | grep om_monitor'" + start_monitor_cmd = "gs_ssh -c 'nohup om_monitor -L %s/cm/om_monitor >> " \ + "/dev/null 2>&1 &'" % gauss_log + self.context.logger.debug("check monitor cmd: " + check_monitor_cmd) + self.context.logger.debug("start monitor cmd: " + start_monitor_cmd) + cluster_start_timeout = 300 + wait_time = 0 + while wait_time < cluster_start_timeout: + status, output = subprocess.getstatusoutput(check_monitor_cmd) + if status == 0 and output.find("FAILURE") == -1: + break + self.context.logger.debug("check monitor output: " + output) + status, output = subprocess.getstatusoutput(start_monitor_cmd) + wait_time += 1 + time.sleep(1) + if wait_time >= cluster_start_timeout: + raise Exception(ErrorCode.GAUSS_516["GAUSS_51607"] % ("cluster in %ds" % cluster_start_timeout) + + "\nSome om_monitor is not running, please check.\n" + "Hint: please check max number of open files limit.") + + # remove cluster_manual_start file to start cluster + cluster_manual_start_file = os.path.join(gauss_home, "bin", "cluster_manual_start") + cmd = "source %s ; gs_ssh -c 'rm %s -f'" % ( + self.context.userProfile, cluster_manual_start_file) + self.context.logger.debug("cm start cluster cmd: %s" % cmd) + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception(ErrorCode.GAUSS_516["GAUSS_51607"] % "cluster" + + "cmd: %s\nOutput: %s" % (cmd, output)) + cmd = "source %s ;gs_om -t query" % self.context.userProfile + while wait_time < cluster_start_timeout: + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception(ErrorCode.GAUSS_516["GAUSS_51607"] % "cluster" + + "Output: %s" % output) + if output.find("cluster_state : Normal") != -1: + break + time.sleep(1) + wait_time += 1 + if wait_time >= cluster_start_timeout: + raise Exception(ErrorCode.GAUSS_516["GAUSS_51607"] % ("cluster in %ds" % cluster_start_timeout) + + "Current cluster state:\n%s" % output) + self.context.logger.debug("Start cluster with cm successfully.") + def get_cms_num(self, cluster_config_file): """ Get cm_server num from static config file @@ -3799,7 +3851,7 @@ class UpgradeImpl: else: self.startCluster() else: - self.om_start_cluster() + self.cm_start_cluster() def stop_strategy(self, is_final=True): """ -- Gitee From c9a744f22b3912f0673c30b9301365bbdcaac4cd Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Tue, 5 Jul 2022 22:30:09 +0800 Subject: [PATCH 75/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8D=87=E7=BA=A7om?= =?UTF-8?q?=E5=9C=A8=E5=88=87=E6=8D=A2dn=E9=98=BB=E5=A1=9E=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/common/Common.py | 25 ++++++++++++++----------- script/local/UpgradeUtility.py | 3 ++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/script/gspylib/common/Common.py b/script/gspylib/common/Common.py index 2d1016fa..18022432 100644 --- a/script/gspylib/common/Common.py +++ b/script/gspylib/common/Common.py @@ -2340,7 +2340,7 @@ class DefaultValue(): return False @staticmethod - def getSpecificNode(userProfile, flagStr, logger=None): + def getSpecificNode(userProfile, flagStr, logger=None, with_cm=False): """ :param flagStr: Primary/Standby/Cascade :return: correspond nodes @@ -2351,15 +2351,18 @@ class DefaultValue(): cmd = "source {0} && gs_om -t query".format( userProfile) (status, output) = subprocess.getstatusoutput(cmd) - if status == 0 and ("cluster_state : Normal" in output \ - or "cluster_state : Degraded" in output): + if status == 0 and not with_cm: break - if count == 2: - start_cmd = "source {0} && gs_om -t start --time-out 30".format(userProfile) - _, output = subprocess.getstatusoutput(start_cmd) - if logger: - logger.debug("Start cluster for get current primary datanode, " - "the result is : \n{0}".format(output)) + + if status == 0 and with_cm: + if ("cluster_state : Normal" in output or "cluster_state : Degraded" in output): + break + if count == 2: + start_cmd = "source {0} && gs_om -t start --time-out 30".format(userProfile) + _, output = subprocess.getstatusoutput(start_cmd) + if logger: + logger.debug("Start cluster for get current primary datanode, " + "the result is : \n{0}".format(output)) time.sleep(10) count += 1 if status != 0: @@ -2376,12 +2379,12 @@ class DefaultValue(): raise Exception(str(e)) @staticmethod - def getPrimaryNode(userProfile, logger=None): + def getPrimaryNode(userProfile, logger=None, with_cm=False): """ :param :return: PrimaryNode """ - return DefaultValue.getSpecificNode(userProfile, "Primary", logger) + return DefaultValue.getSpecificNode(userProfile, "Primary", logger, with_cm) @staticmethod def getStandbyNode(userProfile, logger=None): diff --git a/script/local/UpgradeUtility.py b/script/local/UpgradeUtility.py index 77072db8..2c08f7fe 100644 --- a/script/local/UpgradeUtility.py +++ b/script/local/UpgradeUtility.py @@ -4341,11 +4341,12 @@ def isKillDn(): # get the dn and cn name dnInst = None clusterNodes = g_clusterInfo.dbNodes + with_cm = True if g_clusterInfo.cmscount > 0 else False for dbNode in clusterNodes: if len(dbNode.datanodes) == 0: continue dnInst = dbNode.datanodes[0] - primaryDnNode, _ = DefaultValue.getPrimaryNode(g_opts.userProfile) + primaryDnNode, _ = DefaultValue.getPrimaryNode(g_opts.userProfile, with_cm=with_cm) if dnInst.hostname not in primaryDnNode: continue break -- Gitee From a62863647c7becec38e17d0624bfb29c84f6dbd0 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Sat, 26 Aug 2023 17:21:43 +0800 Subject: [PATCH 76/96] =?UTF-8?q?gs=5Fctl=20stop=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E5=86=8D=E6=9F=A5=E8=AF=A2=E8=BF=9B=E7=A8=8B=E7=A1=AE=E8=AE=A4?= =?UTF-8?q?=E6=98=AF=E5=90=A6=E6=AD=A3=E5=B8=B8=E5=81=9C=E6=AD=A2=E6=88=96?= =?UTF-8?q?=E8=80=85=E7=94=B1=E4=BA=8Ecore=E5=AF=BC=E8=87=B4=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E7=BB=88=E6=AD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/component/Kernel/Kernel.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/script/gspylib/component/Kernel/Kernel.py b/script/gspylib/component/Kernel/Kernel.py index 7cbb7253..d9dc3a6e 100644 --- a/script/gspylib/component/Kernel/Kernel.py +++ b/script/gspylib/component/Kernel/Kernel.py @@ -127,6 +127,18 @@ class Kernel(BaseComponent): if status != 0: raise Exception(ErrorCode.GAUSS_516["GAUSS_51610"] % "instance" + " Error: \n%s." % output) + if output.find("No such process") > 0: + cmd = "ps c -eo pid,euid,cmd | grep gaussdb | grep -v grep | " \ + "awk '{if($2 == curuid && $1!=\"-n\") " \ + "print \"/proc/\"$1\"/cwd\"}' curuid=`id -u`|" \ + " xargs ls -l |awk '{if ($NF==\"%s\") print $(NF-2)}' | " \ + "awk -F/ '{print $3 }'" % (self.instInfo.datadir) + (status, rightpid) = subprocess.getstatusoutput(cmd) + if rightpid and rightpid.find("Permission denied") > -1: + self.logger.debug("stop success with query process %s" % output) + return + if rightpid or status != 0: + GaussLog.exitWithError(output) def isPidFileExist(self): pidFile = "%s/postmaster.pid" % self.instInfo.datadir -- Gitee From 306c57cdaead8cab0f7bf85c57ab6ee101126eef Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Mon, 28 Aug 2023 20:55:04 +0800 Subject: [PATCH 77/96] =?UTF-8?q?gs=5Fsshexkey=E5=AF=86=E7=A0=81=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E5=A4=B1=E8=B4=A5=E6=B7=BB=E5=8A=A0=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_sshexkey | 2 ++ script/gspylib/threads/SshTool.py | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/script/gs_sshexkey b/script/gs_sshexkey index d6ee117b..8596891f 100644 --- a/script/gs_sshexkey +++ b/script/gs_sshexkey @@ -369,6 +369,8 @@ General options: self.hosts_paswd_list.append([sshIp, KeyboardPassword]) break else: + if i < 2: + GaussLog.printMessage("The password is incorrect, please input again.") continue except Exception as e: raise Exception(ErrorCode.GAUSS_512["GAUSS_51220"] % sshIp + diff --git a/script/gspylib/threads/SshTool.py b/script/gspylib/threads/SshTool.py index 3ee09583..e965e4dd 100644 --- a/script/gspylib/threads/SshTool.py +++ b/script/gspylib/threads/SshTool.py @@ -207,10 +207,7 @@ class SshTool(): cmd = "su - %s -c \"%s\" 2>&1" % (user, cmd) else: cmd += " 2>&1" - if user == "root": - status, output = subprocess.getstatusoutput(cmd) - else: - status = os.system(cmd) + status = os.system(cmd) if status != 0: # we can not print cmd here, because it include user's passwd -- Gitee From e8154f6420739f093af2db3881045243a2b80063 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Tue, 29 Aug 2023 10:41:07 +0800 Subject: [PATCH 78/96] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BC=A0=E5=85=A5?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/local/PreInstallUtility.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 505bde0b..e3f30cdf 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -909,9 +909,9 @@ Common options: raise Exception(ErrorCode.GAUSS_504["GAUSS_50419"] % (lun_dss_vgname, lun_map)) - for i in range(0, len(lun_map), 2): - key = lun_map[i] - value = lun_map[i + 1] + for i in range(0, len(infos), 2): + key = infos[i] + value = infos[i + 1] dss_dict[key] = value context = [':'.join([k, v]) for k, v in dss_dict.items()] -- Gitee From 729f813891a41fed7ec30063082331d7bbb34bca Mon Sep 17 00:00:00 2001 From: chentingting <2393940156@qq.com> Date: Tue, 29 Aug 2023 17:37:30 +0800 Subject: [PATCH 79/96] fix bug for priate vgname not found --- script/gspylib/component/DSS/dss_comp.py | 38 +++---------------- .../component/Kernel/DN_OLAP/DN_OLAP.py | 7 +--- script/impl/install/OLAP/InstallImplOLAP.py | 12 +++--- script/local/PreInstallUtility.py | 33 ++++------------ 4 files changed, 21 insertions(+), 69 deletions(-) diff --git a/script/gspylib/component/DSS/dss_comp.py b/script/gspylib/component/DSS/dss_comp.py index acd807fa..66f0f915 100644 --- a/script/gspylib/component/DSS/dss_comp.py +++ b/script/gspylib/component/DSS/dss_comp.py @@ -64,48 +64,22 @@ class DssInst(): else: raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % self.cfg_path) return items - - @staticmethod - def get_private_vg_num(dss_home): - ''' - Obtaining Private Volumes - ''' - - - vg_cfg = os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini') - if os.path.isfile(vg_cfg): - try: - with open(vg_cfg, "r") as fp: - context = fp.read().strip() - pris = re.findall( - '(.*):/dev/.*private_.*', context) - if pris: - return len(pris) - else: - raise Exception(ErrorCode.GAUSS_504["GAUSS_50416"] % - 'in dss_vg_conf.ini') - except Exception as eds: - raise Exception(ErrorCode.GAUSS_504["GAUSS_50414"] % eds) - else: - raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % vg_cfg) @staticmethod - def get_private_vgname_by_ini(dss_home, dss_id, xlog_in_one_priv_vg): + def get_private_vgname_by_ini(dss_home, dss_id): ''' Obtaining a Private Volume ''' - - if xlog_in_one_priv_vg: - dss_id = 0 vg_cfg = os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini') if os.path.isfile(vg_cfg): try: with open(vg_cfg, "r") as fp: context = fp.read().strip() - pris = re.findall( - '(.*):/dev/.*'.format(str(dss_id)), context) - if pris: - return pris[0].strip() + pris = re.findall('(.+):(.+)', context) + if pris and len(pris) == 2: + return pris[1][0].strip() + elif pris and len(pris) > dss_id + 1: + return pris[dss_id + 1][0].strip() else: raise Exception(ErrorCode.GAUSS_504["GAUSS_50416"] % 'in dss_vg_conf.ini') diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index 5a94e885..db08fc28 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -151,13 +151,8 @@ class DN_OLAP(Kernel): 'dss_nodes_list', self.dss_config, action='decode') cfg_context = DssInst.get_dms_url(dss_nodes_list) - xlog_in_one_priv_vg = False - infos = list(filter(None, re.split(r':|:|,', dss_nodes_list))) - pri_vg_num = DssInst.get_private_vg_num(dss_home) # when use one private vg for xlog, vgname should get from inst_id=0 - if (pri_vg_num < len(infos[::3]) and pri_vg_num == 1): - xlog_in_one_priv_vg = True - pri_vgname = DssInst.get_private_vgname_by_ini(dss_home, inst_id, xlog_in_one_priv_vg) + pri_vgname = DssInst.get_private_vgname_by_ini(dss_home, inst_id) cmd += " -n --vgname=\"{}\" --enable-dss --dms_url=\"{}\" -I {}" \ " --socketpath=\"{}\"".format( "+{},+{}".format(vgname, pri_vgname), cfg_context, inst_id, diff --git a/script/impl/install/OLAP/InstallImplOLAP.py b/script/impl/install/OLAP/InstallImplOLAP.py index cb1810fc..84da2c12 100644 --- a/script/impl/install/OLAP/InstallImplOLAP.py +++ b/script/impl/install/OLAP/InstallImplOLAP.py @@ -196,13 +196,15 @@ class InstallImplOLAP(InstallImpl): return self.context.logger.log('Start to create the dss vg.') - for vgname, dss_disk in UdevContext.get_all_vgname_disk_pair( - self.context.clusterInfo.dss_shared_disks, - self.context.clusterInfo.dss_pri_disks, - self.context.user).items(): + count = 0 + for vgname, dss_disk in { + **self.context.clusterInfo.dss_shared_disks, + **self.context.clusterInfo.dss_pri_disks + }.items(): au_size = '4096' - if dss_disk.find('shared') == -1: + if count != 0: au_size = '65536' + count += 1 source_cmd = "source %s; " % self.context.mpprcFile show_cmd = source_cmd + f'dsscmd showdisk -g {vgname} -s vg_header' cv_cmd = source_cmd + f'dsscmd cv -g {vgname} -v {dss_disk} -s {au_size}' diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index d8c85af3..1aee3a06 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -887,34 +887,15 @@ Common options: ''' Preparing the VG Configuration File ''' - - dss_dict = {} dss_vg_ini = os.path.realpath( os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini')) - - lun_map = self.clusterInfo.dss_vg_info - lun_dss_vgname = self.clusterInfo.dss_vgname - - # dss_vg_info checker - infos = list(filter(None, re.split(r':|,', lun_map))) - - # The volume name must correspond to the disk. - if (lun_map.count(':') != lun_map.count(',') + 1) or ( - not infos) or (infos and len(infos) % 2 != 0): - raise Exception(ErrorCode.GAUSS_504["GAUSS_50414"] % - "The volume name must correspond to the disk.") - - # The shared volume must be in vg_config. - if lun_dss_vgname not in infos[::2]: - raise Exception(ErrorCode.GAUSS_504["GAUSS_50419"] % - (lun_dss_vgname, lun_map)) - - for i in range(0, len(infos), 2): - key = infos[i] - value = infos[i + 1] - dss_dict[key] = value - - context = [':'.join([k, v]) for k, v in dss_dict.items()] + + context = [ + ':'.join([k, v]) for k, v in { + **self.clusterInfo.dss_shared_disks, + **self.clusterInfo.dss_pri_disks + }.items() + ] FileUtil.write_custom_context( dss_vg_ini, context, authority=DefaultValue.KEY_FILE_MODE_IN_OS) -- Gitee From b61281d802a55712b50bdbabde331e3247ba0914 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Wed, 30 Aug 2023 16:21:07 +0800 Subject: [PATCH 80/96] Revert "fix bug for priate vgname not found" --- script/gspylib/component/DSS/dss_comp.py | 38 ++++++++++++++++--- .../component/Kernel/DN_OLAP/DN_OLAP.py | 7 +++- script/impl/install/OLAP/InstallImplOLAP.py | 12 +++--- script/local/PreInstallUtility.py | 14 +++---- 4 files changed, 49 insertions(+), 22 deletions(-) diff --git a/script/gspylib/component/DSS/dss_comp.py b/script/gspylib/component/DSS/dss_comp.py index 66f0f915..2329f75d 100644 --- a/script/gspylib/component/DSS/dss_comp.py +++ b/script/gspylib/component/DSS/dss_comp.py @@ -64,22 +64,48 @@ class DssInst(): else: raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % self.cfg_path) return items + + @staticmethod + def get_private_vg_num(dss_home): + ''' + Obtaining Private Volumes + ''' + + + vg_cfg = os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini') + if os.path.isfile(vg_cfg): + try: + with open(vg_cfg, "r") as fp: + context = fp.read().strip() + pris = re.findall( + '(.*):/dev/.*private_.*', context) + if pris: + return len(pris) + else: + raise Exception(ErrorCode.GAUSS_504["GAUSS_50416"] % + 'in dss_vg_conf.ini') + except Exception as eds: + raise Exception(ErrorCode.GAUSS_504["GAUSS_50414"] % eds) + else: + raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % vg_cfg) @staticmethod - def get_private_vgname_by_ini(dss_home, dss_id): + def get_private_vgname_by_ini(dss_home, dss_id, xlog_in_one_priv_vg): ''' Obtaining a Private Volume ''' + + if xlog_in_one_priv_vg: + dss_id = 0 vg_cfg = os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini') if os.path.isfile(vg_cfg): try: with open(vg_cfg, "r") as fp: context = fp.read().strip() - pris = re.findall('(.+):(.+)', context) - if pris and len(pris) == 2: - return pris[1][0].strip() - elif pris and len(pris) > dss_id + 1: - return pris[dss_id + 1][0].strip() + pris = re.findall( + '(.*):/dev/.*private_{}'.format(str(dss_id)), context) + if pris: + return pris[0].strip() else: raise Exception(ErrorCode.GAUSS_504["GAUSS_50416"] % 'in dss_vg_conf.ini') diff --git a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py index db08fc28..5a94e885 100644 --- a/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py +++ b/script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py @@ -151,8 +151,13 @@ class DN_OLAP(Kernel): 'dss_nodes_list', self.dss_config, action='decode') cfg_context = DssInst.get_dms_url(dss_nodes_list) + xlog_in_one_priv_vg = False + infos = list(filter(None, re.split(r':|:|,', dss_nodes_list))) + pri_vg_num = DssInst.get_private_vg_num(dss_home) # when use one private vg for xlog, vgname should get from inst_id=0 - pri_vgname = DssInst.get_private_vgname_by_ini(dss_home, inst_id) + if (pri_vg_num < len(infos[::3]) and pri_vg_num == 1): + xlog_in_one_priv_vg = True + pri_vgname = DssInst.get_private_vgname_by_ini(dss_home, inst_id, xlog_in_one_priv_vg) cmd += " -n --vgname=\"{}\" --enable-dss --dms_url=\"{}\" -I {}" \ " --socketpath=\"{}\"".format( "+{},+{}".format(vgname, pri_vgname), cfg_context, inst_id, diff --git a/script/impl/install/OLAP/InstallImplOLAP.py b/script/impl/install/OLAP/InstallImplOLAP.py index 84da2c12..cb1810fc 100644 --- a/script/impl/install/OLAP/InstallImplOLAP.py +++ b/script/impl/install/OLAP/InstallImplOLAP.py @@ -196,15 +196,13 @@ class InstallImplOLAP(InstallImpl): return self.context.logger.log('Start to create the dss vg.') - count = 0 - for vgname, dss_disk in { - **self.context.clusterInfo.dss_shared_disks, - **self.context.clusterInfo.dss_pri_disks - }.items(): + for vgname, dss_disk in UdevContext.get_all_vgname_disk_pair( + self.context.clusterInfo.dss_shared_disks, + self.context.clusterInfo.dss_pri_disks, + self.context.user).items(): au_size = '4096' - if count != 0: + if dss_disk.find('shared') == -1: au_size = '65536' - count += 1 source_cmd = "source %s; " % self.context.mpprcFile show_cmd = source_cmd + f'dsscmd showdisk -g {vgname} -s vg_header' cv_cmd = source_cmd + f'dsscmd cv -g {vgname} -v {dss_disk} -s {au_size}' diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 1aee3a06..95b87c16 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -887,15 +887,14 @@ Common options: ''' Preparing the VG Configuration File ''' + dss_vg_ini = os.path.realpath( os.path.join(dss_home, 'cfg', 'dss_vg_conf.ini')) + lun_map = UdevContext.get_all_vgname_disk_pair( + self.clusterInfo.dss_shared_disks, self.clusterInfo.dss_pri_disks, + self.user) - context = [ - ':'.join([k, v]) for k, v in { - **self.clusterInfo.dss_shared_disks, - **self.clusterInfo.dss_pri_disks - }.items() - ] + context = [':'.join([k, v]) for k, v in lun_map.items()] FileUtil.write_custom_context( dss_vg_ini, context, authority=DefaultValue.KEY_FILE_MODE_IN_OS) @@ -911,8 +910,7 @@ Common options: self.prepareGivenPath(dss_cfg, False) self.prepareGivenPath(dss_log, False) self.prepare_dss_inst_ini(dss_home, dss_id) - # this function does not need to be detected under multipath - # self.prepare_dss_soft_link() + self.prepare_dss_soft_link() self.prepare_dss_vg_ini(dss_home) -- Gitee From 4d2e213bf1c31469349f5bf963d352e81593ac55 Mon Sep 17 00:00:00 2001 From: xue_meng_en <1836611252@qq.com> Date: Fri, 1 Sep 2023 09:21:55 +0800 Subject: [PATCH 81/96] =?UTF-8?q?=E6=B5=81=E5=BC=8F=E5=AE=B9=E7=81=BE?= =?UTF-8?q?=E5=88=87=E6=8D=A2=E4=B8=BB=E9=99=8D=E5=A4=87=E5=89=8D=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E9=9B=86=E7=BE=A4=E5=8F=AA=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../streaming_base.py | 57 ++++++++++++++++++- .../streaming_constants.py | 1 + .../streaming_disaster_recovery_switchover.py | 6 ++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/script/impl/streaming_disaster_recovery/streaming_base.py b/script/impl/streaming_disaster_recovery/streaming_base.py index 5eddb634..ab67686c 100644 --- a/script/impl/streaming_disaster_recovery/streaming_base.py +++ b/script/impl/streaming_disaster_recovery/streaming_base.py @@ -2101,6 +2101,9 @@ class StreamingBase(object): cluster_normal_status = [DefaultValue.CLUSTER_STATUS_NORMAL, DefaultValue.CLUSTER_STATUS_DEGRADED] self.check_cluster_status(cluster_normal_status, check_current=True) + if action_flag == StreamingConstants.ACTION_SWITCHOVER: + self.set_cluster_read_only_params({"default_transaction_read_only": "off"}) + self.revise_dn_readonly_status_in_switch_process("end") cluster_info = self.query_cluster_info() self.parse_cluster_status(current_status=cluster_info) if action_flag != StreamingConstants.ACTION_SWITCHOVER: @@ -2113,9 +2116,58 @@ class StreamingBase(object): else: self.update_streaming_info("cluster", "archive") + def set_cluster_read_only_params(self, params_dict, guc_type="reload"): + """ + set datanode params + """ + if not params_dict: + return + + cmd = "" + for param_name, value in params_dict.items(): + cmd += " -c \"%s=%s\"" % (param_name, value) + guc_cmd = "source %s; gs_guc %s -Z datanode -N all -I all %s" % (EnvUtil.getMpprcFile(), guc_type, cmd) + (status, output) = CmdUtil.retryGetstatusoutput(guc_cmd) + self.logger.debug("The params dict %s %s status %s, output %s." % (params_dict, guc_type, status, output)) + + def revise_dn_readonly_status_in_switch_process(self, action, guc_type="reload"): + """ + revise dn readonly status in switch process + """ + file_name = os.path.join(EnvUtil.getTmpDirFromEnv(), StreamingConstants.SWITCH_ENABLE_READ_ONLY_FILE) + if action == "start": + all_cms = [ + cm_inst for dbonde in self.cluster_info.dbNodes for cm_inst in dbonde.cmservers + if cm_inst.hostname in self.normal_cm_ips + ] + for cm_inst in all_cms: + cmd = "source %s; pssh -s -H %s \"grep enable_transaction_read_only " \ + "%s/cm_server.conf\"" % (EnvUtil.getMpprcFile(), cm_inst.hostname, cm_inst.datadir) + (status, output) = CmdUtil.retryGetstatusoutput(cmd) + self.logger.debug("Check enable transaction read only status:%s, output:%s." % (status, output)) + if status != 0 or output.find("=") < -1: + continue + params_dict = {"enable_transaction_read_only": output.split("=")[-1].strip()} + with os.fdopen(os.open(file_name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + DefaultValue.KEY_FILE_MODE_IN_OS), "w") as fp_w: + json.dump(params_dict, fp_w) + self.set_cmserver_guc("enable_transaction_read_only", "off", guc_type) + self.logger.debug("The parameter enable_transaction_read_only is disabled.") + else: + if not os.path.isfile(file_name): + self.logger.debug("The enable transaction read only file not exist.") + return + content = DefaultValue.obtain_file_content(file_name, is_list=False) + loads_value = json.loads(content) + param_value = loads_value.get("enable_transaction_read_only") + value = param_value if param_value else "on" + self.set_cmserver_guc("enable_transaction_read_only", value, guc_type) + os.remove(file_name) + self.logger.debug("The parameter enable_transaction_read_only is enabled.") + def streaming_clean_archive_slot(self): """ - drop lot_type is physical and slot_name not contain (gs_roach_full，gs_roach_inc， + drop lot_type is physical and slot_name not contain (gs_roach_full, gs_roach_inc, cn_xxx，dn_xxx, dn_xxx_hadr) on all cn node and all primary dn node if the slot_name exists when the disaster cluster become primary cluster """ @@ -2266,6 +2318,9 @@ class StreamingBase(object): self.stream_disaster_set_cmagent_guc("agent_backup_open", "0", "set") self.set_stream_cluster_run_mode_guc("set", fail_over=True) self.write_streaming_step("3_set_backup_open_for_failover") + if action_flag == StreamingConstants.ACTION_SWITCHOVER: + self.revise_dn_readonly_status_in_switch_process("start", guc_type="set") + self.set_cluster_read_only_params({"default_transaction_read_only": "on"}, guc_type="set") # 4.Delete the relevant guc parameters and remove the disaster tolerance relationship # based on streaming disaster recovery cluster, No need to delete for switchover. if not action_flag: diff --git a/script/impl/streaming_disaster_recovery/streaming_constants.py b/script/impl/streaming_disaster_recovery/streaming_constants.py index 0a1d3120..0b04a0c7 100644 --- a/script/impl/streaming_disaster_recovery/streaming_constants.py +++ b/script/impl/streaming_disaster_recovery/streaming_constants.py @@ -52,6 +52,7 @@ class StreamingConstants: HADR_ESTABLISH_STAT = ".hadr_establish_stat" STREAM_DISTRIBUTE_ACTION = "distribute_stream_failover" + SWITCH_ENABLE_READ_ONLY_FILE = ".switch_readonly_stat_file" # GUC CHANGE MAP GUC_CHANGE_MAP = {"most_available_sync": "on", "synchronous_commit": "on"} diff --git a/script/impl/streaming_disaster_recovery/streaming_modules/streaming_disaster_recovery_switchover.py b/script/impl/streaming_disaster_recovery/streaming_modules/streaming_disaster_recovery_switchover.py index 25ff78e7..2956c9d3 100644 --- a/script/impl/streaming_disaster_recovery/streaming_modules/streaming_disaster_recovery_switchover.py +++ b/script/impl/streaming_disaster_recovery/streaming_modules/streaming_disaster_recovery_switchover.py @@ -106,6 +106,8 @@ class StreamingSwitchoverHandler(StreamingBase): self.set_cmagent_guc("agent_backup_open", "2", "set") self.set_stream_cluster_run_mode_guc("set") self.write_streaming_step("3_set_backup_open_2_done") + self.revise_dn_readonly_status_in_switch_process("start", guc_type="set") + self.set_cluster_read_only_params({"default_transaction_read_only": "on"}, guc_type="set") if stream_disaster_step < 4: self.update_streaming_info(StreamingConstants.ACTION_SWITCHOVER, "50%") self.remove_cluster_maintance_file_for_switchover() @@ -115,6 +117,8 @@ class StreamingSwitchoverHandler(StreamingBase): if stream_disaster_step < 5: self.wait_for_normal(timeout=self.params.waitingTimeout, streaming_switchover="streaming_switchover") + self.set_cluster_read_only_params({"default_transaction_read_only": "off"}) + self.revise_dn_readonly_status_in_switch_process("end") self.streaming_clean_replication_slot() self.update_streaming_info("cluster", "recovery") except Exception as error: @@ -127,6 +131,8 @@ class StreamingSwitchoverHandler(StreamingBase): if rollback_step < 4 or (rollback_step >= 4 and self.streaming_switchover_roll_back_condition()): self.streaming_switchover_roll_back(update_query=True) + self.set_cluster_read_only_params({"default_transaction_read_only": "off"}, guc_type="set") + self.revise_dn_readonly_status_in_switch_process("end", guc_type="set") self.clean_step_file() raise Exception(error) self.remove_hadr_switchover_process_file() -- Gitee From 886b59f5cb58857d8fec856c53aa46db3fb3a71f Mon Sep 17 00:00:00 2001 From: z00793368 Date: Sat, 12 Aug 2023 16:24:44 +0800 Subject: [PATCH 82/96] fix A12:time consis < 6s --- script/gs_checkos | 15 ++++++++++----- script/gspylib/inspection/config/items.xml | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/script/gs_checkos b/script/gs_checkos index 038d6950..1d29a3ac 100644 --- a/script/gs_checkos +++ b/script/gs_checkos @@ -94,6 +94,7 @@ g_OSCheckOpts = { 'B7': ['Setting items', '[ Set RemoveIPC value ]', 'Normal', 'OK', 'OK'], 'B8': ['Setting items', '[ Set Session Process ]', 'Normal', 'OK', 'OK']} DEFAULT_INTERVAL = 60 +DEFAULT_CONSISTENCY_INTERVAL = 6 CHECK_ITEMNUMLIST = ['A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'A8', 'A9', 'A10', 'A11', 'A12', 'A13', 'A14'] SET_ITEMNUMLIST = ['B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8'] @@ -955,26 +956,30 @@ def checkTimeConsistency(): nodeValue, "Failed to obtain localtime information.") else: baseTime = datetime.strptime(keystr, "%Y-%m-%d %H:%M:%S") - startTime = baseTime - timedelta(seconds=DEFAULT_INTERVAL) - endTime = baseTime + timedelta(seconds=DEFAULT_INTERVAL) + if g_clusterInfo.enable_dss == "on": + startTime = baseTime - timedelta(seconds=DEFAULT_CONSISTENCY_INTERVAL) + endTime = baseTime + timedelta(seconds=DEFAULT_CONSISTENCY_INTERVAL) + else: + startTime = baseTime - timedelta(seconds=DEFAULT_INTERVAL) + endTime = baseTime + timedelta(seconds=DEFAULT_INTERVAL) for node in list(status.keys()): if (status[node] == DefaultValue.SUCCESS): tmpstr = outputMap[node].strip().split(',')[1].strip() tmpTime = datetime.strptime(tmpstr, "%Y-%m-%d %H:%M:%S") if (tmpTime < startTime or tmpTime > endTime): - g_OSCheckOpts['A12'][2] = 'Warning' + g_OSCheckOpts['A12'][2] = 'Abnormal' parRes += " [%s]\n " \ "The current system time = (%s)\n"\ % (node, tmpstr) detail_msg += " [%s]\n Variable:" \ "'current system time' RealValue:" \ "'%s' ExpectedValue:'%s' " \ - "[Warning]\n"\ + "[Abnormal]\n"\ % (node, tmpstr, baseTime.strftime("%Y-%m-%d %H:%M:%S")) - if (g_OSCheckOpts['A12'][2] == 'Warning'): + if (g_OSCheckOpts['A12'][2] == 'Warning' or g_OSCheckOpts['A12'][2] == 'Abnormal'): g_OSCheckOpts['A12'][3] = "\n%s" % parRes else: nodeValue = list(status.keys())[0].strip() diff --git a/script/gspylib/inspection/config/items.xml b/script/gspylib/inspection/config/items.xml index bb4143c9..8026aced 100644 --- a/script/gspylib/inspection/config/items.xml +++ b/script/gspylib/inspection/config/items.xml @@ -20,7 +20,7 @@ 检查是否安装并开启ntpq服务，并与ntp时钟源同步。 - 检查系统NTPD服务，如果服务开启且各节点时间误差在1分钟以内则检查项通过，否则检查项不通过。 + 检查系统NTPD服务，如果服务开启且各节点时间误差在60s(资源池化部署环境下：6s)以内则检查项通过，否则检查项不通过。 os user -- Gitee From a9e5fe9eb9203f1a006851122ba94c374a52a5db Mon Sep 17 00:00:00 2001 From: xue_meng_en <1836611252@qq.com> Date: Wed, 6 Sep 2023 16:06:50 +0800 Subject: [PATCH 83/96] =?UTF-8?q?gs=5Fom=20-t=20query=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E9=9B=86=E7=BE=A4=E7=8A=B6=E6=80=81=E6=97=B6debug=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E5=A2=9E=E5=8A=A0azPriority=E7=9A=84=E4=BF=A1?= =?UTF-8?q?=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/common/DbClusterInfo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/gspylib/common/DbClusterInfo.py b/script/gspylib/common/DbClusterInfo.py index 91564fca..d7f3cb48 100644 --- a/script/gspylib/common/DbClusterInfo.py +++ b/script/gspylib/common/DbClusterInfo.py @@ -652,6 +652,8 @@ class instanceInfo(): ret += ",controlPort=%s" % self.controlPort if self.azName: ret += ",azName=%s" % self.azName + if hasattr(self, 'azPriority') and self.azPriority > 0: + ret += ",azPriority=%s" % self.azPriority if self.clusterName: ret += ",clusterName=%s" % self.clusterName if self.peerPort: -- Gitee From 5bf5ab5c84f1f17ebd23ede21fd343b3494c6268 Mon Sep 17 00:00:00 2001 From: xue_meng_en <1836611252@qq.com> Date: Sat, 9 Sep 2023 11:53:20 +0800 Subject: [PATCH 84/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=B1=E5=9C=B0?= =?UTF-8?q?=E5=8D=87=E7=BA=A7=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/upgrade/UpgradeImpl.py | 14 ++++++++++++-- script/local/UpgradeUtility.py | 6 +++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index 0540a7f6..9df252fa 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -3843,9 +3843,14 @@ class UpgradeImpl: Start cluster """ cm_strategy = self.get_upgrade_cm_strategy() + if cm_strategy == -1: + raise Exception("cm_strategy = -1. This is usually impossible.\n" + "Hint: please check \n" + "1. whether the old and new static files exist." + "2. whether the upgrade strategy is \"have_cm\" to \"have_no_cm\".") if cm_strategy == 0: self.startCluster() - if cm_strategy == 1: + elif cm_strategy == 1: if is_final: self.om_start_cluster() else: @@ -3858,9 +3863,14 @@ class UpgradeImpl: Start cluster """ cm_strategy = self.get_upgrade_cm_strategy() + if cm_strategy == -1: + raise Exception("cm_strategy = -1. This is usually impossible.\n" + "Hint: please check \n" + "1. whether the old and new static files exist." + "2. whether the upgrade strategy is \"have_cm\" to \"have_no_cm\".") if cm_strategy == 0: self.stopCluster() - if cm_strategy == 1: + elif cm_strategy == 1: if is_final: self.om_stop_cluster() else: diff --git a/script/local/UpgradeUtility.py b/script/local/UpgradeUtility.py index 2c08f7fe..724a0271 100644 --- a/script/local/UpgradeUtility.py +++ b/script/local/UpgradeUtility.py @@ -1584,8 +1584,12 @@ def install_cm_agent(node_info, gauss_home): def install_cm_server(node_info, gauss_home): """ - Install CM agent instance + Install CM server instance """ + if len(node_info.cmservers) == 0: + g_logger.debug("No need to install cm_server becuase the user did not " + "configure the cm_server information of the current node.") + return g_logger.debug("Start install cm_server instance.") server_component = CM_OLAP() server_component.instInfo = node_info.cmservers[0] -- Gitee From d4724d8150cf29679161ec74df7fb67f47df8fa1 Mon Sep 17 00:00:00 2001 From: "jerad.k" Date: Fri, 9 Jun 2023 16:11:14 +0800 Subject: [PATCH 85/96] fix(I7C5OG): add 'KRB5_CONFIG' to env file when 'gs_om -t kerberos -m install' --- script/local/KerberosUtility.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/script/local/KerberosUtility.py b/script/local/KerberosUtility.py index 8dff3250..7472c948 100644 --- a/script/local/KerberosUtility.py +++ b/script/local/KerberosUtility.py @@ -357,6 +357,15 @@ class Kerberos(): if status != 0: raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd + "Error:\n%s." % output) + + # SET KRB5_CONFIG + cmd = "echo \"export KRB5_CONFIG=%s/krb5.conf\"" \ + " >> %s" % (os.path.dirname(g_opts.mpprcFile), + g_opts.mpprcFile) + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % + cmd + "Error:\n%s." % output) g_logger.log("Config environment variable KRB5RCACHETYPE " "successfully.") except Exception as e: -- Gitee From 2e9959da5a724f652b855207b8a0de53599fbe8b Mon Sep 17 00:00:00 2001 From: liuheng Date: Wed, 13 Sep 2023 09:47:57 +0800 Subject: [PATCH 86/96] =?UTF-8?q?fix=20gs=5Fsshexkey=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E4=BA=92=E4=BF=A1=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_preinstall | 39 ++-------------------- script/gs_sshexkey | 2 ++ script/gspylib/common/copy_python_lib.py | 42 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 37 deletions(-) create mode 100644 script/gspylib/common/copy_python_lib.py diff --git a/script/gs_preinstall b/script/gs_preinstall index 98512b2a..8c6b56ec 100644 --- a/script/gs_preinstall +++ b/script/gs_preinstall @@ -51,43 +51,8 @@ else: check_os_and_package_arch() check_python_version() check_python_compiler_option() - source = os.path.join(sys.path[0], '../lib/bcrypt/lib3.' + \ - str(sys.version_info[1]), '_bcrypt.abi3.so') - dest = os.path.join(sys.path[0], '../lib/bcrypt/') - cmd = f"cp {source} {dest}" - (status, output) = subprocess.getstatusoutput(cmd) - if status != 0: - GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % - (output, cmd)) - - source = os.path.join(sys.path[0], '../lib/_cffi_backend_3.' + \ - str(sys.version_info[1]), '_cffi_backend.so') - dest = os.path.join(sys.path[0], '../lib/') - cmd = f"cp {source} {dest}" - (status, output) = subprocess.getstatusoutput(cmd) - if status != 0: - GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % - (output, cmd)) - - - source = os.path.join(sys.path[0], '../lib/cryptography/hazmat/bindings/lib3.' + \ - str(sys.version_info[1]), '*.so') - dest = os.path.join(sys.path[0], '../lib/cryptography/hazmat/bindings/') - cmd = f"cp {source} {dest}" - (status, output) = subprocess.getstatusoutput(cmd) - if status != 0: - GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % - (output, cmd)) - - - source = os.path.join(sys.path[0], '../lib/nacl/lib3.' + \ - str(sys.version_info[1]), '_sodium.abi3.so') - dest = os.path.join(sys.path[0], '../lib/nacl/') - cmd = f"cp {source} {dest}" - (status, output) = subprocess.getstatusoutput(cmd) - if status != 0: - GaussLog.exitWithError("cp file failed.\nError:%s\nThe cmd is: %s\n" % - (output, cmd)) + from gspylib.common.copy_python_lib import copy_lib + copy_lib() from gspylib.common.Common import DefaultValue from gspylib.common.ErrorCode import ErrorCode diff --git a/script/gs_sshexkey b/script/gs_sshexkey index 8596891f..9fc01623 100644 --- a/script/gs_sshexkey +++ b/script/gs_sshexkey @@ -49,8 +49,10 @@ from base_utils.os.password_util import PasswordUtil from base_utils.os.net_util import NetUtil from subprocess import PIPE from base_utils.common.fast_popen import FastPopen +from gspylib.common.copy_python_lib import copy_lib DefaultValue.doConfigForParamiko() +copy_lib() try: import paramiko diff --git a/script/gspylib/common/copy_python_lib.py b/script/gspylib/common/copy_python_lib.py new file mode 100644 index 00000000..95efe9f8 --- /dev/null +++ b/script/gspylib/common/copy_python_lib.py @@ -0,0 +1,42 @@ +import os +import sys +import subprocess + + +def copy_lib(): + current_path = os.path.dirname(os.path.abspath(__file__)) + source = os.path.join(current_path, '../../../lib/bcrypt/lib3.' + \ + str(sys.version_info[1]), '_bcrypt.abi3.so') + dest = os.path.join(current_path, '../../../lib/bcrypt/') + cmd = f"cp {source} {dest}" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + + source = os.path.join(current_path, '../../../lib/_cffi_backend_3.' + \ + str(sys.version_info[1]), '_cffi_backend.so') + dest = os.path.join(current_path, '../../../lib/') + cmd = f"cp {source} {dest}" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + + source = os.path.join(current_path, '../../../lib/cryptography/hazmat/bindings/lib3.' + \ + str(sys.version_info[1]), '*.so') + dest = os.path.join(current_path, '../../../lib/cryptography/hazmat/bindings/') + cmd = f"cp {source} {dest}" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) + + source = os.path.join(current_path, '../../../lib/nacl/lib3.' + \ + str(sys.version_info[1]), '_sodium.abi3.so') + dest = os.path.join(current_path, '../../../lib/nacl/') + cmd = f"cp {source} {dest}" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception("cp file failed.\nError:%s\nThe cmd is: %s\n" % + (output, cmd)) -- Gitee From 78b6fc390d1b8d9f4d333ecfa94b7724b234ed08 Mon Sep 17 00:00:00 2001 From: muyulinzhong Date: Wed, 13 Sep 2023 21:23:40 +0800 Subject: [PATCH 87/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BD=93=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E6=A8=A1=E5=9D=97=E4=BE=9D=E8=B5=96=E4=B8=8D=E6=BB=A1?= =?UTF-8?q?=E8=B6=B3=E5=BE=AA=E7=8E=AF=E6=8A=A5=E9=94=99=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_sshexkey | 8 ++++---- script/gspylib/threads/SshTool.py | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/script/gs_sshexkey b/script/gs_sshexkey index 8596891f..885ac3d0 100644 --- a/script/gs_sshexkey +++ b/script/gs_sshexkey @@ -64,10 +64,10 @@ except ImportError as ex: os.environ['LD_LIBRARY_PATH'] = clib_path else: os.environ['LD_LIBRARY_PATH'] = clib_path + ":" + ld_path - try: - os.execl(sys.executable, sys.executable, *sys.argv) - except Exception as ex: - sys.exit("Failed to set the enviroment variable: %s" % str(ex)) + try: + os.execl(sys.executable, sys.executable, *sys.argv) + except Exception as ex: + sys.exit("Failed to set the enviroment variable: %s" % str(ex)) import paramiko except ImportError as ex: raise Exception(ErrorCode.GAUSS_522["GAUSS_52200"] % str(ex)) diff --git a/script/gspylib/threads/SshTool.py b/script/gspylib/threads/SshTool.py index e965e4dd..d00c9cd0 100644 --- a/script/gspylib/threads/SshTool.py +++ b/script/gspylib/threads/SshTool.py @@ -41,7 +41,6 @@ from gspylib.common.Constants import Constants try: import paramiko except ImportError as ex: - print(ex) try: local_path = os.path.dirname(os.path.realpath(__file__)) clib_path = os.path.realpath(os.path.join(local_path, "../../gspylib/clib/")) @@ -51,10 +50,10 @@ except ImportError as ex: os.environ['LD_LIBRARY_PATH'] = clib_path else: os.environ['LD_LIBRARY_PATH'] = clib_path + ":" + ld_path - try: - os.execl(sys.executable, sys.executable, *sys.argv) - except Exception as ex: - sys.exit("Failed to set the enviroment variable: %s" % str(ex)) + try: + os.execl(sys.executable, sys.executable, *sys.argv) + except Exception as ex: + sys.exit("Failed to set the enviroment variable: %s" % str(ex)) import paramiko except ImportError as ex: raise Exception(ErrorCode.GAUSS_522["GAUSS_52200"] % str(ex)) -- Gitee From a60795201cbe3ea187496122c0fd19aa8dce0019 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Mon, 4 Sep 2023 21:01:16 +0800 Subject: [PATCH 88/96] =?UTF-8?q?=E6=A3=80=E6=9F=A5=E9=A1=B9A9=E5=8F=AA?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E9=9B=86=E7=BE=A4=E4=B8=AD=E7=9A=84=E8=AE=BE?= =?UTF-8?q?=E5=A4=87=E7=9B=B8=E5=85=B3=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/local/LocalCheckOS.py | 94 +++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 11 deletions(-) diff --git a/script/local/LocalCheckOS.py b/script/local/LocalCheckOS.py index 6f39fd6f..8daac4e0 100644 --- a/script/local/LocalCheckOS.py +++ b/script/local/LocalCheckOS.py @@ -26,6 +26,7 @@ import getopt import subprocess import platform import time +import re from datetime import datetime localDirPath = os.path.dirname(os.path.realpath(__file__)) @@ -93,6 +94,7 @@ g_logger = None g_opts = None g_clusterInfo = None netWorkBondInfo = None +g_readlist = None ########################################################################### @@ -1512,6 +1514,59 @@ def CheckLinuxMounts(): ############################################################################# +def IterateClass(obj): + """ + function : Check item in class + input : obj + output : list + """ + if not hasattr(obj, '__dict__'): + return + + for attr_name in obj.__dict__: + attr_value = getattr(obj, attr_name) + if isinstance(attr_value, list): + for item in attr_value: + if hasattr(item, '__dict__'): + IterateClass(item) + else: + g_readlist.append(item) + elif isinstance(attr_value, dict): + for key, value in attr_value.items(): + if hasattr(value, '__dict__'): + IterateClass(value) + else: + g_readlist.append(value) + elif hasattr(attr_value, '__dict__'): + IterateClass(attr_value) + else: + g_readlist.append(attr_value) + +def IsLinuxPath(input_str): + """ + function : Check whether a character string is a file path + input : String + output : Bool + """ + linux_path_pattern = r'^/([A-Za-z0-9_\-]+/)*[A-Za-z0-9_\-]+\/*$' + return re.match(linux_path_pattern, input_str) is not None + +def GetMountInfo(file_path): + """ + function : Determine the mount disk of the path + input : String + output : Bool + """ + try: + dev = os.stat(file_path).st_dev + except FileNotFoundError: + return None + + for partition in psutil.disk_partitions(all=True): + if os.stat(partition.mountpoint).st_dev == dev: + return partition.device + return None + def CheckBlockdev(isSetting=False): """ function : Check Block dev @@ -1519,19 +1574,34 @@ def CheckBlockdev(isSetting=False): output : NA """ expectedReadAhead = "16384" + my_mount = [] + IterateClass(g_clusterInfo) + + for path in g_readlist: + mount_info = GetMountInfo(str(path)) + if mount_info is not None: + my_mount.append(mount_info) + else: + continue + unique_elements = list(set(my_mount)) + data = collectBlockdev() for dev in list(data.ra.keys()): - ra = data.ra[dev] - if int(ra) < int(expectedReadAhead): - if not isSetting: - g_logger.log("On device (%s) 'blockdev readahead' RealValue" - " '%s' ExpectedValue '%s'." - % (dev, ra, expectedReadAhead)) - else: - SetBlockdev(expectedReadAhead, dev) - g_logger.log("On device (%s) set 'blockdev readahead' from" - " '%s' to '%s'." % (dev, ra, expectedReadAhead)) - + for s in unique_elements: + if dev not in s: + continue + ra = data.ra[dev] + if int(ra) < int(expectedReadAhead): + if not isSetting: + g_logger.log("On device (%s) 'blockdev readahead' RealValue" + " '%s' ExpectedValue '%s'." + % (dev, ra, expectedReadAhead)) + else: + SetBlockdev(expectedReadAhead, dev) + g_logger.log("On device (%s) set 'blockdev readahead' from" + " '%s' to '%s'." % (dev, ra, expectedReadAhead)) + + def SetBlockdev(expectedReadAhead, devname): """ @@ -1965,6 +2035,8 @@ def initGlobals(): """ global g_logger global g_clusterInfo + global g_readlist + g_readlist = [] g_logger = GaussLog(g_opts.logFile, "LocalCheckOS") -- Gitee From a133a42d1c028d378900947f578206a34e064cf3 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Tue, 12 Sep 2023 15:31:29 +0800 Subject: [PATCH 89/96] check the enable_dss environment before gs_install --- script/gspylib/common/ErrorCode.py | 5 +++-- script/impl/install/InstallImpl.py | 18 ++++++++++++++++++ script/impl/preinstall/PreinstallImpl.py | 24 ++++++++++++++++++++++++ script/impl/uninstall/UninstallImpl.py | 23 +++++++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index 4ab660d2..82de327f 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -998,8 +998,9 @@ class ErrorCode(): 'GAUSS_53031': "[GAUSS-53031] : The cluster is None.", 'GAUSS_53032': "[GAUSS-53032] : The speed limit must " "be a nonnegative integer.", - 'GAUSS_53033': "[GAUSS-53033] : Invalid User : %s." - + 'GAUSS_53033': "[GAUSS-53033] : Invalid User : %s.", + 'GAUSS_53034': "[GAUSS-53034] : In the resource pooling environment, do not run " + "gs_install after gs_uninstall. please gs_preinstall first." } ########################################################################### diff --git a/script/impl/install/InstallImpl.py b/script/impl/install/InstallImpl.py index 8e21db04..921c1b76 100644 --- a/script/impl/install/InstallImpl.py +++ b/script/impl/install/InstallImpl.py @@ -89,6 +89,8 @@ class InstallImpl: function: run method """ try: + # check enable_dss + self.check_enable_dss() # check timeout time. # Notice: time_out is not supported under TP branch self.checkTimeout() @@ -113,6 +115,22 @@ class InstallImpl: except Exception as e: GaussLog.exitWithError(str(e)) + def check_enable_dss(self): + """ + function: Check enable_dss + input : NA + output: NA + """ + is_enabledssset = EnvUtil.getEnv("ENABLE_DSS") + cluster_info = dbClusterInfo() + cluster_info.initFromXml(self.context.xmlFile) + + if is_enabledssset and cluster_info.enable_dss == 'on': + raise Exception(ErrorCode.GAUSS_530["GAUSS_53034"]) + else: + self.context.logger.log( + "Successfully checked gs_uninstall on every node.") + def checkSyncNode(self): """ function: check syncNode diff --git a/script/impl/preinstall/PreinstallImpl.py b/script/impl/preinstall/PreinstallImpl.py index a4a5c552..237af3c9 100644 --- a/script/impl/preinstall/PreinstallImpl.py +++ b/script/impl/preinstall/PreinstallImpl.py @@ -1338,6 +1338,29 @@ class PreinstallImpl: output : NA """ pass + + def check_enabledssinenv(self, source_file): + """ + function: check enabledss in env + input : NA + output : NA + """ + if self.context.clusterInfo.enable_dss == 'on': + file_lines = [] + found_enabledss = False + is_enabledssset = "export ENABLE_DSS=" + with open(source_file, "r") as file: + for line in file: + if is_enabledssset in line: + found_enabledss = True + else: + file_lines.append(line) + + if found_enabledss: + with open(source_file, "w") as file: + file.writelines(file_lines) + else: + return def checkRepeat(self): """ @@ -1358,6 +1381,7 @@ class PreinstallImpl: self.context.logger.debug( "There is no environment file, skip check repeat.") return [] + self.check_enabledssinenv(source_file) with open(source_file, 'r') as f: env_list = f.readlines() new_env_list = [] diff --git a/script/impl/uninstall/UninstallImpl.py b/script/impl/uninstall/UninstallImpl.py index eb30a1da..78bbf7b5 100644 --- a/script/impl/uninstall/UninstallImpl.py +++ b/script/impl/uninstall/UninstallImpl.py @@ -30,6 +30,9 @@ from domain_utils.cluster_file.cluster_dir import ClusterDir from base_utils.os.env_util import EnvUtil from base_utils.os.file_util import FileUtil from base_utils.os.net_util import NetUtil +from gspylib.common.DbClusterInfo import dbNodeInfo, \ + dbClusterInfo, compareObject +from domain_utils.cluster_file.profile_file import ProfileFile class UninstallImpl: @@ -440,6 +443,25 @@ class UninstallImpl: "Uninstall a single node instead of the gs_dropnode command.") self.localMode = True + def check_dss(self): + """ + function: make sure the etcd process is clean. + input : NA + output: NA + """ + enable_dssLine = 'export ENABLE_DSS=ON\n' + is_enabledssset = EnvUtil.getEnv("ENABLE_DSS") + is_dsshome = EnvUtil.getEnv("DSS_HOME") + + if self.mpprcFile and os.path.isfile(self.mpprcFile): + source_file = self.mpprcFile + else: + source_file = os.path.join("/etc", "profile") + + if is_dsshome and not is_enabledssset: + with open(source_file, 'a') as file: + file.write(enable_dssLine) + def run(self): """ function: Uninstall database cluster @@ -463,6 +485,7 @@ class UninstallImpl: self.CleanRackFile() self.clean_dss_home() self.CleanLog() + self.check_dss() self.logger.log("Uninstallation succeeded.") except Exception as e: self.logger.logExit(str(e)) -- Gitee From 7742519f1d7642f6f190c0d039b9ea85eacdd3a4 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Sat, 16 Sep 2023 16:16:45 +0800 Subject: [PATCH 90/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dgs=5Fsdr=20start=20|=20?= =?UTF-8?q?stop=20=E6=94=AF=E6=8C=81=20--json=20=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../params_handler.py | 2 +- .../streaming_base.py | 21 ++++++++++++++----- script/impl/upgrade/UpgradeImpl.py | 1 + 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/script/impl/streaming_disaster_recovery/params_handler.py b/script/impl/streaming_disaster_recovery/params_handler.py index 0ab963d2..4dbfe161 100644 --- a/script/impl/streaming_disaster_recovery/params_handler.py +++ b/script/impl/streaming_disaster_recovery/params_handler.py @@ -335,7 +335,7 @@ class ParamsHandler(object): self.__reload_hadr_user_info() for param_name, validate in STREAMING_PARAMS_FOR_MODULE[self.params.task].items(): check_value = getattr(self.params, param_name) - if self.params.task == "stop": + if self.params.task == "stop" or self.params.task == "start": if param_name == "xml_path" and not check_value: check_value = getattr(self.params, 'json_path') validate(check_value) diff --git a/script/impl/streaming_disaster_recovery/streaming_base.py b/script/impl/streaming_disaster_recovery/streaming_base.py index ab67686c..5082b572 100644 --- a/script/impl/streaming_disaster_recovery/streaming_base.py +++ b/script/impl/streaming_disaster_recovery/streaming_base.py @@ -866,14 +866,25 @@ class StreamingBase(object): :return:NA """ self.logger.log("Start update pg_hba config.") - FileUtil.cpFile(self.params.xml_path, self.streaming_xml) - cmd = "source %s; %s -U %s -X '%s' --try-reload" % ( - self.mpp_file, OMCommand.getLocalScript( - "Local_Config_Hba"), self.user, self.streaming_xml) + use_xml_action = True + if self.params.xml_path and os.path.isfile(self.params.xml_path): + self.logger.debug("[update_streaming_pg_hba] use xml file.") + FileUtil.cpFile(self.params.xml_path, self.streaming_xml) + cmd = "source %s; %s -U %s -X '%s' --try-reload" % ( + self.mpp_file, OMCommand.getLocalScript( + "Local_Config_Hba"), self.user, self.streaming_xml) + else: + self.logger.debug("[update_streaming_pg_hba] use json file.") + use_xml_action = False + cmd = "source %s; %s -U %s --try-reload" % ( + self.mpp_file, OMCommand.getLocalScript( + "Local_Config_Hba"), self.user) + self.logger.debug("Command for changing instance pg_hba.conf file: %s" % cmd) self.get_all_connection_node_name("update_streaming_pg_hba") try: - self.ssh_tool.scpFiles(self.streaming_xml, self.streaming_file_dir) + if use_xml_action: + self.ssh_tool.scpFiles(self.streaming_xml, self.streaming_file_dir) self.ssh_tool.executeCommand(cmd, hostList=self.connected_nodes) except Exception as error: msg = ErrorCode.GAUSS_516['GAUSS_51632'] \ diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index 9df252fa..8dafe51c 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -2435,6 +2435,7 @@ class UpgradeImpl: # update catalog # start cluster in normal mode if self.isLargeInplaceUpgrade: + self.modifyPgProcIndex() self.touchRollbackCatalogFlag() self.updateCatalog() self.CopyCerts() -- Gitee From ba55d8079e2add052bfc14e0743b0986cb396536 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Mon, 18 Sep 2023 09:35:41 +0800 Subject: [PATCH 91/96] =?UTF-8?q?gs=5Fcheckos=E4=B8=AD=E7=9A=84A9=E9=80=89?= =?UTF-8?q?=E9=A1=B9=E7=BA=A7=E5=88=AB=E7=94=B1Abnormal=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=B8=BAwarning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_checkos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/gs_checkos b/script/gs_checkos index 85186372..fcebb87f 100644 --- a/script/gs_checkos +++ b/script/gs_checkos @@ -857,7 +857,7 @@ def checkBlockDevConfigure(): g_logger.debug("Checking Pre-read block size value.") performCheckorSetOS(ACTION_CHECK_BLOCKDEV_CONFIGURE, "blockdev", "The value about Pre-read block size is correct.", - "A9", "Abnormal") + "A9", "Warning") g_logger.debug("Successfully checked Pre-read block size value.") -- Gitee From 5e9acc777103116c83b3c2aa1f45d284e9dea0f3 Mon Sep 17 00:00:00 2001 From: zhang_xubo <2578876417@qq.com> Date: Mon, 18 Sep 2023 15:26:57 +0800 Subject: [PATCH 92/96] =?UTF-8?q?=E4=BF=AE=E5=A4=8D3.0.0-3.1.0-5.1.0?= =?UTF-8?q?=E8=B7=A8=E7=89=88=E6=9C=AC=E5=B0=B1=E5=9C=B0=E5=8D=87=E7=BA=A7?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/impl/upgrade/UpgradeImpl.py | 104 ++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 17 deletions(-) diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index 8dafe51c..c51ce024 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -2339,6 +2339,7 @@ class UpgradeImpl: # 4.record the old and new app dir in file self.recordDirFile() if self.isLargeInplaceUpgrade: + self.rebuild_pg_proc_index() self.recordLogicalClusterName() # 6. reload vacuum_defer_cleanup_age to new value if self.isLargeInplaceUpgrade: @@ -2435,7 +2436,6 @@ class UpgradeImpl: # update catalog # start cluster in normal mode if self.isLargeInplaceUpgrade: - self.modifyPgProcIndex() self.touchRollbackCatalogFlag() self.updateCatalog() self.CopyCerts() @@ -3878,6 +3878,72 @@ class UpgradeImpl: self.stopCluster() else: self.om_stop_cluster() + + def rebuild_pg_proc_index(self): + self.context.logger.debug("Begin to modify pg_proc index.") + self.setUpgradeMode(2) + database_list = self.getDatabaseList() + sql = """ +DO $$ +DECLARE +ans boolean; +BEGIN + alter index pg_proc_oid_index rebuild; + select case when count(*)=1 then true else false end as ans from (select indexname from pg_indexes where indexname = 'pg_proc_proname_args_nsp_index' limit 1) into ans; + if ans = true then + alter index pg_proc_proname_args_nsp_index rebuild; + end if; +END$$;""" + for eachdb in database_list: + self.context.logger.debug(f"rebuild_pg_proc_index at database {eachdb}") + (status, output) = ClusterCommand.remoteSQLCommand( + sql, self.context.user, + self.dnInst.hostname, self.dnInst.port, False, + eachdb, IsInplaceUpgrade=True) + if status != 0: + raise Exception(ErrorCode.GAUSS_513["GAUSS_51300"] % sql + + " Error: \n%s" % str(output)) + self.setUpgradeMode(0) + self.context.logger.debug("Success to modify pg_proc index.") + + def rebuild_sha2_func(self): + """ + sha2 function is added at version 3.1.0 with pg_catalog.sha2(text, int). + but at version 5.0.0 changed to pg_catalog.sha2(text, bigint). + we need to re create the function with new type while upgrade from + multi version such as (3.0.0 - 3.1.0 - 5.1.0) + """ + SHA2_CREATE_VERSION = 92776 + SHA2_UPDATE_VERSION = 92806 + old_version = int(float(self.context.oldClusterNumber) * 1000) + new_version = int(float(self.context.newClusterNumber) * 1000) + if old_version > SHA2_UPDATE_VERSION or \ + old_version < SHA2_CREATE_VERSION or \ + new_version < SHA2_UPDATE_VERSION: + return + + self.context.logger.debug("Need to re-create sha2 functoin") + sql = """ +BEGIN; +SET IsInplaceUpgrade = on; +DROP FUNCTION IF EXISTS pg_catalog.sha2(text, int) CASCADE; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 560; +CREATE OR REPLACE FUNCTION pg_catalog.sha2(text, bigint) + RETURNS text + LANGUAGE internal + IMMUTABLE NOT FENCED NOT SHIPPABLE +AS $function$sha2$function$; +comment on function PG_CATALOG.sha2(text,bigint) is 'use the sha2 algorithm to hash'; +END;""" + database_list = self.getDatabaseList() + for eachdb in database_list: + (status, output) = ClusterCommand.remoteSQLCommand( + sql, self.context.user, + self.dnInst.hostname, self.dnInst.port, False, + eachdb, IsInplaceUpgrade=True) + if status != 0: + self.context.logger.debug("re-create sha2 functoin failed. Error: %s" % str(output)) + self.context.logger.debug("Re-create sha2 functoin success.") def modifyPgProcIndex(self): """ @@ -3887,6 +3953,7 @@ class UpgradeImpl: 4. start cluster :return: """ + self.rebuild_sha2_func() self.context.logger.debug("Begin to modify pg_proc index.") time.sleep(3) database_list = self.getDatabaseList() @@ -3897,22 +3964,7 @@ class UpgradeImpl: true,0,0,0,2690;CREATE UNIQUE INDEX pg_proc_oid_index ON pg_proc USING btree (oid);SET LOCAL inplace_upgrade_next_system_object_oids=IUO_CATALOG,false, - true,0,0,0,0;commit;CHECKPOINT;""" - for eachdb in database_list: - (status, output) = ClusterCommand.remoteSQLCommand( - sql, self.context.user, - self.dnInst.hostname, self.dnInst.port, False, - eachdb, IsInplaceUpgrade=True) - if status != 0: - raise Exception(ErrorCode.GAUSS_513["GAUSS_51300"] % sql + - " Error: \n%s" % str(output)) - sql = """START TRANSACTION;SET IsInplaceUpgrade = on; - drop index if exists pg_proc_proname_args_nsp_index;SET LOCAL - inplace_upgrade_next_system_object_oids=IUO_CATALOG,false, - true,0,0,0,2691;create UNIQUE INDEX pg_proc_proname_args_nsp_index - ON pg_proc USING btree (proname, proargtypes, pronamespace);SET - LOCAL inplace_upgrade_next_system_object_oids=IUO_CATALOG,false, - true,0,0,0,0;commit;CHECKPOINT;""" + true,0,0,0,0;alter index pg_proc_oid_index rebuild;commit;CHECKPOINT;""" for eachdb in database_list: (status, output) = ClusterCommand.remoteSQLCommand( sql, self.context.user, @@ -3921,6 +3973,24 @@ class UpgradeImpl: if status != 0: raise Exception(ErrorCode.GAUSS_513["GAUSS_51300"] % sql + " Error: \n%s" % str(output)) + + NSP_INDEX_DEL_VERSION = 92848 + if int(float(self.context.newClusterNumber) * 1000) < NSP_INDEX_DEL_VERSION: + sql = """START TRANSACTION;SET IsInplaceUpgrade = on; + drop index if exists pg_proc_proname_args_nsp_index;SET LOCAL + inplace_upgrade_next_system_object_oids=IUO_CATALOG,false, + true,0,0,0,2691;create UNIQUE INDEX pg_proc_proname_args_nsp_index + ON pg_proc USING btree (proname, proargtypes, pronamespace);SET + LOCAL inplace_upgrade_next_system_object_oids=IUO_CATALOG,false, + true,0,0,0,0;commit;CHECKPOINT;""" + for eachdb in database_list: + (status, output) = ClusterCommand.remoteSQLCommand( + sql, self.context.user, + self.dnInst.hostname, self.dnInst.port, False, + eachdb, IsInplaceUpgrade=True) + if status != 0: + raise Exception(ErrorCode.GAUSS_513["GAUSS_51300"] % sql + + " Error: \n%s" % str(output)) # stop cluster self.stop_strategy() # start cluster -- Gitee From b3cb5dd13a5045c4e8d19e636583b0b9b362e3e0 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Tue, 19 Sep 2023 17:23:50 +0800 Subject: [PATCH 93/96] =?UTF-8?q?A13=E6=A3=80=E6=9F=A5=E9=A1=B9=E6=94=AF?= =?UTF-8?q?=E6=8C=81kylin=E6=93=8D=E4=BD=9C=E7=B3=BB=E7=BB=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/local/LocalCheckOS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/local/LocalCheckOS.py b/script/local/LocalCheckOS.py index 6f39fd6f..9077ce0a 100644 --- a/script/local/LocalCheckOS.py +++ b/script/local/LocalCheckOS.py @@ -1392,7 +1392,7 @@ def collectfirewall(): """ data = firewall() distname = LinuxDistro.linux_distribution()[0] - if distname in ("redhat", "centos", "euleros", "openEuler", "FusionOS"): + if distname in ("redhat", "centos", "euleros", "openEuler", "FusionOS", "kylin"): data.distname = distname.upper() if g_Platform.isPlatFormEulerOSOrRHEL7X(): cmd = "systemctl status firewalld.service" @@ -1407,7 +1407,7 @@ def collectfirewall(): data.errormsg = output return data - if distname in ("redhat", "centos", "euleros", "openEuler", "FusionOS"): + if distname in ("redhat", "centos", "euleros", "openEuler", "FusionOS", "kylin"): if g_Platform.isPlatFormEulerOSOrRHEL7X(): if (output.strip()).find("Active: " "active (running)") > 0: -- Gitee From 35786aa3b8bdc12732e7da546511e5b4f623ff32 Mon Sep 17 00:00:00 2001 From: liuheng Date: Tue, 26 Sep 2023 15:40:27 +0800 Subject: [PATCH 94/96] =?UTF-8?q?fix=20ssh=E4=BA=92=E4=BF=A1=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E6=B2=A1=E6=9C=89=E6=8D=95=E8=8E=B7=E5=88=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_sshexkey | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/gs_sshexkey b/script/gs_sshexkey index 9fc01623..97fe8f35 100644 --- a/script/gs_sshexkey +++ b/script/gs_sshexkey @@ -821,7 +821,7 @@ General options: cmd += '&& sed -i "$ s/$/ #OM/" %s ' % self.known_hosts_fname cmd += "&& chmod %s %s" % (DefaultValue.KEY_FILE_MODE, self.known_hosts_fname) (status, output) = subprocess.getstatusoutput(cmd) - if status != 0: + if status != 0 or "Name or service not known".lower() in output.lower(): raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd + " Error:\n%s" % output) (status, output) = self.checkAuthentication(self.localHost) if not status: -- Gitee From 3c246bdd1a5b4d21d203f30247be7b68fe630942 Mon Sep 17 00:00:00 2001 From: z00793368 Date: Tue, 26 Sep 2023 19:30:57 +0800 Subject: [PATCH 95/96] =?UTF-8?q?=E5=A2=9E=E5=8A=A0A5=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E9=A1=B9=E6=8A=A5=E9=94=99=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gspylib/common/ErrorCode.py | 3 ++- script/impl/preinstall/PreinstallImpl.py | 2 -- script/impl/uninstall/UninstallImpl.py | 2 -- script/local/LocalCheckOS.py | 7 +++++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index 82de327f..a4e4cc95 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -288,7 +288,8 @@ class ErrorCode(): "the actual memory.", 'GAUSS_50501': "[GAUSS-50501] : Shared_buffers must be less than " "shmmax. Please check it.", - 'GAUSS_50502': "[GAUSS-50502] : Failed to obtain %s information." + 'GAUSS_50502': "[GAUSS-50502] : Failed to obtain %s information.", + 'GAUSS_50503': "[GAUSS-50503] : %s, program exists abnormally." } ########################################################################### diff --git a/script/impl/preinstall/PreinstallImpl.py b/script/impl/preinstall/PreinstallImpl.py index 237af3c9..e1f0f622 100644 --- a/script/impl/preinstall/PreinstallImpl.py +++ b/script/impl/preinstall/PreinstallImpl.py @@ -1359,8 +1359,6 @@ class PreinstallImpl: if found_enabledss: with open(source_file, "w") as file: file.writelines(file_lines) - else: - return def checkRepeat(self): """ diff --git a/script/impl/uninstall/UninstallImpl.py b/script/impl/uninstall/UninstallImpl.py index 78bbf7b5..da6aeb18 100644 --- a/script/impl/uninstall/UninstallImpl.py +++ b/script/impl/uninstall/UninstallImpl.py @@ -455,8 +455,6 @@ class UninstallImpl: if self.mpprcFile and os.path.isfile(self.mpprcFile): source_file = self.mpprcFile - else: - source_file = os.path.join("/etc", "profile") if is_dsshome and not is_enabledssset: with open(source_file, 'a') as file: diff --git a/script/local/LocalCheckOS.py b/script/local/LocalCheckOS.py index 7fcc3e38..658d1d72 100644 --- a/script/local/LocalCheckOS.py +++ b/script/local/LocalCheckOS.py @@ -1995,8 +1995,11 @@ def CheckMemInfo(): input : NA output : NA """ - memdata = collectMemInfo() - swapdata = collectSwapInfo() + try: + memdata = collectMemInfo() + swapdata = collectSwapInfo() + except SystemExit as e: + raise Exception(ErrorCode.GAUSS_505["GAUSS_50503"] % e) if (swapdata.swapvalue > memdata.memvalue): g_logger.log("SwapMemory %s TotalMemory %s" % (swapdata.swapvalue, memdata.memvalue)) -- Gitee From f21c5387c408678139963f32a8866a963796191b Mon Sep 17 00:00:00 2001 From: liuheng Date: Mon, 16 Dec 2024 16:07:17 +0800 Subject: [PATCH 96/96] =?UTF-8?q?fix=20gs=5Fcheckos=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E6=B2=A1=E6=9C=89ntpd=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E4=B9=9F=E8=A6=81=E6=AF=94=E8=BE=83=E5=90=84=E4=B8=AA?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E7=9A=84=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/gs_checkos | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/script/gs_checkos b/script/gs_checkos index 5f952275..3e930d88 100644 --- a/script/gs_checkos +++ b/script/gs_checkos @@ -965,7 +965,7 @@ def checkTimeConsistency(): Local_CheckOs, ACTION_CHECK_TIME_CONSISTENCY, g_opts.localLog) (status, output, outputMap) = getCmdOutput(cmd) parRes = "" - detail_msg = "" + success_msg = " The system time is consistent." for node in list(status.keys()): outputMap[node] = outputMap[node].strip().split("\n")[0].strip() for node in list(status.keys()): @@ -982,11 +982,11 @@ def checkTimeConsistency(): % (node, outputMap[node].strip().split(',')[1].strip()) - if ((parRes == "") and (len(list(status.keys())) > 1)): + if len(list(status.keys())) > 1: nodeValue = list(status.keys())[0].strip() keystr = outputMap[nodeValue].strip().split(',')[1].strip() if (keystr == ""): - g_OSCheckOpts['A12'][2] = 'Warning' + g_OSCheckOpts['A12'][2] = 'Abnormal' parRes += " [%s]\n %s\n" % ( nodeValue, "Failed to obtain localtime information.") else: @@ -1004,24 +1004,19 @@ def checkTimeConsistency(): "%Y-%m-%d %H:%M:%S") if (tmpTime < startTime or tmpTime > endTime): g_OSCheckOpts['A12'][2] = 'Abnormal' - parRes += " [%s]\n " \ - "The current system time = (%s)\n"\ - % (node, tmpstr) - detail_msg += " [%s]\n Variable:" \ + parRes += " [%s]\n Variable:" \ "'current system time' RealValue:" \ "'%s' ExpectedValue:'%s' " \ "[Abnormal]\n"\ % (node, tmpstr, baseTime.strftime("%Y-%m-%d %H:%M:%S")) - if (g_OSCheckOpts['A12'][2] == 'Warning' or g_OSCheckOpts['A12'][2] == 'Abnormal'): - g_OSCheckOpts['A12'][3] = "\n%s" % parRes + if (g_OSCheckOpts['A12'][2] == 'Normal'): + g_OSCheckOpts['A12'][3] = "\n%s" % success_msg + g_OSCheckOpts['A12'][4] = "\n%s" % success_msg else: - nodeValue = list(status.keys())[0].strip() - keystr = outputMap[nodeValue].strip().split(',')[1].strip() - g_OSCheckOpts['A12'][3] = "\n The ntpd service is " \ - "started, local time is \"%s\"." % keystr - g_OSCheckOpts['A12'][4] = "\n%s" % detail_msg + g_OSCheckOpts['A12'][3] = "\n%s" % parRes + g_OSCheckOpts['A12'][4] = "\n%s" % parRes except Exception as e: g_logger.debug(str(e)) -- Gitee