From 5f78a9cf8689d4745c1d6cb60ca82993e3037ad5 Mon Sep 17 00:00:00 2001 From: liuheng Date: Tue, 7 Nov 2023 15:30:55 +0800 Subject: [PATCH] =?UTF-8?q?om=E5=8E=BB=E9=99=A4root=E6=9D=83=E9=99=90?= =?UTF-8?q?=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/base_utils/common/dialog.py | 260 +++++++++ script/base_utils/os/cmd_util.py | 67 +++ script/base_utils/os/cpu_util.py | 124 +++++ script/base_utils/os/disk_util.py | 49 ++ script/base_utils/os/dmidecode_util.py | 248 +++++++++ script/base_utils/os/env_util.py | 7 +- script/base_utils/os/memory_util.py | 84 +++ script/base_utils/os/sysctl_util.py | 55 ++ .../domain_utils/cluster_file/package_info.py | 6 +- .../domain_utils/cluster_file/profile_file.py | 9 +- .../domain_common/cluster_constants.py | 1 + script/gs_checkos | 68 ++- script/gs_expansion | 76 ++- script/gs_install | 5 + script/gs_perfconfig | 316 +++++++++++ script/gs_preinstall | 124 +++-- script/gs_sshexkey | 4 +- script/gspylib/common/Common.py | 9 +- script/gspylib/common/ErrorCode.py | 5 +- script/gspylib/common/GaussLog.py | 5 +- script/gspylib/common/OMCommand.py | 17 +- script/gspylib/common/ParallelBaseOM.py | 5 +- script/gspylib/common/ParameterParsecheck.py | 13 +- script/gspylib/threads/SshTool.py | 5 +- script/impl/dropnode/DropnodeImpl.py | 2 +- script/impl/expansion/ExpansionImpl.py | 51 +- .../impl/expansion/expansion_impl_with_cm.py | 5 +- script/impl/install/InstallImpl.py | 18 + script/impl/perf_config/__init__.py | 0 script/impl/perf_config/basic/__init__.py | 0 script/impl/perf_config/basic/anti.py | 131 +++++ script/impl/perf_config/basic/guc.py | 241 +++++++++ script/impl/perf_config/basic/probe.py | 100 ++++ script/impl/perf_config/basic/project.py | 498 ++++++++++++++++++ script/impl/perf_config/basic/tuner.py | 221 ++++++++ script/impl/perf_config/perf_probe.py | 45 ++ script/impl/perf_config/perf_tuner.py | 94 ++++ script/impl/perf_config/preset/__init__.py | 0 script/impl/perf_config/preset/default.json | 10 + .../perf_config/preset/kunpeng-4P-tpcc.json | 10 + script/impl/perf_config/preset/preset.py | 314 +++++++++++ script/impl/perf_config/probes/__init__.py | 0 script/impl/perf_config/probes/business.py | 348 ++++++++++++ script/impl/perf_config/probes/cpu.py | 44 ++ script/impl/perf_config/probes/db.py | 93 ++++ script/impl/perf_config/probes/disk.py | 128 +++++ script/impl/perf_config/probes/memory.py | 65 +++ script/impl/perf_config/probes/network.py | 130 +++++ script/impl/perf_config/probes/os.py | 78 +++ script/impl/perf_config/probes/user.py | 40 ++ script/impl/perf_config/scripts/__init__.py | 0 .../impl/perf_config/scripts/irq_operate.sh | 56 ++ .../impl/perf_config/scripts/isolated_xlog.sh | 23 + script/impl/perf_config/test/__init__.py | 0 script/impl/perf_config/test/data/__init__.py | 0 .../test/data/test-preset-wrong-format.json | 10 + .../test/data/test-preset-wrong-format2.conf | 10 + .../test/data/test-preset-wrong-param.json | 10 + .../test/data/test-preset-wrong-val.json | 10 + .../test/data/test-preset-wrong-val2.json | 10 + .../test/data/test-preset-wrong-val3.json | 10 + .../impl/perf_config/test/test_always_ok.sh | 33 ++ script/impl/perf_config/test/test_common.sh | 24 + script/impl/perf_config/test/test_preset.sh | 56 ++ script/impl/perf_config/test/test_recover.sh | 23 + script/impl/perf_config/test/test_tune.sh | 55 ++ .../perf_config/test/test_tune_with_err.sh | 10 + script/impl/perf_config/tuners/__init__.py | 0 script/impl/perf_config/tuners/guc.py | 134 +++++ .../impl/perf_config/tuners/gucs/__init__.py | 0 script/impl/perf_config/tuners/gucs/common.py | 277 ++++++++++ .../perf_config/tuners/gucs/connection.py | 67 +++ .../impl/perf_config/tuners/gucs/execute.py | 88 ++++ .../perf_config/tuners/gucs/ha_cluster.py | 109 ++++ script/impl/perf_config/tuners/gucs/ops.py | 206 ++++++++ .../impl/perf_config/tuners/gucs/optimizer.py | 238 +++++++++ script/impl/perf_config/tuners/gucs/other.py | 229 ++++++++ .../impl/perf_config/tuners/gucs/security.py | 124 +++++ .../impl/perf_config/tuners/gucs/storage.py | 288 ++++++++++ script/impl/perf_config/tuners/os.py | 287 ++++++++++ script/impl/perf_config/tuners/setup.py | 50 ++ .../preinstall/OLAP/PreinstallImplOLAP.py | 25 +- script/impl/preinstall/PreinstallImpl.py | 102 +++- script/impl/upgrade/UpgradeImpl.py | 8 +- script/local/PreInstallUtility.py | 132 +++-- 85 files changed, 6731 insertions(+), 201 deletions(-) create mode 100644 script/base_utils/common/dialog.py create mode 100644 script/base_utils/os/dmidecode_util.py create mode 100644 script/base_utils/os/sysctl_util.py create mode 100644 script/gs_perfconfig create mode 100644 script/impl/perf_config/__init__.py create mode 100644 script/impl/perf_config/basic/__init__.py create mode 100644 script/impl/perf_config/basic/anti.py create mode 100644 script/impl/perf_config/basic/guc.py create mode 100644 script/impl/perf_config/basic/probe.py create mode 100644 script/impl/perf_config/basic/project.py create mode 100644 script/impl/perf_config/basic/tuner.py create mode 100644 script/impl/perf_config/perf_probe.py create mode 100644 script/impl/perf_config/perf_tuner.py create mode 100644 script/impl/perf_config/preset/__init__.py create mode 100644 script/impl/perf_config/preset/default.json create mode 100644 script/impl/perf_config/preset/kunpeng-4P-tpcc.json create mode 100644 script/impl/perf_config/preset/preset.py create mode 100644 script/impl/perf_config/probes/__init__.py create mode 100644 script/impl/perf_config/probes/business.py create mode 100644 script/impl/perf_config/probes/cpu.py create mode 100644 script/impl/perf_config/probes/db.py create mode 100644 script/impl/perf_config/probes/disk.py create mode 100644 script/impl/perf_config/probes/memory.py create mode 100644 script/impl/perf_config/probes/network.py create mode 100644 script/impl/perf_config/probes/os.py create mode 100644 script/impl/perf_config/probes/user.py create mode 100644 script/impl/perf_config/scripts/__init__.py create mode 100644 script/impl/perf_config/scripts/irq_operate.sh create mode 100644 script/impl/perf_config/scripts/isolated_xlog.sh create mode 100644 script/impl/perf_config/test/__init__.py create mode 100644 script/impl/perf_config/test/data/__init__.py create mode 100644 script/impl/perf_config/test/data/test-preset-wrong-format.json create mode 100644 script/impl/perf_config/test/data/test-preset-wrong-format2.conf create mode 100644 script/impl/perf_config/test/data/test-preset-wrong-param.json create mode 100644 script/impl/perf_config/test/data/test-preset-wrong-val.json create mode 100644 script/impl/perf_config/test/data/test-preset-wrong-val2.json create mode 100644 script/impl/perf_config/test/data/test-preset-wrong-val3.json create mode 100644 script/impl/perf_config/test/test_always_ok.sh create mode 100644 script/impl/perf_config/test/test_common.sh create mode 100644 script/impl/perf_config/test/test_preset.sh create mode 100644 script/impl/perf_config/test/test_recover.sh create mode 100644 script/impl/perf_config/test/test_tune.sh create mode 100644 script/impl/perf_config/test/test_tune_with_err.sh create mode 100644 script/impl/perf_config/tuners/__init__.py create mode 100644 script/impl/perf_config/tuners/guc.py create mode 100644 script/impl/perf_config/tuners/gucs/__init__.py create mode 100644 script/impl/perf_config/tuners/gucs/common.py create mode 100644 script/impl/perf_config/tuners/gucs/connection.py create mode 100644 script/impl/perf_config/tuners/gucs/execute.py create mode 100644 script/impl/perf_config/tuners/gucs/ha_cluster.py create mode 100644 script/impl/perf_config/tuners/gucs/ops.py create mode 100644 script/impl/perf_config/tuners/gucs/optimizer.py create mode 100644 script/impl/perf_config/tuners/gucs/other.py create mode 100644 script/impl/perf_config/tuners/gucs/security.py create mode 100644 script/impl/perf_config/tuners/gucs/storage.py create mode 100644 script/impl/perf_config/tuners/os.py create mode 100644 script/impl/perf_config/tuners/setup.py diff --git a/script/base_utils/common/dialog.py b/script/base_utils/common/dialog.py new file mode 100644 index 00000000..cc588d77 --- /dev/null +++ b/script/base_utils/common/dialog.py @@ -0,0 +1,260 @@ +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : Provides a series of methods for dialogue, question and input. +############################################################################# + +import os +from gspylib.common.GaussLog import GaussLog + + +class DialogError(Exception): + def __init__(self, complete_question, msg): + self.complete_question = complete_question + self.msg = msg + + def __str__(self): + return self.msg + + +class DialogUtil(object): + """ + Provides a set of interfaces for dialogue and interactive questioning. + + Each interface has its own label. + + If it is a question, you can select whether it is required, and the + required question will be marked with an '*'. + """ + @staticmethod + def Message(message): + """ + :param message: message + :return: NA + """ + complete_message = f'[ Message ] {message}\n' + GaussLog.printMessage(complete_message) + + @staticmethod + def yesOrNot(question, required=True, max_retry=99999): + """ + raise a question, and want a yes or not. + + example: + *[ yesOrNot ] Am I handsome? (y/n)? y + + :param question: question + :param required: must input something. if not, we can return None + :param max_retry: max retry times + :return: True, False, None + """ + question_tag = '{0}[ yesOrNot ]'.format('*' if required else '') + complete_question = '{0} {1}(y/n)? '.format( + question_tag, question + ) + answer = input(complete_question) + while True: + if not required and answer == '': + return + + if answer.lower() in ('y', 'n'): + return answer.lower() == 'y' + + max_retry -= 1 + if max_retry < 0: + raise DialogError(complete_question, 'Too many failed retry.') + + answer = input('Please input y or n: ') + + @staticmethod + def singleAnswerQuestion(question, options, required=True, max_retry=99999): + """ + raise a question, and want an option. + + example: + *[ Single Answer Question ] What would you like for dinner? + A. carrot + B. rice + C. chicken + Single answer: a + + :param question: question + :param options: options to choose. + :param required: must input something. if not, we can return None + :param max_retry: max retry times + :return: the index of option which user choose, or None + """ + assert len(options) <= 26 + question_tag = '{0}[ Single Answer Question ]'.format('*' if required else '') + code_id = [chr(c) for c in range(ord('A'), ord('A') + len(options))] + question_options = '' + for i, option in enumerate(options): + question_options += ' {0}. {1}\n'.format(code_id[i], option) + + complete_question = "{0} {1}\n{2}Single answer: ".format( + question_tag, question, question_options) + answer = input(complete_question) + while True: + if not required and answer == '': + return + if answer.upper() in code_id: + return code_id.index(answer.upper()) + + max_retry -= 1 + if max_retry < 0: + raise DialogError(complete_question, 'Too many failed retry.') + + answer = input('Please input A~{}:'.format(code_id[-1])) + + @staticmethod + def multipleAnswerQuestion(question, options, required=True, max_retry=99999): + """ + raise a question, and want an option. + + example: + *[ Multiple Answer Question ] What would you like for dinner? + A. carrot + B. rice + C. chicken + Multiple answer: ac + + :param question: question + :param options: options to choose + :param required: must input something. if not, we can return None + :param max_retry: max retry times + :return: the index list of option which user choose, or [] + """ + assert len(options) <= 26 + code_id = [chr(c) for c in range(ord('A'), ord('A') + len(options))] + question_options = '' + for i, option in enumerate(options): + question_options += ' {0}. {1} \n'.format(code_id[i], option) + complete_question = "{0}[ Multiple Answer Question ] {1}\n{2}Multiple answer: ".format( + '*' if required else '', question, question_options) + + def _analyse_answer(_answer): + _chosen = set() + for c in _answer: + if c == ' ' or c == '\t' or c == ',': + continue + if c.upper() not in code_id: + GaussLog.printMessage(f'Invalid select code: {c}.', end='') + return + _chosen.add(ord(c.upper()) - ord('A')) + _res = list(_chosen) + _res.sort() + if required and len(_res) == 0: + GaussLog.printMessage('Please make at least one choice.') + return + + return _res + + answer = input(complete_question) + while True: + res = _analyse_answer(answer) + if res is not None: + return res + + max_retry -= 1 + if max_retry < 0: + raise DialogError(complete_question, 'Too many failed retry.') + + answer = input('Please input A~{}:'.format(code_id[-1])) + + @staticmethod + def askANumber(question, check_func=None, required=True, max_retry=99999): + """ + ask a number. + + for example: + *[ Input number ] How old are you? + Please answer: 1000 + + :param question: question + :param check_func: Verify that the number entered is legitimate. Otherwise, return an error msg + :param required: must input something. if not, we can return None + :param max_retry: max retry times + :return: the number, or None + """ + complete_question = '{0}[ Input number ] {1}\nPlease answer: '.format( + '*' if required else '', question + ) + answer = input(complete_question) + while True: + retry_msg = None + if not required and answer == '': + return + + if not answer.isdigit(): + retry_msg = 'Invalid integer, please again: ' + elif check_func is not None: + errmsg = check_func(int(answer)) + if errmsg is None: + return int(answer) + retry_msg = f'{errmsg}. please again: ' + else: + return int(answer) + + max_retry -= 1 + if max_retry < 0: + raise DialogError(complete_question, 'Too many failed retry.') + + answer = input(retry_msg) + + @staticmethod + def askAPath(question, check_access=None, required=True, max_retry=99999): + """ + ask a path. + + for example: + *[ Input path ] Where is the file? + Please input the path: /home/user/abc + + :param question: question + :param check_access: check the path is access. + :param required: must input something. if not, we can return None + :param max_retry: max retry times + :return: the path, or None + """ + complete_question = '{0}[ Input path ] {1} \nPlease input the path: '.format( + '*' if required else '', question + ) + answer = input(complete_question) + while True: + retry_msg = None + if not required and answer == '': + return + + if check_access and not os.access(answer, os.F_OK): + retry_msg = 'Could not access path, please again: ' + else: + return answer + + max_retry -= 1 + if max_retry < 0: + raise DialogError(complete_question, 'Too many failed retry.') + + answer = input(retry_msg) + + +if __name__ == '__main__': + res = DialogUtil.multipleAnswerQuestion( + 'test question', + ['x', 'xx', 'xxx'] + ) + GaussLog.printMessage(res) + diff --git a/script/base_utils/os/cmd_util.py b/script/base_utils/os/cmd_util.py index 7e1d8815..e4bb836d 100644 --- a/script/base_utils/os/cmd_util.py +++ b/script/base_utils/os/cmd_util.py @@ -46,6 +46,20 @@ class CmdUtil(object): "if [ $MPPDB_ENV_SEPARATE_PATH ]; " \ "then source $MPPDB_ENV_SEPARATE_PATH; fi" + @staticmethod + def execCmd(cmd, noexcept=False): + """ + function: execute cmd + input: cmd, noexcept + output: output of cmd + """ + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + if noexcept: + return output + raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd + " Error: \n%s" % str(output)) + return output + @staticmethod def findCmdInPath(cmd, additional_paths=None, print_error=True): """ @@ -388,6 +402,15 @@ class CmdUtil(object): return CmdUtil.findCmdInPath('systemctl') + BLANK_SPACE + action + \ BLANK_SPACE + service_name + @staticmethod + def getSysctlCmd(): + """ + function: get sysctl cmd + input : NA + output : str + """ + return CmdUtil.findCmdInPath('sysctl') + @staticmethod def getUlimitCmd(): """ @@ -433,6 +456,33 @@ class CmdUtil(object): """ return CmdUtil.findCmdInPath('tail') + @staticmethod + def getWhichCmd(): + """ + function: get which cmd + input : NA + output : str + """ + return CmdUtil.findCmdInPath('which') + + @staticmethod + def getLscpuCmd(): + """ + function: get lscpu cmd + input : NA + output : str + """ + return CmdUtil.findCmdInPath('lscpu') + + @staticmethod + def getDmidecodeCmd(): + """ + function: get dmidecode cmd + input : NA + output : str + """ + return CmdUtil.findCmdInPath('dmidecode') + @staticmethod def getSshCmd(address, timeout=None): """ @@ -678,3 +728,20 @@ class CmdUtil(object): proc.stdin.flush() output, error = proc.communicate() return proc.returncode, output, error + + @staticmethod + def doesBinExist(bin): + """ + function : which bin + input : bin name + output: bool + """ + cmd = CmdUtil.getWhichCmd() + BLANK_SPACE + bin + try: + status, output = subprocess.getstatusoutput(cmd) + if status == 0: + return True + except Exception: + pass + + return False \ No newline at end of file diff --git a/script/base_utils/os/cpu_util.py b/script/base_utils/os/cpu_util.py index 3a6b08d4..a70ca613 100644 --- a/script/base_utils/os/cpu_util.py +++ b/script/base_utils/os/cpu_util.py @@ -23,8 +23,11 @@ try: import subprocess import sys import multiprocessing + from enum import Enum sys.path.append(sys.path[0] + "/../../") from base_utils.common.constantsbase import ConstantsBase + from base_utils.os.cmd_util import CmdUtil + from gspylib.common.ErrorCode import ErrorCode except ImportError as e: sys.exit("[GAUSS-52200] : Unable to import module: %s." % str(e)) @@ -36,6 +39,21 @@ Requirements: """ +class CpuArchitecture(Enum): + UNKNOWN = 0 + AARCH64 = 1 + X86_64 = 2 + + @staticmethod + def parse(arch): + if arch == CpuArchitecture.AARCH64.name.lower(): + return CpuArchitecture.AARCH64 + elif arch == CpuArchitecture.X86_64.name.lower(): + return CpuArchitecture.X86_64 + else: + return CpuArchitecture.UNKNOWN + + class CpuUtil(object): """ function: Init the CpuInfo options @@ -91,3 +109,109 @@ class CpuUtil(object): if cpu_set > 1: return cpu_set return ConstantsBase.DEFAULT_PARALLEL_NUM + + @staticmethod + def getCpuArchitecture(): + """ + function: get cpu architecture of current board + lscpu | grep Architecture + input: NA + output: cpu architecture + """ + cmd = f"{CmdUtil.getLscpuCmd()} | {CmdUtil.getGrepCmd} Architecture" + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + return CpuArchitecture.UNKNOWN + arch = output.split(':')[1].strip() + return CpuArchitecture.parse(arch) + + @staticmethod + def getCpuModelName(): + """ + function: get cpu mode name of current board + lscpu | grep Architecture + input: NA + output: cpu Model name + """ + cmd = f"{CmdUtil.getLscpuCmd()} | {CmdUtil.getGrepCmd} 'Model name'" + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + return '' + mode_name = output.split(':')[1].strip() + return mode_name + + @staticmethod + def getCpuVendor(): + """ + function: get cpu vendor of current board + lscpu | grep 'Vendor ID' + input: NA + output: cpu vendor + """ + cmd = f"{CmdUtil.getLscpuCmd()} | {CmdUtil.getGrepCmd} 'Vendor ID'" + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + return '' + vendor = output.split(':')[1].strip() + return vendor + + @staticmethod + def cpuListToCpuRangeStr(cpu_list): + """ + function: transform cpu id list to cpu range str + input: cpu id list, like (0,1,2,3,4,5,6,11,12,13,15,16,20) + output: cpu range str, like '0-6,11-16,20' + """ + if len(cpu_list) == 0: + return '' + + start = cpu_list[0] + pre = cpu_list[0] + cpu_range_str = [] + for cpuid in cpu_list: + if int(cpuid) > int(pre) + 1: + this_range = str(pre) if pre == start else '{0}-{1}'.format(start, pre) + cpu_range_str.append(this_range) + start = pre = cpuid + continue + pre = cpuid + last_range = str(pre) if pre == start else '{0}-{1}'.format(start, pre) + cpu_range_str.append(last_range) + return ','.join(cpu_range_str) + + @staticmethod + def cpuRangeStrToCpuList(cpu_range_str): + """ + function: transform cpu range str to cpu id list + input: cpu range str, like '0-6,11-16,20' + output: cpu id list, like (0,1,2,3,4,5,6,11,12,13,15,16,20) + """ + cpu_list = [] + for part_range in cpu_range_str.split(','): + p = part_range.split('-') + if len(p) == 1: + cpu_list.append(int(p[0])) + continue + cpu_list += [i for i in range(int(p[0]), int(p[1]) + 1)] + + return tuple(cpu_list) + + @staticmethod + def getCpuNumaList(): + """ + function: get cpu numa list of current board + lscpu | grep 'NUMA node' | grep 'CPU(s)' + input: NA + output: [[cpu id of numa 0], [cpu id of numa 1], ...] + """ + cmd = f"{CmdUtil.getLscpuCmd()} | {CmdUtil.getGrepCmd} 'NUMA node' | {CmdUtil.getGrepCmd} 'CPU(s)'" + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd + + " Error: \n%s" % str(output)) + range_info = [line.split(':')[1].strip() for line in output.split('\n')] + numa_list = [] + for p in range_info: + numa_list.append(CpuUtil.cpuRangeStrToCpuList(p)) + + return numa_list diff --git a/script/base_utils/os/disk_util.py b/script/base_utils/os/disk_util.py index 722588df..b42f8f5a 100644 --- a/script/base_utils/os/disk_util.py +++ b/script/base_utils/os/disk_util.py @@ -24,11 +24,24 @@ import subprocess import sys import psutil import math +from enum import Enum sys.path.append(sys.path[0] + "/../../") from base_utils.os.cmd_util import CmdUtil from gspylib.common.ErrorCode import ErrorCode +class FSType(Enum): + EXT4 = 0 + XFS = 1 + NTFS = 2 + + +class DiskType(Enum): + MECHANICAL = 0 + SSD = 1 + NVME = 2 + + class DiskUtil(object): """ function: Init the DiskUsage options @@ -44,6 +57,42 @@ class DiskUtil(object): """ return psutil.disk_partitions(all_info) + @staticmethod + def getTotalSize(device): + """ + get device total size. Unit is MB + :param device: device path or mount point + :return: total size. Unit is MB + """ + cmd = "%s -m | %s \"%s\" | %s '{print $2}'" % ( + CmdUtil.findCmdInPath('df'), CmdUtil.getGrepCmd(), device, CmdUtil.getAwkCmd() + ) + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd + + " Error: \n%s" % str(output)) + return int(output) + + @staticmethod + def getAvailSize(device): + """ + get device available size. Unit is MB + :param device: device path or mount point + :return: available size. Unit is MB + """ + cmd = "%s -m | %s \"%s\" | %s '{print $4}'" % ( + CmdUtil.findCmdInPath('df'), + CmdUtil.getGrepCmd(), + device, + CmdUtil.getAwkCmd() + ) + (status, output) = subprocess.getstatusoutput(cmd) + if status == 0: + return int(output) + else: + raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd + + " Error: \n%s" % str(output)) + @staticmethod def getUsageSize(directory): """ diff --git a/script/base_utils/os/dmidecode_util.py b/script/base_utils/os/dmidecode_util.py new file mode 100644 index 00000000..224ea01a --- /dev/null +++ b/script/base_utils/os/dmidecode_util.py @@ -0,0 +1,248 @@ +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : Provides a series of methods for parsing 'dmidecode'. +############################################################################# + + +import subprocess +from enum import Enum +from base_utils.os.cmd_util import CmdUtil +from gspylib.common.ErrorCode import ErrorCode + + +class DMIType(Enum): + BIOS = 0 + SYSTEM = 1 + BASEBOARD = 2 + CHASSIS = 3 + PROCESSOR = 4 + MEMORY_CONTROLLER = 5 + MEMORY_MODULE = 6 + CACHE = 7 + PORT_CONNECTOR = 8 + SYSTEM_SLOTS = 9 + ON_BOARD_DEVICES = 10 + OEM_STRINGS = 11 + SYSTEM_CONFIGURATION_OPTIONS = 12 + BIOS_LANGUAGE = 13 + GROUP_ASSOCIATIONS = 14 + SYSTEM_EVENT_LOG = 15 + PHYSICAL_MEMORY_ARRAY = 16 + MEMORY_DEVICE = 17 + BIT_MEMORY_ERROR_32_BIT = 18 + MEMORY_ARRAY_MAPPED_ADDRESS = 19 + MEMORY_DEVICE_MAPPED_ADDRESS = 20 + BUILT_IN_POINTING_DEVICE = 21 + PORTABLE_BATTERY = 22 + SYSTEM_RESET = 23 + HARDWARE_SECURITY = 24 + SYSTEM_POWER_CONTROLS = 25 + VOLTAGE_PROBE = 26 + COOLING_DEVICE = 27 + TEMPERATURE_PROBE = 28 + ELECTRICAL_CURRENT_PROBE = 29 + OUT_OF_BAND_REMOTE_ACCESS = 30 + BOOT_INTEGRITY_SERVICES = 31 + SYSTEM_BOOT = 32 + MEMORY_ERROR_64_BIT = 33 + MANAGEMENT_DEVICE = 34 + MANAGEMENT_DEVICE_COMPONENT = 35 + MANAGEMENT_DEVICE_THRESHOLD_DATA = 36 + MEMORY_CHANNEL = 37 + IPMI_DEVICE = 38 + POWER_SUPPLY = 39 + ADDITIONAL_INFORMATION = 40 + ONBOARD_DEVICES_EXTENDED_INFORMATION = 41 + MANAGEMENT_CONTROLLER_HOST_INTERFACE = 42 + + +class DMITypeCategory(Enum): + BIOS = 'bios' + SYSTEM = 'system' + BASEBOARD = 'baseboard' + CHASSIS = 'chassis' + PROCESSOR = 'processor' + MEMORY = 'memory' + CACHE = 'cache' + CONNECTOR = 'connector' + SLOT = 'slot' + + +class DMIDevice(object): + def __init__(self, src): + self.src = src + self.handle = 'Unknown' + self.dmi_type = 'Unknown' + self.name = 'Unknown' + self.title = 'Unknown' + self.attrs = {} + self._key = None + + self._parse() + + def __str__(self): + return self.src + + def __getitem__(self, item): + return self.attrs.get(item) + + def _parse(self): + def _get_level(_line): + if _line.strip() == '': + return -1 + _level = 0 + for i in range(0, len(_line)): + if _line[i] == '\t': + _level += 1 + else: + break + return _level + + lines = self.src.split('\n') + for line in lines: + level = _get_level(line) + if level == -1: + continue + elif level == 1: + self._parseAttr(line) + elif level == 2: + self._appendAttrVal(line) + elif line.startswith('Handle'): + self._parseHandle(line) + elif line.find('Information'): + self._parseTitle(line) + elif line.find('Table at') or line.find('End Of Table'): + pass + else: + print('ERROR, UNKNOWN DMIDECODE INFO:', line) + assert False + + def _parseHandle(self, line): + # Handle 0x0401, DMI type 4, 48 bytes + assert line.startswith('Handle') + parts = line[:-1].split(',') + self.handle = parts[0].split(' ')[1] + self.dmi_type = parts[1].split(' ')[2] + + def _parseTitle(self, line): + self.title = line + self.name = line[:-11] + + def _parseAttr(self, line): + parts = line.split(':') + key = parts[0].strip() + value = ':'.join(parts[1:]).strip() + self.attrs[key] = value + self._key = key + + def _appendAttrVal(self, line): + if not isinstance(self.attrs[self._key], list): + self.attrs[self._key] = [] + self.attrs[self._key].append(line.strip()) + + +class DmiDecodeTable(object): + def __init__(self, src): + self.src = src + self.version = None + self.devices = [] + self._iter = 0 + + self._parse() + + def __iter__(self): + self._idx = 0 + return self + + def __next__(self): + if self._iter >= len(self.devices): + self._iter = 0 + raise StopIteration + res = self.devices[self._iter] + self._iter += 1 + return res + + def _parse(self): + lines = self.src.split('\n') + + is_first_part = True + tmp = '' + for line in lines: + if line.strip() == '': + if is_first_part: + is_first_part = False + pass # ignore parse version + else: + self.devices.append(DMIDevice(tmp)) + tmp = '' + continue + tmp += (line + '\n') + + def __str__(self): + return self.src + + +class DmidecodeUtil(object): + + @staticmethod + def getDmidecodeTableByType(dmitype=None): + """ + execute dmidecode [-t type] and get the parsed result: DmiDecodeTable. + :param dmitype: dmi type. None | int | str | DMIType | DMITypeCategory + :return: DmiDecodeTable + """ + if dmitype is None: + cmd = CmdUtil.getDmidecodeCmd() + elif isinstance(dmitype, DMIType) or isinstance(dmitype, DMITypeCategory): + cmd = f'{CmdUtil.getDmidecodeCmd()} -t {dmitype.value} ' + elif isinstance(dmitype, int) or isinstance(dmitype, str): + cmd = f'{CmdUtil.getDmidecodeCmd()} -t {dmitype} ' + else: + assert False + + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd + " Error: \n%s" % str(output)) + + return DmiDecodeTable(output) + + @staticmethod + def getDmidecodeTable(): + """ + execute dmidecode and get the parsed result: DmiDecodeTable. + :return: DmiDecodeTable + """ + return DmidecodeUtil.getDmidecodeTableByType() + + @staticmethod + def getDmidecodeVersion(): + """ + execute dmidecode --version and get the result. + :return: dmidecode --version + """ + cmd = f'{CmdUtil.getDmidecodeCmd()} --version' + status, output = subprocess.getstatusoutput(cmd) + if status != 0: + raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd + " Error: \n%s" % str(output)) + return output + + +if __name__ == '__main__': + dmidecode = DmidecodeUtil.getDmidecodeTable() + for item in dmidecode: + print(item) diff --git a/script/base_utils/os/env_util.py b/script/base_utils/os/env_util.py index e54ed657..0e5ec637 100644 --- a/script/base_utils/os/env_util.py +++ b/script/base_utils/os/env_util.py @@ -82,14 +82,17 @@ class EnvUtil(object): return EnvUtil.getEnvironmentParameterValue("PGHOST", user) @staticmethod - def getTempDir(dir_name): + def getTempDir(dir_name, user = ""): """ function: create temp directory in PGHOST input: dir_name output: pathName """ - tmp_path = EnvUtil.getTmpDirFromEnv() + if user == "": + tmp_path = EnvUtil.getTmpDirFromEnv() + else: + tmp_path = EnvUtil.getTmpDirFromEnv(user) return os.path.join(tmp_path, dir_name) @staticmethod diff --git a/script/base_utils/os/memory_util.py b/script/base_utils/os/memory_util.py index 6a71d1f1..2aeae3d1 100644 --- a/script/base_utils/os/memory_util.py +++ b/script/base_utils/os/memory_util.py @@ -21,6 +21,7 @@ try: import sys import psutil sys.path.append(sys.path[0] + "/../../") + from base_utils.os.cmd_util import CmdUtil from gspylib.common.ErrorCode import ErrorCode except ImportError as e: sys.exit("[GAUSS-52200] : Unable to import module: %s." % str(e)) @@ -31,6 +32,8 @@ class MemoryUtil(object): function: Init the MemInfo options """ + MEM_INFO_FILE = '/proc/meminfo' + @staticmethod def getMemTotalSize(): """ @@ -43,3 +46,84 @@ class MemoryUtil(object): except Exception as excep: raise Exception(ErrorCode.GAUSS_505["GAUSS_50502"] % "system memory usage" + "Error: %s" % str(excep)) + + @staticmethod + def getMemAvailableSize(): + """ + function : Get system virtual memory available size + input : null + output : available virtual memory(byte) + """ + try: + return psutil.virtual_memory().available + except Exception as excep: + raise Exception(ErrorCode.GAUSS_505["GAUSS_50502"] % + "system memory usage" + "Error: %s" % str(excep)) + + @staticmethod + def getMemUsage(): + """ + function : Get system virtual memory usage + input : null + output : memory usage + """ + try: + return psutil.virtual_memory().percent + except Exception as excep: + raise Exception(ErrorCode.GAUSS_505["GAUSS_50502"] % + "system memory usage" + "Error: %s" % str(excep)) + + @staticmethod + def selectMemInfo(attr=None): + """ + + :param attr: + :return: + """ + if attr is None: + output = CmdUtil.execCmd(f'{CmdUtil.getCatCmd()} {MemoryUtil.MEM_INFO_FILE}') + res = {} + for line in output.split('\n'): + kv = line.split(':') + res[kv[0].strip()] = kv[1].strip() + return res + + cmd = "%s %s | %s '%s' | %s '{print $2}'" % ( + CmdUtil.getCatCmd(), + MemoryUtil.MEM_INFO_FILE, + CmdUtil.getGrepCmd(), + attr, + CmdUtil.getAwkCmd() + ) + output = CmdUtil.execCmd(cmd) + return output + + @staticmethod + def getPhysicalMemTotalSize(): + """ + function : Get system physical memory total size + input : null + output : total physical memory(byte) + """ + cmd = "%s | %s Mem | %s '{print $2}'" % ( + CmdUtil.findCmdInPath('free'), + CmdUtil.getGrepCmd(), + CmdUtil.getAwkCmd() + ) + output = CmdUtil.execCmd(cmd) + return int(output) + + @staticmethod + def getPhysicalMemUsedSize(): + """ + function : Get system physical memory total size + input : null + output : total physical memory(byte) + """ + cmd = "%s | %s Mem | %s '{print $3}'" % ( + CmdUtil.findCmdInPath('free'), + CmdUtil.getGrepCmd(), + CmdUtil.getAwkCmd() + ) + output = CmdUtil.execCmd(cmd) + return int(output) diff --git a/script/base_utils/os/sysctl_util.py b/script/base_utils/os/sysctl_util.py new file mode 100644 index 00000000..471e7a91 --- /dev/null +++ b/script/base_utils/os/sysctl_util.py @@ -0,0 +1,55 @@ +# -*- coding:utf-8 -*- +############################################################################# +# Portions Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : sshd config file operation. +############################################################################# + + +try: + import sys + sys.path.append(sys.path[0] + "/../../") + from base_utils.os.cmd_util import CmdUtil + from gspylib.common.ErrorCode import ErrorCode +except ImportError as e: + sys.exit("[GAUSS-52200] : Unable to import module: %s." % str(e)) + + +class SysctlUtil(object): + + @staticmethod + def getAll(): + cmd = f'{CmdUtil.getSysctlCmd()} -a' + output = CmdUtil.execCmd(cmd, noexcept=False) + res = {} + for line in output.split('\n'): + kv = line.split(' = ') + if len(kv) == 1: + continue + res[kv[0]] = ' = '.join(kv[1:]) + return res + + @staticmethod + def get(name): + output = CmdUtil.execCmd(f'{CmdUtil.getSysctlCmd()} {name}', noexcept=False) + kv = output.split(' = ') + if len(kv) == 1: + return + return ' = '.join(kv[1:]) + + @staticmethod + def set(name, value): + CmdUtil.execCmd(f'{CmdUtil.getSysctlCmd()} {name} {value}', noexcept=False) diff --git a/script/domain_utils/cluster_file/package_info.py b/script/domain_utils/cluster_file/package_info.py index 68644f5f..62a1db9b 100644 --- a/script/domain_utils/cluster_file/package_info.py +++ b/script/domain_utils/cluster_file/package_info.py @@ -157,6 +157,8 @@ class PackageInfo(object): integrity_file_name = PackageInfo.getSHA256FilePath() cm_package = "%s-cm.tar.gz" % PackageInfo.getPackageFile( "bz2File").replace(".tar.bz2", "") + om_package = "%s-om.tar.gz" % PackageInfo.getPackageFile( + "bz2File").replace(".tar.bz2", "") tar_lists = SingleInstDiff.get_package_tar_lists(is_single_inst, os.path.normpath(package_path)) @@ -171,8 +173,8 @@ class PackageInfo(object): # do not tar *.log files cmd += CompressUtil.getCompressFilesCmd(PackageInfo.get_package_back_name(), tar_lists) - cmd += " %s %s " % (os.path.basename(bz2_file_name), - os.path.basename(integrity_file_name)) + cmd += " %s %s %s " % (os.path.basename(bz2_file_name), + os.path.basename(integrity_file_name), os.path.basename(om_package)) # add CM package to bak package if os.path.isfile(os.path.realpath(os.path.join(package_path, cm_package))): cmd += "%s " % os.path.basename(cm_package) diff --git a/script/domain_utils/cluster_file/profile_file.py b/script/domain_utils/cluster_file/profile_file.py index a44929d4..c995ad91 100644 --- a/script/domain_utils/cluster_file/profile_file.py +++ b/script/domain_utils/cluster_file/profile_file.py @@ -167,7 +167,7 @@ class ProfileFile: if not status: return False, output - if user and os.getuid() == 0: + if user: execute_cmd = "%s '%s' && %s '%s'" % (CmdUtil.SOURCE_CMD, ClusterConstants.ETC_PROFILE, CmdUtil.SOURCE_CMD, @@ -208,9 +208,4 @@ class ProfileFile: input : username output : NA """ - cmd = "su - %s -c \"echo ~\" 2>/dev/null" % username - (status, output) = subprocess.getstatusoutput(cmd) - if status == 0: - return output + "/.bashrc" - else: - return ClusterConstants.HOME_USER_BASHRC % username + return ClusterConstants.HOME_USER_BASHRC % username \ No newline at end of file diff --git a/script/domain_utils/domain_common/cluster_constants.py b/script/domain_utils/domain_common/cluster_constants.py index 7f4e84d9..43baef94 100644 --- a/script/domain_utils/domain_common/cluster_constants.py +++ b/script/domain_utils/domain_common/cluster_constants.py @@ -42,4 +42,5 @@ class ClusterConstants: LCCTL_LOG_FILE = "gs_lcctl.log" RESIZE_LOG_FILE = "gs_resize.log" HOTPATCH_LOG_FILE = "gs_hotpatch.log" + GS_CHECKOS_LOG_FILE = "gs_checkos.log" diff --git a/script/gs_checkos b/script/gs_checkos index 5f952275..f4743a58 100644 --- a/script/gs_checkos +++ b/script/gs_checkos @@ -37,6 +37,8 @@ from base_utils.os.file_util import FileUtil from base_utils.os.net_util import NetUtil from domain_utils.domain_common.cluster_constants import ClusterConstants from base_utils.common.constantsbase import ConstantsBase +from base_utils.os.user_util import UserUtil +from domain_utils.cluster_file.cluster_config_file import ClusterConfigFile ############################################################################# @@ -134,6 +136,7 @@ ACTION_SET_IO_CONFIGURE = "Set_IO_Configure" ACTION_SET_IO_REQUEST = "Set_IO_REQUEST" ACTION_SET_ASYNCHRONOUS_IO_REQUEST = "Set_Asynchronous_IO_Request" +current_user_root = False ####################################################### class CmdOptions(): @@ -157,10 +160,17 @@ class CmdOptions(): self.localMode = False self.skipOSCheck = [] self.skip_OS_Check = {} + self.skip_cgroup_set = False ######################################################### +def check_current_user(): + if os.getuid() == 0: + current_user_root = True + else: + current_user_root = False + # Init global log ######################################################### def initGlobals(): @@ -331,6 +341,15 @@ def parseCommandLine(): if (ParaDict.__contains__("skipOSCheck")): g_opts.skipOSCheck = ParaDict.get("skipOSCheck") +def get_user_info(): + """ + function: get user + input: NA + output: NA + """ + user_info = UserUtil.getUserInfo() + g_opts.user = user_info.get("name") + DefaultValue.checkPathVaild(g_opts.user) def readHostFile(hostfile): """ @@ -373,19 +392,38 @@ def checkConfigFile(): GaussLog.exitWithError( ErrorCode.GAUSS_502["GAUSS_50201"] % g_opts.confFile) +def get_checkos_log_path(log_name, xml): + """ + function: get the checkos log path + input: log_name, xml + output: fullLogPath + """ + try: + gs_checkos_log_path = "" + # get the log path + configedLogPath = ClusterConfigFile.getOneClusterConfigItem("gaussdbLogPath", xml) + DefaultValue.checkPathVaild(configedLogPath) + # check gaussdbLogPath is not null + if configedLogPath == "": + gs_checkos_log_path = "%s/%s/om/%s" % ( + ClusterConstants.GAUSSDB_DIR, g_opts.user, log_name) + else: + gs_checkos_log_path = "%s/%s/om/%s" % ( + os.path.normpath(configedLogPath), g_opts.user, log_name) + UserUtil.check_path_owner(gs_checkos_log_path) + return gs_checkos_log_path + except Exception as e: + GaussLog.exitWithError(str(e)) def setLogFile(): """ """ if (g_opts.logFile == ""): - cmd = "(if [ ! -d %s ]; then mkdir -p %s -m %s; fi)" % ( - LOG_DIR, LOG_DIR, DefaultValue.KEY_DIRECTORY_MODE) - (status, output) = subprocess.getstatusoutput(cmd) - if (status != 0): - GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50208"] - % "log of gs_checkos" + " Error: \n%s." - % output + "The cmd is %s" % cmd) - g_opts.logFile = os.path.join(LOG_DIR, "gs_checkos.log") + g_opts.logFile = get_checkos_log_path(ClusterConstants.GS_CHECKOS_LOG_FILE, g_opts.confFile) + if (not os.path.isabs(g_opts.logFile)): + GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50213"] + % g_opts.logFile) + UserUtil.check_path_owner(g_opts.logFile) def checkItems(): @@ -409,6 +447,7 @@ def checkParameter(): """ Check parameter from command line """ + get_user_info() ############################################ # check hostlist info ########################################### @@ -482,6 +521,8 @@ def doCheckOS(itemNumber): elif (itemNumber == 'A8' and g_opts.skip_OS_Check["A8"]): checkDiskConfigure() elif (itemNumber == 'A9' and g_opts.skip_OS_Check["A9"]): + if not current_user_root: + return checkBlockDevConfigure() checkLogicalBlock() elif (itemNumber == 'A10' and g_opts.skip_OS_Check["A10"]): @@ -489,6 +530,8 @@ def doCheckOS(itemNumber): checkMaxAsyIOrequests() checkIOConfigure() elif (itemNumber == 'A11' and g_opts.skip_OS_Check["A11"]): + if not current_user_root: + return checkNetworkConfigure() elif (itemNumber == 'A12' and g_opts.skip_OS_Check["A12"]): checkTimeConsistency() @@ -1511,8 +1554,7 @@ def main(): """ main function """ - if (os.getuid() != 0): - GaussLog.exitWithError(ErrorCode.GAUSS_501["GAUSS_50104"]) + check_current_user() global Local_CheckOs global Local_Check @@ -1565,10 +1607,10 @@ def main(): % g_opts.outputfile) for item in itemList: - if (g_opts.set == False): - doCheckOS(item) - else: + if (current_user_root and g_opts.set): doSetOS(item) + else: + doCheckOS(item) DisplayResultInformation(item, output) if (fp): diff --git a/script/gs_expansion b/script/gs_expansion index 1fe22d61..47c2906f 100644 --- a/script/gs_expansion +++ b/script/gs_expansion @@ -47,6 +47,7 @@ from impl.expansion.expansion_impl_with_cm_local import ExpansionImplWithCmLocal from domain_utils.cluster_file.cluster_config_file import ClusterConfigFile from domain_utils.cluster_file.cluster_log import ClusterLog from base_utils.os.env_util import EnvUtil +from base_utils.os.user_util import UserUtil ENV_LIST = ["MPPDB_ENV_SEPARATE_PATH", "GPHOME", "PATH", "LD_LIBRARY_PATH", "PYTHONPATH", "GAUSS_WARNING_TYPE", @@ -113,6 +114,15 @@ General options: """ print(self.usage.__doc__) + def check_current_user(self): + user_info = UserUtil.getUserInfo() + if user_info['uid'] == 0: + self.current_user_root = True + else: + self.current_user_root = False + self.user = user_info['name'] + self.group = user_info['g_name'] + def parseCommandLine(self): """ parse parameter from command line @@ -124,6 +134,8 @@ General options: if (ParaDict.__contains__("helpFlag")): self.usage() sys.exit(0) + # check no root parameter + self.check_no_root_parameter(ParaDict) # Resolves command line arguments # parameter -U if (ParaDict.__contains__("user")): @@ -149,6 +161,18 @@ General options: if (ParaDict.__contains__("time_out")): self.time_out = ParaDict.get("time_out") + def check_no_root_parameter(self, para_dict): + """ + function: Check no root user paramter + input: NA + output: NA + """ + if not self.current_user_root: + if (para_dict.__contains__("user") and (para_dict.get("user") != self.user)): + GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50324"]) + if (para_dict.__contains__("group") and (para_dict.get("group") != self.group)): + GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50324"]) + def checkParameters(self): """ function: Check parameter from command line @@ -157,10 +181,11 @@ General options: """ # check user | group | xmlfile | node - if len(self.user) == 0: - GaussLog.exitWithError(ErrorCode.GAUSS_357["GAUSS_35701"] % "-U") - if len(self.group) == 0: - GaussLog.exitWithError(ErrorCode.GAUSS_357["GAUSS_35701"] % "-G") + if self.current_user_root: + if len(self.user) == 0: + GaussLog.exitWithError(ErrorCode.GAUSS_357["GAUSS_35701"] % "-U") + if len(self.group) == 0: + GaussLog.exitWithError(ErrorCode.GAUSS_357["GAUSS_35701"] % "-G") if len(self.xmlFile) == 0: GaussLog.exitWithError(ErrorCode.GAUSS_357["GAUSS_35701"] % "-X") if len(self.newHostList) == 0: @@ -287,13 +312,6 @@ General options: self.backIpNameMap[backip] = \ self.clusterInfo.getNodeNameByBackIp(backip) - def checkExecutingUser(self): - """ - check whether current user executing this command is root - """ - if os.getuid() != 0: - GaussLog.exitWithError(ErrorCode.GAUSS_501["GAUSS_50104"]) - def checkExecutingHost(self): """ check whether current host is primary host @@ -322,17 +340,19 @@ General options: rootSSHExceptionHosts = [] individualSSHExceptionHosts = [] for host in hostList: - # check root's trust - checkRootTrustCmd = "%s -s -H %s 'pwd'" % (psshPath, host) - (status, output) = subprocess.getstatusoutput(checkRootTrustCmd) - if status != 0: - rootSSHExceptionHosts.append(host) - # check individual user's trust - checkUserTrustCmd = "su - %s -c '%s -s -H %s pwd'" % ( - self.user, psshPath, host) - (status, output) = subprocess.getstatusoutput(checkUserTrustCmd) - if status != 0: - individualSSHExceptionHosts.append(host) + if self.current_user_root: + # check individual user's trust + checkUserTrustCmd = "su - %s -c '%s -s -H %s pwd'" % ( + self.user, psshPath, host) + (status, output) = subprocess.getstatusoutput(checkUserTrustCmd) + if status != 0: + individualSSHExceptionHosts.append(host) + else: + # check current user's trust, if user is root or non root + checkRootTrustCmd = "%s -s -H %s 'pwd'" % (psshPath, host) + (status, output) = subprocess.getstatusoutput(checkRootTrustCmd) + if status != 0: + rootSSHExceptionHosts.append(host) # output ssh exception info if ssh connect failed if rootSSHExceptionHosts or individualSSHExceptionHosts: sshExceptionInfo = "" @@ -353,6 +373,12 @@ General options: check whether info in XML is consistent with environment variable """ self.logger.debug("Checking environment variable.") + if not self.envFile: + self.envFile = "/home/%s/.bashrc" % self.user + cmd = "source %s" % self.envFile + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + raise Expansion("not found envfile.") if not EnvUtil.getEnv("GPHOME"): GaussLog.exitWithError(ErrorCode.GAUSS_518["GAUSS_51802"] % ( "\"GPHOME\", please import environment variable")) @@ -408,7 +434,7 @@ General options: expand_cluster_info.compare_cluster_info(static_cluster_info, xml_cluster_info) - def expand_run(self): + def expand_run(self, expansion): """ This is expansion frame start """ @@ -579,7 +605,7 @@ if __name__ == "__main__": """ """ expansion = Expansion() - expansion.checkExecutingUser() + expansion.check_current_user() expansion.parseCommandLine() expansion.checkParameters() expansion.initLogs() @@ -588,4 +614,4 @@ if __name__ == "__main__": expansion.checkXmlIncludeNewHost() expansion.checkExecutingHost() expansion.checkTrust() - expansion.expand_run() + expansion.expand_run(expansion) diff --git a/script/gs_install b/script/gs_install index e4ec0c82..9e929d1f 100644 --- a/script/gs_install +++ b/script/gs_install @@ -68,6 +68,7 @@ class Install(ParallelBaseOM): self.cm_server_guc_param = [] self.action = "gs_install" self.initStep = "Init Install" + self.enable_perf_config = False def usage(self): """ @@ -92,6 +93,7 @@ General options: For more information, see \"gs_guc --help\". --alarm-component=ALARMCOMPONENT Path of the alarm component. --time-out=SECS Maximum waiting time when start cluster. + --enable-perf-config Use gs_perfconfig to tune database setup and guc after installation. """ print(self.usage.__doc__) @@ -219,6 +221,9 @@ General options: # parameter --dorado-cluster-mode if (ParaDict.__contains__("dorado-cluster-mode")): self.dorado_cluster_mode = ParaDict.get("dorado-cluster-mode") + # parameter --enable-perf-config + if (ParaDict.__contains__("enable-perf-config")): + self.enable_perf_config = True def checkUser(self): """ diff --git a/script/gs_perfconfig b/script/gs_perfconfig new file mode 100644 index 00000000..46c206f1 --- /dev/null +++ b/script/gs_perfconfig @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +import os +import sys +import getopt +import logging +from base_utils.common.dialog import DialogUtil +from impl.perf_config.basic.project import Project, PorjectError, ProjectLogLevel +from impl.perf_config.basic.anti import AntiLog +from impl.perf_config.preset.preset import Preset +from impl.perf_config.perf_probe import PerfProbe +from impl.perf_config.perf_tuner import PerfTuneTarget, PerfTuner + + +def usage(): + Project.msg(""" +Usage of gs_perfconfig: + gs_perfconfig [ACTION [ ACTION-PARAM ] ] + +Action, action-params and functions are as follows: + help (No params) : + Display help document of gs_perfconfig. '--help', '-h', '-?' are also effective. + + tune [ -t [ all,os,setup,guc,suggest ] ] [ --apply ] [--env envfile] [ -y ]: + '-t' include four tuning target modules: + os : (only root) Including os parameters, software, hardware configurations. + setup : Including database setup configuration. + guc : Including database GUC configuration. + suggest : Give some additional suggestions. + 'all' indicates all modules, but 'os' depend on the executor. + '--apply' indicates that the actual application is adjusted, and the database may need + to be restarted multiple times during this period. Otherwise, only one report is generated. + '--env' specifies the environment variables file. + '-y' Accept the requirements for configuring operating system parameters, GUC parameters, + and restarting the database during adjustment. + + When the root user is used to execute the process, the process is completely executed from + the beginning and certain information is generated. When the user executes the command, the + system first attempts to load the information left by the root user, and then skips certain + processes. + + preset [ --help | (No params) | preset-name | ] + Print the preset addition guide, all preset list or specify the contents of the preset. + + recover [ -y ] : + Recover the content of the last tuning. + '-y' Accept the requirements for configuring operating system parameters, GUC parameters, + and restarting the database during adjustment. + + +The environment variable GS_PERFCONFIG_OPTIONS sets some behavior within the tool. + export GS_PERFCONFIG_OPTIONS='param1=value1:param2=value2:param3=value3' + +params and values: + + lowest_print_log_level = log/notice/warning/error/fatal. The lowest level of print logs, default is notice. + You can also set 'msg', but it will not take effect. + """) + + +class TuneTask(object): + """ + This is the tune task. + Responsible for controlling the flow of the entire adjustment task. + The operations include environment initialization, performance data detection, + adjustment process, and rollback process. + """ + def __init__(self, argv): + self.accept_risk = False + target, apply, env = self._parse_arg(argv) + self.tune_target = PerfTuneTarget(target, apply) + if self.tune_target.noTarget(): + return + + # environment initialization + Project.initEnviron(env, True) + Project.initRole() + Project.initProjectLog(Project.environ.run_log) + Project.prepareReport(Project.environ.report) + Project.log(Project.environ.__str__()) + if self.tune_target.apply(): + AntiLog.initAntiLog(Project.environ.anti_log) + Project.setTask(self) + + def _parse_arg(self, argv): + short_options = 't:y' + long_options = ['apply', 'env='] + opts, args = getopt.getopt(argv, short_options, long_options) + if len(args) > 1: + Project.fatal('unknown param ' + args[0]) + target = 'all' + apply = False + env = None + for opt, arg in opts: + if opt == '-t': + target = arg + elif opt == '--apply': + apply = True + elif opt == '--env': + env = arg + elif opt == '-y': + self.accept_risk = True + else: + Project.fatal('unknown param ' + opt) + return target, apply, env + + def run(self): + """ + Task details. Control the entire tune process. + """ + if self.tune_target.noTarget(): + Project.notice('nothing to tune.') + return + do_apply = self.tune_target.apply() + # 1, Operation content and risk tips. + self.risk_disclosure() + + # 2, start probe + Project.notice('Start probe detect.') + infos = PerfProbe() + Project.setGlobalPerfProbe(infos) + infos.detect() + + # 3, shutdown openGauss if necessary + og_alive_at_first = Project.isOpenGaussAlive() + if og_alive_at_first and do_apply: + Project.stopOpenGauss() + + # 4, tune and apply, and rollback apply when errors. + error_occurred = False + Project.notice('Prepare tune plan.') + tuner = PerfTuner() + Project.setGlobalPerfTuner(tuner) + tuner.calculate() + + try: + Project.notice('execute tune plan({0})...'.format( + 'report and apply' if do_apply else 'just report')) + tuner.explain(do_apply) + Project.report.dump() + Project.notice('Tune finish.') + + except PorjectError: + error_occurred = True + except Exception as e: + Project.notice('Some errors have occurred.') + Project.notice(e.__str__()) + logging.exception(e) + error_occurred = True + + if error_occurred and do_apply: + Project.notice('start rollback.') + PerfTuner.rollback(None) + + # 5, start og if necessary. + if og_alive_at_first and do_apply: + Project.startOpenGauss() + + def risk_disclosure(self): + question = ('Certainly, we will perform some stress tests, make adjustments to the \n' + 'configuration of operating system parameters, database parameters, and \n' + 'also perform a database restart during the optimization process. \n' + 'Are you accepting of the aforementioned circumstances?') + Project.log('risk disclosure: ' + question) + if self.accept_risk: + Project.notice(f'user choose yes by "-y" on the question:\n ({question}).') + else: + ok = DialogUtil.yesOrNot(question) + Project.log('user choose: ' + ('y' if ok else 'n')) + if not ok: + Project.fatal('The user has chosen to cancel the operation.') + + +class PresetTask(object): + """ + This is the preset task. + Responsible for showing how many presets there are, the details of the presets, + and showing how to edit the presets. + """ + def __init__(self, argv): + Project.set_lowest_print_level(ProjectLogLevel.WARNING) + Project.initEnviron(None, False) + Project.reset_lowest_print_level() + + self.target_preset = None if (argv is None or len(argv) == 0) else argv[0] + if len(argv) > 1: + Project.fatal('unknown param {}'.format(argv[1])) + + def run(self): + if self.target_preset is None: + self._show_all_presets() + elif self.target_preset.lower() in ['help', '--help', '-h', '-?']: + self._show_preset_usage() + else: + self._show_one_preset(self.target_preset) + + def _show_all_presets(self): + builtins, usersets = Preset.get_all_presets() + + msg = 'Builtin Presets:\n' + for preset in builtins: + msg += f' {preset}\n' + Project.msg(msg) + + msg = 'User Presets:\n' + for preset in usersets: + msg += f' {preset}\n' + Project.msg(msg) + + def _show_one_preset(self, name): + preset = Preset(name) + Project.msg(preset.__str__()) + + def _show_preset_usage(self): + preset_usage = Preset.usage() + Project.msg(preset_usage) + + +class RecoverTask(object): + """ + This is a recovery mission + Read the anti log, undo the last adjustment, and restore the environment + to the way it was before the tune. Only the most recent adjustment can be undone. + """ + def __init__(self, argv): + self.accept_risk = False + if len(argv) > 0: + if len(argv) == 1 and argv[0] == '-y': + self.accept_risk = True + else: + Project.fatal('invalid param {}'.format(' '.join(argv))) + + Project.initEnviron() + Project.initRole() + Project.initProjectLog(Project.environ.run_log) + Project.prepareReport(Project.environ.report) + Project.log(Project.environ.__str__()) + + AntiLog.initAntiLog(Project.environ.anti_log, reload=True) + + def run(self): + Project.notice('start rollback.') + self.risk_disclosure() + + og_is_alive_first = Project.isOpenGaussAlive() + if og_is_alive_first: + Project.stopOpenGauss() + + PerfTuner.rollback(None) + + Project.notice('Destroy anti log.') + AntiLog.destroyAntiLog() + + if og_is_alive_first: + Project.startOpenGauss() + + Project.notice('rollback finish.') + + def risk_disclosure(self): + question = ('Certainly, we will make adjustments to the configuration of operating \n' + 'system parameters, database parameters, and also perform a database \n' + 'restart during the recover process.\n' + 'Are you accepting of the aforementioned circumstances?') + Project.log('risk disclosure: ' + question) + if self.accept_risk: + ok = True + Project.notice(f'user choose yes by "-y" on the question:\n ({question}).') + else: + ok = DialogUtil.yesOrNot(question) + Project.log('user choose: ' + ('y' if ok else 'n')) + + if not ok: + Project.fatal('The user has chosen to cancel the recover.') + + +if __name__ == '__main__': + task = None + + if len(sys.argv) == 1 or sys.argv[1].lower() in ['help', '--help', '-h', '-?']: + usage() + exit(0) + + elif sys.argv[1] == 'tune': + task = TuneTask(sys.argv[2:]) + + elif sys.argv[1] == 'preset': + task = PresetTask(sys.argv[2:]) + + elif sys.argv[1] == 'recover': + task = RecoverTask(sys.argv[2:]) + + else: + Project.fatal('Unknown task: ' + sys.argv[1]) + + task.run() + + diff --git a/script/gs_preinstall b/script/gs_preinstall index 8c6b56ec..a3cc763d 100644 --- a/script/gs_preinstall +++ b/script/gs_preinstall @@ -88,7 +88,7 @@ class Preinstall(ParallelBaseOM): self.envParams = [] self.rootUser = "" self.rootPasswd = "" - self.createUserSshTrust = True + self.create_user_ssh_trust = True self.clusterToolPath = "" self.needFixOwnerPaths = [] self.preMode = False @@ -105,6 +105,9 @@ class Preinstall(ParallelBaseOM): self.enable_dss = "" self.dss_vg_info = "" self.dss_vgname = "" + self.enable_perf_config = False + self.skip_cgroup_set = False + def usage(self): """ @@ -132,6 +135,8 @@ General options: --sep-env-file=ENVFILE Path of the MPP environment file. --skip-hostname-set Whether to skip hostname setting. (The default value is set.) + --skip-cgroup-set Whether to skip cgroup setting. + (The default value is set.) -l Path of log file. -?, --help Show help information for this utility, and exit the command line mode. @@ -147,6 +152,8 @@ General options: (The default value is not deleted) --unused-third-party Whether to use om's third-party. (The default value is used) + --enable-perf-config Use gs_perfconfig to tune os configuration + after pre installation. """ print(self.usage.__doc__) @@ -165,6 +172,8 @@ General options: self.usage() sys.exit(0) + # check no root user paramters + self.check_no_root_paramter(ParaDict) # Resolves command line arguments # parameter -U if (ParaDict.__contains__("user")): @@ -192,6 +201,9 @@ General options: # parameter --skip-hostname-set if (ParaDict.__contains__("skipHostnameSet")): self.skipHostnameSet = ParaDict.get("skipHostnameSet") + # parameter --skip-cgroup-set + if (ParaDict.__contains__("skip_cgroup_set")): + self.skip_cgroup_set = ParaDict.get("skip_cgroup_set") # parameter --skip-os-set if (ParaDict.__contains__("skipOSSet")): self.skipOSSet = ParaDict.get("skipOSSet") @@ -204,7 +216,25 @@ General options: # parameter --delete-root-trust if (ParaDict.__contains__("root_delete_flag")): self.root_delete_flag = ParaDict.get("root_delete_flag") + # parameter --enable-perf-config + if (ParaDict.__contains__("enable_perf_config")): + self.enable_perf_config = True + def check_no_root_paramter(self, para_dict): + """ + function: Check no root user paramter + input: NA + output: NA + """ + if not self.current_user_root: + if (para_dict.__contains__("user") and (para_dict.get("user") != self.user)): + GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50324"]) + if (para_dict.__contains__("group") and (para_dict.get("group") != self.group)): + GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50324"]) + if not self.skipOSSet: + GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50028"] % 'os') + if not self.skip_cgroup_set: + GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50028"] % 'cgroup') def checkUserParameter(self): """ @@ -244,6 +274,16 @@ General options: + "User:Group[%s:%s]" % (self.user, self.group)) + def check_local_node_info(self): + """ + check local node info + """ + hostName = NetUtil.GetHostIpOrName() + g_nodeInfo = self.clusterInfo.getDbNodeByName(hostName) + if (g_nodeInfo is None): + GaussLog.exitWithError(ErrorCode.GAUSS_516["GAUSS_51620"] % "local" + + " It is not a host name %s." % hostName) + def check_config_content(self, g_nodeInfo): UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("gaussdbAppPath", self.xmlFile)) UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("gaussdbToolPath", self.xmlFile)) @@ -349,44 +389,50 @@ General options: GaussLog.exitWithError(ErrorCode.GAUSS_518["GAUSS_51808"] % \ checkoutput + "Please check %s." % envfile) - def checkParameter(self): + def delete_host_ip(self): """ - function: Check parameter from command line + function: delete host_ip env input: NA output: NA """ - ClusterUser.checkGroupParameter(self.user, self.group) - # remove HOST_IP info with /etc/profile and environ - cmd = "sed -i '/^export[ ]*HOST_IP=/d' /etc/profile" - (status, output) = subprocess.getstatusoutput(cmd) - if status != 0: - GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50205"] - % ClusterConstants.ETC_PROFILE + "The cmd is %s" % cmd) + if self.current_user_root: + cmd = "sed -i '/^export[ ]*HOST_IP=/d' /etc/profile" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50205"] + % ClusterConstants.ETC_PROFILE + "The cmd is %s" % cmd) + if "HOST_IP" in os.environ.keys(): os.environ.pop("HOST_IP") + def checkParameter(self): + """ + function: Check parameter from command line + input: NA + output: NA + """ + if self.current_user_root: + # check -G parameter + ClusterUser.checkGroupParameter(self.user, self.group) + + # remove HOST_IP info with /etc/profile and environ + self.delete_host_ip() # check config file ClusterConfigFile.checkConfigFile(self.xmlFile) # check user info self.checkUserParameter() # check user group match self.checkUserAndGroup() + # init cluster info self.initClusterInfo() - # check config content - hostName = NetUtil.GetHostIpOrName() - g_nodeInfo = self.clusterInfo.getDbNodeByName(hostName) - if (g_nodeInfo is None): - GaussLog.exitWithError(ErrorCode.GAUSS_516["GAUSS_51620"] % "local" + - " It is not a host name %s." % hostName) - + # check local node info + self.check_local_node_info() # check env-val self.checkEnvValueParameter() # check mpprc file self.checkMpprcFile() - # check log file self.checkLogFile() - # skip OS check self.checkskipOS() @@ -571,25 +617,43 @@ General options: if subdir_path.startswith(parentdir_path): GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50240"] % dss_home_path) -def clearHistTimeFormat(): - cmd = "sed -i '/HISTTIMEFORMAT=/d' /etc/profile" - (status, output) = subprocess.getstatusoutput(cmd) - if status != 0: - GaussLog.exitWithError("Clear HISTTIMEFORMAT from /etc/profile " - "failed.\nError: %s\nThe cmd is: %s\n" % - (output,cmd)) + def clear_hist_time_format(self): + if self.current_user_root: + cmd = "sed -i '/HISTTIMEFORMAT=/d' /etc/profile" + (status, output) = subprocess.getstatusoutput(cmd) + if status != 0: + GaussLog.exitWithError("Clear HISTTIMEFORMAT from /etc/profile " + "failed.\nError: %s\nThe cmd is: %s\n" % + (output,cmd)) + def check_current_user(self): + user_info = UserUtil.getUserInfo() + if user_info.get("uid") == 0: + self.current_user_root = True + else: + self.current_user_root = False + self.user = user_info.get("name") + self.group = user_info.get("g_name") + self.skipOSSet = True + self.skip_cgroup_set = True + +def remove_mpp_env(): + mpprc = os.environ.get('MPPDB_ENV_SEPARATE_PATH') + if mpprc: + os.environ.pop('MPPDB_ENV_SEPARATE_PATH') if __name__ == '__main__': """ main function """ - # check if user is root - if os.getuid() != 0: - GaussLog.exitWithError(ErrorCode.GAUSS_501["GAUSS_50104"]) - clearHistTimeFormat() try: + # if not remove mpprc env, it will affect preinstall init + remove_mpp_env() # Objectize class preinstall = Preinstall() + # check if user is root + preinstall.check_current_user() + # remove HISTTIMEFORMAT with /etc/profile for root + preinstall.clear_hist_time_format() # set LD_LIBRARY_PATH preinstall.setLibPath() # parse cmd lines diff --git a/script/gs_sshexkey b/script/gs_sshexkey index b669f166..e5f0ad8d 100644 --- a/script/gs_sshexkey +++ b/script/gs_sshexkey @@ -920,7 +920,7 @@ General options: self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50230"] % "known hosts file" + " Error:\n%s" % str(e)) - # eliminate duploicates in authorized_keys file + # eliminate duplicates in authorized_keys file try: tab = self.readAuthorizedKeys() self.writeAuthorizedKeys(tab) @@ -1053,7 +1053,7 @@ General options: status, output, cmd = self.send_trust_file(hostip) while status != 0 and max_try_times > 0: - self.logger.debug(f"send_trust_file failed, coutdown {max_try_times}, retry again.") + self.logger.debug(f"send_trust_file failed, countdown {max_try_times}, retry again.") self.logger.debug("errorinfo: hostip: %s, status: %d, output: %s, " % (hostip, status, output)) self.logger.debug("check os info: %s\n %s" % ( diff --git a/script/gspylib/common/Common.py b/script/gspylib/common/Common.py index 18022432..0e36f723 100644 --- a/script/gspylib/common/Common.py +++ b/script/gspylib/common/Common.py @@ -128,6 +128,7 @@ from gspylib.common.Constants import Constants from domain_utils.cluster_file.profile_file import ProfileFile from domain_utils.domain_common.cluster_constants import ClusterConstants from gspylib.common.aes_cbc_util import AesCbcUtil +from base_utils.os.user_util import UserUtil noPassIPs = [] g_lock = thread.allocate_lock() @@ -856,7 +857,7 @@ class DefaultValue(): dirName = os.path.dirname(os.path.realpath(__file__)) # Get the startup file of suse or redhat os - if (os.path.isdir(systemDir)): + if (os.path.isdir(systemDir) and os.getpid() == 0): # Judge if cgroup para 'Delegate=yes' is written in systemFile cgroup_gate = False cgroup_gate_para = "Delegate=yes" @@ -2280,10 +2281,12 @@ class DefaultValue(): input : module output: NAself """ - if os.getuid() == 0: + user_info = UserUtil.getUserInfo() + if user_info['uid'] == 0: return + user_name = user_info.get('name') # Get the temporary directory from PGHOST - tmpDir = EnvUtil.getTmpDirFromEnv() + tmpDir = EnvUtil.getTmpDirFromEnv(user_name) if not tmpDir: raise Exception(ErrorCode.GAUSS_518["GAUSS_51802"] % "PGHOST") # check if tmp dir exists diff --git a/script/gspylib/common/ErrorCode.py b/script/gspylib/common/ErrorCode.py index a4e4cc95..558471eb 100644 --- a/script/gspylib/common/ErrorCode.py +++ b/script/gspylib/common/ErrorCode.py @@ -108,7 +108,7 @@ class ErrorCode(): 'GAUSS_50025': "[GAUSS-50025] : There is illegal character '%s' in parameter %s.", 'GAUSS_50026': "[GAUSS-50026] : Failed to check %s parameters in the XML file.", 'GAUSS_50027': "[GAUSS-50027] : Parameter '%s' format error.", - + 'GAUSS_50028': "[GAUSS-50028] : Non root users do not support setting %s parameters." } ########################################################################### @@ -234,7 +234,8 @@ class ErrorCode(): "from /etc/ssh/sshd_config.", 'GAUSS_50322': "[GAUSS-50322] : Failed to encrypt the password for %s", 'GAUSS_50323': "[GAUSS-50323] : The user %s is not the cluster " - "installation user " + "installation user ", + 'GAUSS_50324': "[GAUSS-50324] : Non root user, -U -G parameter must be the current user and group." } ########################################################################### diff --git a/script/gspylib/common/GaussLog.py b/script/gspylib/common/GaussLog.py index eeddee8f..ee424f0f 100644 --- a/script/gspylib/common/GaussLog.py +++ b/script/gspylib/common/GaussLog.py @@ -454,13 +454,14 @@ class GaussLog: sys.exit(status) @staticmethod - def printMessage(msg): + def printMessage(msg, end='\n'): """ function: Print the String message input: msg + input: end output: NA """ - sys.stdout.write("%s\n" % msg) + sys.stdout.write("%s%s" % (msg, end)) class FormatColor(object): diff --git a/script/gspylib/common/OMCommand.py b/script/gspylib/common/OMCommand.py index 337d08f5..14291ff9 100644 --- a/script/gspylib/common/OMCommand.py +++ b/script/gspylib/common/OMCommand.py @@ -234,14 +234,15 @@ class OMCommand(): retry += 1 time.sleep(1) - hostnameCmd = "pssh -s -H %s 'cat /etc/hostname'" % (nodename) - (status, output) = subprocess.getstatusoutput(hostnameCmd) - if status == 0 and output.strip() == nodename: - pass - else: - raise Exception(ErrorCode.GAUSS_512["GAUSS_51248"] % nodename - + " Command: \"%s\". Error: \n%s" - % (hostnameCmd, output)) + if os.getuid() == 0: + hostnameCmd = "pssh -s -H %s 'cat /etc/hostname'" % (nodename) + (status, output) = subprocess.getstatusoutput(hostnameCmd) + if status == 0 and output.strip() == nodename: + pass + else: + raise Exception(ErrorCode.GAUSS_512["GAUSS_51248"] % nodename + + " Command: \"%s\". Error: \n%s" + % (hostnameCmd, output)) except Exception as e: raise Exception(str(e)) diff --git a/script/gspylib/common/ParallelBaseOM.py b/script/gspylib/common/ParallelBaseOM.py index 4f2a9539..82ac859b 100644 --- a/script/gspylib/common/ParallelBaseOM.py +++ b/script/gspylib/common/ParallelBaseOM.py @@ -18,6 +18,7 @@ import os import socket import sys import getpass +import pwd from subprocess import PIPE sys.path.append(sys.path[0] + "/../../") @@ -67,7 +68,7 @@ class ParallelBaseOM(object): self.mpprcFile = "" # Temporary catalog for install self.operateStepDir = EnvUtil.getTempDir( - "%s_step" % self.__class__.__name__.lower()) + "%s_step" % self.__class__.__name__.lower(), pwd.getpwuid(os.getuid()).pw_name) # Temporary files for install step self.operateStepFile = "%s/%s_step.dat" % ( self.operateStepDir, self.__class__.__name__.lower()) @@ -113,6 +114,8 @@ class ParallelBaseOM(object): # Adapt to 200 and 300 self.productVersion = None + # current user root + self.current_user_root = True def initComponent(self): """ diff --git a/script/gspylib/common/ParameterParsecheck.py b/script/gspylib/common/ParameterParsecheck.py index 99752f1e..b796e9e3 100644 --- a/script/gspylib/common/ParameterParsecheck.py +++ b/script/gspylib/common/ParameterParsecheck.py @@ -64,11 +64,13 @@ VALUE_CHECK_LIST = ["|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"", "{", # no child branch gs_preinstall = ["-?", "--help", "-V", "--version", "-U:", "-G:", "-L", "--skip-os-set", "-X:", "--skip-os-check", - "--env-var=", "--sep-env-file=", "--skip-hostname-set", - "-l:", "--non-interactive", "--delete-root-trust", "--unused-third-party"] + "--env-var=", "--sep-env-file=", "--skip-hostname-set", "--skip-cgroup-set", + "-l:", "--non-interactive", "--delete-root-trust", "--unused-third-party", + "--enable-perf-config"] gs_install = ["-?", "--help", "-V", "--version", "-X:", "-l:", "--gsinit-parameter=", "--dn-guc=", "--cms-guc=", - "--time-out=", "--dorado-cluster-mode=", "--alarm-component="] + "--time-out=", "--dorado-cluster-mode=", "--alarm-component=", + "--enable-perf-config"] gs_uninstall = ["-?", "--help", "-V", "--version", "-l:", "-L", "--delete-data"] gs_postuninstall = ["-?", "--help", "-V", "--version", "--delete-user", @@ -349,6 +351,7 @@ class Parameter(): "--non-interactive": "preMode", "--skip-os-set": "skipOSSet", "--skip-hostname-set": "skipHostnameSet", + "--skip-cgroup-set": "skip_cgroup_set", "--reset": "reset", "--parameter": "isParameter", "--binary": "isBinary", @@ -382,7 +385,8 @@ class Parameter(): "--non-print": "nonPrinting", "--dynamic": "dynamic", "--delete-root-trust": "root_delete_flag", - "--unused-third-party": "unused_third_party" + "--unused-third-party": "unused_third_party", + "--enable-perf-config": "enable-perf-config" } parameterIsBool_keys = parameterIsBool.keys() @@ -477,6 +481,7 @@ class Parameter(): PARAMETER_VALUEDICT["upgrade-package"] = value.strip() elif key == "--dorado-cluster-mode": PARAMETER_VALUEDICT["dorado-cluster-mode"] = value.strip() + # Only check / symbol for gs_lcct. if key in ("--name", "--nodegroup-name"): self.checkLcGroupName(key, value) diff --git a/script/gspylib/threads/SshTool.py b/script/gspylib/threads/SshTool.py index d00c9cd0..d6441b39 100644 --- a/script/gspylib/threads/SshTool.py +++ b/script/gspylib/threads/SshTool.py @@ -233,7 +233,10 @@ class SshTool(): output: True/False """ ownerPath = os.path.split(filePath)[0] - cmd = "su - %s -c 'cd %s'" % (username, ownerPath) + if os.getuid() == 0: + cmd = "su - %s -c 'cd %s'" % (username, ownerPath) + else: + cmd = "cd %s" % ownerPath (status, output) = subprocess.getstatusoutput(cmd) if status != 0: raise Exception(ErrorCode.GAUSS_500["GAUSS_50004"] diff --git a/script/impl/dropnode/DropnodeImpl.py b/script/impl/dropnode/DropnodeImpl.py index 7a7b1d04..5d40262f 100644 --- a/script/impl/dropnode/DropnodeImpl.py +++ b/script/impl/dropnode/DropnodeImpl.py @@ -74,7 +74,7 @@ class DropnodeImpl(): if envFile: self.envFile = envFile else: - self.envFile = ClusterConstants.ETC_PROFILE + self.envFile = ClusterConstants.HOME_USER_BASHRC % self.user gphomepath = EnvUtil.getEnv("GPHOME") if gphomepath: self.gphomepath = gphomepath diff --git a/script/impl/expansion/ExpansionImpl.py b/script/impl/expansion/ExpansionImpl.py index 6e627c66..7aa4bca7 100644 --- a/script/impl/expansion/ExpansionImpl.py +++ b/script/impl/expansion/ExpansionImpl.py @@ -147,7 +147,7 @@ class ExpansionImpl(): self.logger.log("Failed to rollback wal_keep_segments, please manually " "set it to original value %s." % self.walKeepSegments) else: - self.reloadPrimaryConf(self.user) + self.reloadPrimaryConf() def final(self): """ @@ -546,7 +546,7 @@ class ExpansionImpl(): else: command = "source /etc/profile;source %s;"\ "gs_om -t status --detail" % self.envFile - if isRootUser: + if isRootUser and self.context.current_user_root: command = "su - %s -c '%s'" % (self.user, command) self.logger.debug(command) sshTool = SshTool([primaryHost]) @@ -753,7 +753,7 @@ gs_guc set -D {dn} -c "available_zone='{azName}'" primaryHost = self.getPrimaryHostName() dataNode = self.context.clusterInfoDict[primaryHost]["dataNode"] command = "" - if user: + if self.context.current_user_root: command = "su - %s -c 'source %s;gs_ctl reload -D %s'" % \ (user, self.envFile, dataNode) else: @@ -1343,12 +1343,11 @@ remoteservice={remoteservice}'" self.logger.debug("Checking the consistence of datanodes.") primaryName = self.getPrimaryHostName() cmd = "" - if EnvUtil.getEnv("MPPDB_ENV_SEPARATE_PATH"): - cmd = "su - %s -c 'source %s;gs_om -t status --detail'" % \ + if self.context.current_user_root: + cmd = "su - %s -c 'source /etc/profile;source %s;gs_om -t status --detail'" % \ (self.user, self.envFile) else: - cmd = "su - %s -c 'source /etc/profile;source %s;"\ - "gs_om -t status --detail'" % (self.user, self.envFile) + cmd = "source %s;gs_om -t status --detail" % (self.envFile) sshTool = SshTool([primaryName]) resultMap, outputCollect = sshTool.getSshStatusOutput(cmd, [primaryName], self.envFile) @@ -1392,14 +1391,14 @@ remoteservice={remoteservice}'" if hostName == primary: continue dataNode = clusterInfoDict[hostName]["dataNode"] - if EnvUtil.getEnv("MPPDB_ENV_SEPARATE_PATH"): - cmd = "su - %s -c 'source %s;" \ - "gs_guc check -D %s -c \"available_zone\"'" % \ - (self.user, self.envFile, dataNode) - else: + if self.context.current_user_root: cmd = "su - %s -c 'source /etc/profile;source %s;" \ "gs_guc check -D %s -c \"available_zone\"'" % \ (self.user, self.envFile, dataNode) + else: + cmd = "source /etc/profile;source %s;" \ + "gs_guc check -D %s -c \"available_zone\"" % \ + (self.envFile, dataNode) sshTool = SshTool([hostIp]) resultMap, output = sshTool.getSshStatusOutput(cmd, [hostIp], self.envFile) @@ -1424,12 +1423,11 @@ remoteservice={remoteservice}'" curHostName = socket.gethostname() command = "" - if EnvUtil.getEnv("MPPDB_ENV_SEPARATE_PATH"): - command = "su - %s -c 'source %s;gs_om -t status --detail'" % \ - (self.user, self.envFile) - else: - command = "su - %s -c 'source /etc/profile;source %s;"\ + if self.context.current_user_root: + command = "su - %s -c 'source %s;"\ "gs_om -t status --detail'" % (self.user, self.envFile) + else: + command = "source %s; gs_om -t status --detail" % (self.envFile) sshTool = SshTool([curHostName]) resultMap, outputCollect = sshTool.getSshStatusOutput(command, [curHostName], self.envFile) @@ -1501,11 +1499,14 @@ remoteservice={remoteservice}'" if (fstat[stat.ST_UID] == uid and (mode & stat.S_IRUSR > 0)) or \ (fstat[stat.ST_GID] == gid and (mode & stat.S_IRGRP > 0)): pass - else: + elif self.context.current_user_root: self.logger.debug(ErrorCode.GAUSS_501["GAUSS_50100"] % (xmlFile, self.user)) os.chown(xmlFile, uid, gid) os.chmod(xmlFile, stat.S_IRUSR) + else: + GaussLog.exitWithError(ErrorCode.GAUSS_501["GAUSS_50100"] + % (xmlFile, self.user)) def checkUserAndGroupExists(self): """ @@ -1632,11 +1633,15 @@ remoteservice={remoteservice}'" self.logger.debug("[%s] rollbackPg_hbaCmd:%s" % (host, rollbackPg_hbaCmd)) sshTool.getSshStatusOutput(rollbackPg_hbaCmd, [host]) - reloadGUCCommand = "su - %s -c 'source %s; gs_ctl reload " \ - "-D %s'" % (self.user, self.envFile, dataNode) - self.logger.debug(reloadGUCCommand) + if self.context.current_user_root: + reload_guc_command = "su - %s -c 'source %s; gs_ctl reload " \ + "-D %s'" % (self.user, self.envFile, dataNode) + else: + reload_guc_command = "'source %s; gs_ctl reload " \ + "-D %s'" % (self.envFile, dataNode) + self.logger.debug(reload_guc_command) resultMap, outputCollect = sshTool.getSshStatusOutput( - reloadGUCCommand, [host], self.envFile) + reload_guc_command, [host], self.envFile) self.logger.debug(resultMap) self.logger.debug(outputCollect) self.cleanSshToolFile(sshTool) @@ -1762,7 +1767,7 @@ class GsCtlCommon: """ value = "" command = "" - if user: + if os.getuid() == 0 and user: command = "su - %s -c 'source %s; gs_guc check -D %s -c \"%s\"'" % \ (user, env, datanode, para) else: diff --git a/script/impl/expansion/expansion_impl_with_cm.py b/script/impl/expansion/expansion_impl_with_cm.py index 8f7f0089..a8fb2a0f 100644 --- a/script/impl/expansion/expansion_impl_with_cm.py +++ b/script/impl/expansion/expansion_impl_with_cm.py @@ -359,7 +359,10 @@ class ExpansionImplWithCm(ExpansionImpl): "\\\"%s='%s'\\\"" % (self.envFile, guc_path, "pgxc_node_name", self._get_pgxc_node_name_for_single_inst()) - su_cmd = """su - {0} -c "{1}" """.format(self.user, cmd) + if self.context.current_root_user: + su_cmd = """su - {0} -c "{1}" """.format(self.user, cmd) + else: + su_cmd = cmd self.logger.debug("Set guc parameter command: {0}".format(su_cmd)) status, output = subprocess.getstatusoutput(su_cmd) if status == 0: diff --git a/script/impl/install/InstallImpl.py b/script/impl/install/InstallImpl.py index 921c1b76..19f8db5e 100644 --- a/script/impl/install/InstallImpl.py +++ b/script/impl/install/InstallImpl.py @@ -110,6 +110,8 @@ class InstallImpl: self.context.logger.log("begin deploy..") self.doDeploy() self.context.logger.log("end deploy..") + # gs_perfconfig is not a step in the gs_install, so do it after log succeeded. + self.do_perf_config() # close the log file self.context.logger.closeLog() except Exception as e: @@ -704,6 +706,22 @@ class InstallImpl: self.context.logger.debug("Successfully started the cluster.", "constant") + def do_perf_config(self): + """ + function: start gs_perfconfig + input : NA + output: NA + """ + if not self.context.enable_perf_config: + return + self.context.logger.log("gs_install has finished, start gs_perfconfig now.") + + if False: + self.context.logger.logExit('Could not run gs_perfconfig by gs_preinstall in upgrade case.') + + cmd = 'gs_perfconfig tune -t setup,guc,suggest --apply -y' + CmdExecutor.execCommandLocally(cmd) + def rollbackInstall(self): """ function: Rollback install diff --git a/script/impl/perf_config/__init__.py b/script/impl/perf_config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/basic/__init__.py b/script/impl/perf_config/basic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/basic/anti.py b/script/impl/perf_config/basic/anti.py new file mode 100644 index 00000000..1b4d7bb6 --- /dev/null +++ b/script/impl/perf_config/basic/anti.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : Provides logic for rolling back adjustments. +############################################################################# + +import os +from impl.perf_config.basic.project import Project + + +class AntiLog(object): + """ + Anti log describes the opposite logic for adjusting content. + You can use anti log to roll back the tune information and + restore the environment to its original state. + + Anti log is similar to the write-ahead logging of a database. + Through the tuning log, we can learn about the adjustment details + and how to roll back the adjustment, record the rollback function + in anti log, so we can know how to rollback it. + + Anti log are stored in str format. Each line is a log. Therefore, + the log cannot contain '\n'. + + The format of a typical anti log is as follows: + + "module-name -:|:- log-content" + + 1, we can find a module in AntiLog._modules by module name. + 2, " -:|:- " is a separator. + 3, log-content are produced by the module, the module also needs + to provide an interface to parse and execute the contents of + the log-content. + + """ + init_done = False + _file = None # anti log file + _records = [] # anti log content in memory. [alog1, alog2, ...] + _modules = {} # modules to exec anti log. { module1_name: module1, ...} + + def __init__(self): + assert False, 'AntiLog is just a interface package.' + + @staticmethod + def initAntiLog(file, reload=False): + """ + init anti log + :param file: anti log file + :param reload: reload anti log from file or recreate a new empty anti log. + :return: NA + """ + AntiLog._file = file + if reload: + if not os.access(file, os.F_OK): + Project.fatal('Could not access anti log: ' + file) + with open(file, 'r') as f: + AntiLog._records = f.readlines() + else: + with open(file, 'w') as f: + pass + Project.role.chown_to_user(file) + AntiLog.init_done = True + Project.notice('Anti Log ({0}) {1} done '.format(file, 'reload' if reload else 'init')) + + @staticmethod + def destroyAntiLog(): + """ + destory anti log + :return: NA + """ + os.remove(AntiLog._file) + AntiLog.init_done = False + AntiLog._file = None + AntiLog._records = [] + AntiLog._modules = {} + + @staticmethod + def write(module_name, alog): + """ + write an anti log in to memory and file. + :param module_name: module to exec this log + :param alog: log content + :return: + """ + assert isinstance(alog, str) + assert alog.find('\n') < 0, 'Anti log can not contain "\\n".' + + record = '{0} -:|:- {1}\n'.format(module_name, alog) + AntiLog._records.append(record) + with open(AntiLog._file, 'a+') as f: + f.write(record) + f.flush() + + @staticmethod + def register(module): + """ + register a module. + :param module: module to exec this log + :return: + """ + name = module.__name__ + if AntiLog._modules.get(name) is None: + Project.log('AntiLog register module '.format(name)) + AntiLog._modules[name] = module + + @staticmethod + def rollback(): + """ + Traverse the anti log in reverse order, invoke the corresponding module to rollback. + """ + for alog in AntiLog._records[::-1]: + parts = alog.split(' -:|:- ') + module = AntiLog._modules.get(parts[0]) + assert module is not None, '{} is not registered.'.format(parts[0]) + module.rollback(parts[1]) + diff --git a/script/impl/perf_config/basic/guc.py b/script/impl/perf_config/basic/guc.py new file mode 100644 index 00000000..6b88b1e6 --- /dev/null +++ b/script/impl/perf_config/basic/guc.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : A series of adjustment methods for guc. +############################################################################# + + +import os +import json +import shutil + +from base_utils.os.cmd_util import CmdUtil +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.anti import AntiLog +from impl.perf_config.basic.tuner import Tuner, TunerGroup + + +class GucMap(object): + """ + GucMap is a non-instantiated class. The class variable globalGucDict stores all + registered GUC parameters and operates these GUC parameters through a series + of static interfaces. + """ + _globalGucDict = {} + tunePlan = [] + + def __init__(self): + assert False, 'class GucMap is set only as a series of global interfaces package.' + + @staticmethod + def set(name, value): + GucMap._globalGucDict[name].set(value) + + @staticmethod + def show(name): + return GucMap._globalGucDict[name].value + + @staticmethod + def register(name): + """ + :param name: name of a guc + :return: GUCTunePoint + """ + assert GucMap._globalGucDict.get(name) is None, '{} is duplicate.'.format(name) + guc = GUCTunePoint(name) + GucMap._globalGucDict[name] = guc + return guc + + +class GUCTunePoint(Tuner): + """ + Components for adjusting individual GUCs. + + The guc parameters are adjusted on a group basis, rather than individually. + So there are special modules to do the calculations. + + All we need to do here is provide an interface to receive the value and + make the value effective. + + However, we do not need to provide a rollback method, because there is a + better solution: save the postgresql.conf in advance, and restore it directly + when we need to recover. see: GucRootTuner + """ + def __init__(self, name): + super(GUCTunePoint, self).__init__(name) + self.name = name + self.value = None + + def set(self, value): + self.value = value + + def turn_off(self): + self.value = 'off' + + def turn_on(self): + self.value = 'on' + + def calculate(self): + """ + no calculate function. guc tuner group will do this. + :return: + """ + assert False, ' should not be here (calculate).'.format(self.name) + + def explain(self, apply=False): + """ + not need alog. + :return: + """ + if self.value is None: + return + + infos = Project.getGlobalPerfProbe() + Project.report.record(f'- set guc {self.name} to {self.value}') + if apply: + cmd = """gs_guc set -D {0} -c "{1}='{2}'" """.format( + infos.db.gauss_data, self.name, self.value) + GucMap.tunePlan.append(cmd) + Project.log('prepare guc set cmd: ' + cmd) + + @staticmethod + def rollback(alog): + """ + no rollback function. guc tuner group will do this. + :return: + """ + assert False, ' should not be here (rollback).' + + +class GUCTuneGroup(Tuner): + """ + The guc parameters are adjusted on a group basis, rather than individually. + + We group the guc according to different modules and logic, and each group is a GUCTuneGroup. + + """ + def __init__(self): + super(GUCTuneGroup, self).__init__() + self._guc_list = [] + + def bind(self, name): + """ + bind a guc on this group. + :param name: tuner or tuner group. + :return: NA + """ + guc = GucMap.register(name) + self._guc_list.append(guc) + return guc + + def explain(self, apply=False): + """ + Iterate sub tuner and explain it in turn. + :return: NA + """ + for guc in self._guc_list: + guc.explain(apply) + + +class GucRootTuner(TunerGroup): + """ + Root node of the guc tuning logical tree. It's also an entry point for tune. + + GucRootTuner manages different GUCTuneGroup and different GUCTuneGroup manage different GucTunePoint. + + Anti log records here. + We save postgresql.conf in advance and record the save location in anti log, + then calculate and tune guc parameters normally. When we need to roll back, read the anti log, + get the location of the postgresql.conf backup and restore the it directly. + """ + def __init__(self): + super(GucRootTuner, self).__init__() + self.postgresql_conf = None + self.postgresql_conf_bak = None + self.omm = None + self.omm_uid = None + self.omm_gid = None + self.tmp_script = None + + def calculate(self): + infos = Project.getGlobalPerfProbe() + self.postgresql_conf = os.path.join(infos.db.gauss_data, 'postgresql.conf') + self.postgresql_conf_bak = os.path.join(Project.environ.workspace2 or Project.environ.workspace1, + 'postgresql.conf.bak') + self.omm = infos.db.omm + self.omm_uid = infos.db.omm_uid + self.omm_gid = infos.db.omm_gid + + self.tmp_script = os.path.join(Project.environ.workspace2 or Project.environ.workspace1, + 'guc_tune_plan.sh') + + super(GucRootTuner, self).calculate() + + def explain(self, apply=False): + """ + explain the tune logic. if apply, save postgresql.conf in advance and record the save + location in anti log + """ + super(GucRootTuner, self).explain(apply) + + if apply: + infos = Project.getGlobalPerfProbe() + shutil.copy2(self.postgresql_conf, self.postgresql_conf_bak) + Project.role.chown_to_user(self.postgresql_conf_bak) + + alog = self._make_alog() + AntiLog.write(self.__class__.__name__, alog) + + tmp_script_content = '' + if Project.environ.env is not None: + tmp_script_content += f'source {Project.environ.env} \n' + tmp_script_content += '\n'.join(GucMap.tunePlan) + with open(self.tmp_script, 'w') as f: + f.write(tmp_script_content) + Project.role.chown_to_user(self.tmp_script) + + cmd = f"sh {self.tmp_script}" if not Project.haveRootPrivilege() else \ + f'su - {self.omm} -c "sh {self.tmp_script}"' + Project.log('Tune guc by command:' + cmd) + Project.log('tmp guc tune script content:\n' + tmp_script_content) + output = CmdUtil.execCmd(cmd) + Project.log('Output: ' + output) + + + def _make_alog(self): + alog = { + 'postgresql_conf': self.postgresql_conf, + 'postgresql_conf_bak': self.postgresql_conf_bak, + 'omm_uid': self.omm_uid, + 'omm_gid': self.omm_gid + } + return json.dumps(alog) + + @staticmethod + def _parse_alog(alog): + return json.loads(alog) + + @staticmethod + def rollback(alog): + content = GucRootTuner._parse_alog(alog) + shutil.copy2(content['postgresql_conf_bak'], content['postgresql_conf']) + Project.notice('rollback GUC. cp {0} {1}'.format( + content['postgresql_conf_bak'], content['postgresql_conf']) + ) + os.chown(content['postgresql_conf'], content['omm_uid'], content['omm_gid']) + diff --git a/script/impl/perf_config/basic/probe.py b/script/impl/perf_config/basic/probe.py new file mode 100644 index 00000000..697984b3 --- /dev/null +++ b/script/impl/perf_config/basic/probe.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : probe.py setup a base class model for organizing information +############################################################################# + + +from impl.perf_config.basic.project import Project + + +class ProbeNotebook(object): + """ + A notepad, an organization pattern of key-value pairs, + used to record some information at any time. + """ + def __init__(self): + self._notebook = {} + + def read(self, title): + return self._notebook.get(title) + + def write(self, title, content): + self._notebook[title] = content + + def delete(self, title): + if self._notebook.get(title) is None: + return + return self._notebook.pop(title) + + +class Probe(object): + """ + This is a base class for any probes. + It's essentially a collection of information. A class of information is + stored or an interface for obtaining information. + + There is also a built-in notebook, which is convenient to record something + at any time. + """ + def __init__(self, probe_name=''): + """ + :param probe_name: + """ + Project.log('Probe init {1}'.format(self.__class__.__name__, probe_name)) + self.probe_name = probe_name + self.notebook = ProbeNotebook() + + def detect(self): + """ + detect some perf information + :return: + """ + assert False, 'Subclass must override this function.' + + def refresh(self): + self.detect() + + +class ProbeGroup(Probe): + """ + Similar to TunerGroup, this is a component that manages probe groups. + """ + def __init__(self, probe_name=''): + super(ProbeGroup, self).__init__(probe_name) + self._sub_probe_groups = [] + + def add(self, sub_probe): + """ + + :param sub_probe: + :return: + """ + self._sub_probe_groups.append(sub_probe) + return sub_probe + + def detect(self): + """ + detect performance information for subset + :return: + """ + for sub in self._sub_probe_groups: + sub.detect() + + def refresh(self): + self.detect() diff --git a/script/impl/perf_config/basic/project.py b/script/impl/perf_config/basic/project.py new file mode 100644 index 00000000..cc7f23d9 --- /dev/null +++ b/script/impl/perf_config/basic/project.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : +############################################################################# + +import os +import sys +import pwd +import time +import psutil +import getpass +import traceback +from enum import Enum +from datetime import datetime +from base_utils.os.cmd_util import CmdUtil +from gspylib.common.GaussLog import GaussLog + + +class PorjectError(Exception): + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg + + +class ProjectEnv(object): + """ + Some environment-related information, including environment variables, + path configuration, parameter configuration, etc. + """ + def __init__(self, env=None, do_init=False): + """ + read some environ, prepare some information, if do_init, create dir and so on. + """ + self.id = '{0}-{1}'.format(time.time(), os.getpid()) + + # GS_PERFCONFIG_OPTIONS is a environ where we can set some params to control behavior of tool. + self.gs_perfconfig_options_str = self.read_env('GS_PERFCONFIG_OPTIONS') + self.gs_perfconfig_options = { + 'lowest_print_log_level': ProjectLogLevel.NOTICE + } + self._apply_gs_perfconfig_options() + + self.env = os.path.abspath(env) if env is not None else None + self.source_env(self.env) + + self.gauss_home = self.read_env('GAUSSHOME') + self.gauss_data = self.read_env('PGDATA') + self.gauss_log = self.read_env('GAUSSLOG') + if self.gauss_home is None or self.gauss_data is None or self.gauss_log is None: + Project.fatal('Could not find $GAUSSHOME, $GAUSSLOG or $PGDATA.\n' + 'Please check the environment variables or specified by --env.') + + self.workspace1 = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'impl', 'perf_config')) + self.workspace2 = os.path.abspath(os.path.join(self.gauss_log, 'om', 'perf_config')) + + self.preset_dir1 = os.path.join(self.workspace1, 'preset') + self.preset_dir2 = os.path.join(self.workspace2, 'preset') + + self.run_log = os.path.join(self.workspace2, 'run.log') + self.anti_log = os.path.join(self.workspace2, 'anti.log') + self.report = os.path.join(self.workspace2, f'report-{self.id}.md') + + if do_init: + self._do_init() + + Project.log('workspace1: {}'.format(self.workspace1)) + Project.log('workspace2: {}'.format(self.workspace2)) + Project.notice('Environment init done.') + + + def get_builtin_script(self, name): + """ + There are some built-in shell scripts that perform a set of shell operations. + Pass in the script name and get the absolute path to the script. + """ + builtin_script = os.path.join(self.workspace1, 'scripts', name) + if not os.access(builtin_script, os.F_OK): + Project.fatal(f'builtin script {builtin_script} does not exist.') + return builtin_script + + def _apply_gs_perfconfig_options(self): + """ + GS_PERFCONFIG_OPTIONS is a environ where we can set some params to control behavior of tool. + """ + def _get_a_bool(_value, _name, _default): + if _value in [False, '0', 'off', 'false']: + return False + elif _value in [True, '1', 'on', 'true']: + return True + else: + Project.warning('invalid value of GS_PERFCONFIG_OPTIONS option: ' + _name) + return _default + + if self.gs_perfconfig_options_str is None: + return + + for option in self.gs_perfconfig_options_str.split(':'): + kv = option.split('=') + if len(kv) != 2: + Project.warning('Could not parse option in GS_PERFCONFIG_OPTIONS: ' + option) + continue + elif kv[0] == 'lowest_print_log_level': + try: + level = ProjectLogLevel.get_level_by_str(kv[1]) + self.gs_perfconfig_options[kv[0]] = level + ProjectLog.set_lowest_print_level(level) + except: + Project.warning('invalid value of GS_PERFCONFIG_OPTIONS option: lowest_print_log_level') + else: + Project.warning('Unknown option in GS_PERFCONFIG_OPTIONS: ' + option) + + + def _do_init(self): + """ + create the dir. + """ + if not os.access(self.workspace2, os.F_OK): + os.mkdir(self.workspace2, 0o755) + + if not os.access(self.preset_dir2, os.F_OK): + os.mkdir(self.preset_dir2, 0o755) + + def __str__(self): + return str({ + 'id': self.id, + 'gauss_home': self.gauss_home, + 'gauss_data': self.gauss_data, + 'gauss_log': self.gauss_log, + 'workspace1': self.workspace1, + 'workspace2': self.workspace2, + 'gs_perfconfig_options_str': self.gs_perfconfig_options_str, + 'gs_perfconfig_options': self.gs_perfconfig_options + }) + + @staticmethod + def read_env(name): + val = os.getenv(name) + Project.log(f'read env {name}={val}') + return val + + @staticmethod + def source_env(env): + """ + read some environ from env file. + """ + if env is None: + return + cmd = f'{CmdUtil.SOURCE_CMD} {env} && env' + output = CmdUtil.execCmd(cmd) + for line in output.splitlines(): + kv = line.split('=') + if kv[0] not in ['GAUSSHOME', 'PGDATA', 'GAUSSLOG']: + continue + os.environ[kv[0]] = kv[1] + Project.log(f'export env: {kv[0]}={kv[1]}') + + +class ProjectLogLevel(Enum): + LOG = 0 + MSG = 1 + NOTICE = 2 + WARNING = 3 + ERR = 4 + FATAL = 5 + + @staticmethod + def get_level_by_str(string): + for level in ProjectLogLevel: + if string.lower() == level.name.lower(): + return level + raise PorjectError('unknown level string ' + string) + + +class ProjectLog(object): + # Sometimes, it is preferable not to display certain information on the screen. + # You can set this value to control what is printed on the screen. + # but 'ProjectLogLevel.MSG' is not controlled. + _lowest_print_level = ProjectLogLevel.NOTICE + + @staticmethod + def set_lowest_print_level(level): + ProjectLog._lowest_print_level = level + + @staticmethod + def show_lowest_print_level(): + return ProjectLog._lowest_print_level + + @staticmethod + def reset_lowest_print_level(): + ProjectLog._lowest_print_level = Project.environ.gs_perfconfig_options['lowest_print_log_level'] + + def __init__(self, file): + self.file = file + self._FILE = open(file, 'a') + Project.role.chown_to_user(file) + + # just some flag + Project.notice('Run log init done.') + self._FILE.write('\n' * 10) + self._FILE.write('#' * 30) + self._FILE.write('\n>>>>>>>>>> NEW LOG START <<<<<<<<<<\n') + + def __del__(self): + self._FILE.close() + + def do_log(self, level, content): + now = datetime.now() + formatted_time = now.strftime("%Y-%m-%d %H:%M:%S.%f") + level_tag = '[{}]'.format(level.name) + log_content = (content + '\n') if level == ProjectLogLevel.MSG else \ + '{0} {1}: {2}\n'.format(formatted_time, level_tag, content) + + self._FILE.write(log_content) + + if level == ProjectLogLevel.LOG: + ProjectLog.print_msg(log_content, level, end='') + + elif level == ProjectLogLevel.NOTICE: + ProjectLog.print_msg(log_content, level, end='') + + elif level == ProjectLogLevel.MSG: + ProjectLog.print_msg(log_content, level, end='') + + elif level == ProjectLogLevel.WARNING: + ProjectLog.print_msg(log_content, level, end='') + + elif level == ProjectLogLevel.ERR: + bt = ''.join(traceback.format_stack()[0:-1]) + ProjectLog.print_msg(log_content + '\n' + bt, level, end='') + raise PorjectError(content + '\n' + bt) + + elif level == ProjectLogLevel.FATAL: + bt = ''.join(traceback.format_stack()[0:-1]) + ProjectLog.print_msg(log_content + '\n' + bt, level, end='') + exit(1) + + else: + assert False + + @staticmethod + def print_msg(msg, level, end='\n'): + if level.value >= ProjectLog._lowest_print_level.value or level == ProjectLogLevel.MSG: + GaussLog.printMessage(msg, end=end) + + +class ProjectReport(object): + """ + Structure for storing reports and suggestions. + The content needs to be assembled in advance according to the markdown style. + """ + def __init__(self, file): + self._file = file + self._records = [] + self._suggestions = [] + Project.notice('Report log init done.') + + def record(self, content): + self._records.append(content) + self._records.append('\n') + + def suggest(self, content): + self._suggestions.append(content) + + def dump(self): + with open(self._file, 'w') as f: + f.write('# Tune Report\n\n') + f.write('\n'.join(self._records)) + if Project.getTask().tune_target.hasSuggest(): + f.write('\n\n# More Suggestions\n\n') + f.write('\n\n'.join(self._suggestions)) + Project.role.chown_to_user(self._file) + Project.notice('Report: ' + self._file) + + +class ProjectRole(object): + """ + Role control. Check who the current user is and who the omm user is. + Use gausshome's folder owner to automatically find omm users and optimize the use experience. + """ + def __init__(self): + self.current_role = getpass.getuser() + + stat = os.stat(Project.environ.gauss_home) + self.user_name = pwd.getpwuid(stat.st_uid).pw_name + self.user_uid = stat.st_uid + self.user_gid = stat.st_gid + + if self.current_role != 'root' and self.current_role != self.user_name: + Project.fatal(f'Illegal access detected. Current role is {self.current_role}, ' + f'but owner of $GAUSSHOME is {self.user_name}.') + + Project.notice(f'Role init done(current:{self.current_role}, user:{self.user_name}).') + + def chown_to_user(self, file): + if self.current_role != 'root': + assert self.current_role == self.user_name + return + os.chown(file, self.user_uid, self.user_gid) + + +class Project(object): + """ + Project is mainly to provide some global information, module interface. + """ + ################################################### + # TASK + ################################################### + _task = None + + @staticmethod + def getTask(): + return Project._task + + @staticmethod + def setTask(task): + assert Project._task is None + Project._task = task + + ################################################### + # ENVIRON OF PROJECT + ################################################### + environ = None + + @staticmethod + def initEnviron(env=None, do_init=False): + Project.environ = ProjectEnv(env, do_init) + + ################################################### + # ROLE CTRL OF PROJECT + ################################################### + role = None + + @staticmethod + def initRole(): + Project.role = ProjectRole() + + @staticmethod + def haveRootPrivilege(): + return getpass.getuser() == 'root' + + ################################################### + # LOG MODULE OF PROJECT + ################################################### + _log = None + + @staticmethod + def initProjectLog(file): + Project._log = ProjectLog(file) + + @staticmethod + def set_lowest_print_level(level): + ProjectLog.set_lowest_print_level(level) + + @staticmethod + def show_lowest_print_level(): + return ProjectLog.show_lowest_print_level(level) + + @staticmethod + def reset_lowest_print_level(): + ProjectLog.reset_lowest_print_level() + + + @staticmethod + def msg(content): + """ + message must print on screen. and write a LOG in log file. + """ + if Project._log is None: + ProjectLog.print_msg(content, ProjectLogLevel.MSG) + return + + Project._log.do_log(ProjectLogLevel.MSG, content) + + @staticmethod + def log(content): + if Project._log is None: + ProjectLog.print_msg(f'{ProjectLogLevel.LOG.name}: {content}', ProjectLogLevel.LOG) + return + Project._log.do_log(ProjectLogLevel.LOG, content) + + @staticmethod + def notice(content): + if Project._log is None: + ProjectLog.print_msg(f'{ProjectLogLevel.NOTICE.name}: {content}', ProjectLogLevel.NOTICE) + return + Project._log.do_log(ProjectLogLevel.NOTICE, content) + + @staticmethod + def warning(content): + if Project._log is None: + ProjectLog.print_msg(f'{ProjectLogLevel.WARNING.name}: {content}', ProjectLogLevel.WARNING) + return + Project._log.do_log(ProjectLogLevel.WARNING, content) + + @staticmethod + def err(content): + if Project._log is None: + ProjectLog.print_msg(f'{ProjectLogLevel.ERR.name}: {content}', ProjectLogLevel.ERR) + exit(1) + Project._log.do_log(ProjectLogLevel.ERR, content) + + @staticmethod + def fatal(content): + if Project._log is None: + ProjectLog.print_msg(f'{ProjectLogLevel.FATAL.name}: {content}', ProjectLogLevel.FATAL) + exit(1) + Project._log.do_log(ProjectLogLevel.FATAL, content) + + ################################################### + # PROBE AND TUNER OF PROJECT + ################################################### + _globalPerfProbe = None + _globalPerfTuner = None + + @staticmethod + def setGlobalPerfProbe(probe): + Project._globalPerfProbe = probe + + @staticmethod + def getGlobalPerfProbe(): + return Project._globalPerfProbe + + @staticmethod + def setGlobalPerfTuner(tuner): + Project._globalPerfTuner = tuner + + @staticmethod + def getGlobalPerfTuner(): + return Project._globalPerfTuner + + ################################################### + # Project report + ################################################### + report = None + + @staticmethod + def prepareReport(file): + Project.report = ProjectReport(file) + + ################################################### + # openGauss operate + ################################################### + @staticmethod + def startOpenGauss(): + cmd = 'gs_om -t start' + Project.notice('start openGauss.....') + + output = CmdUtil.execCmd(cmd) + + Project.notice('start openGauss finish.') + Project.log(output) + + @staticmethod + def stopOpenGauss(): + cmd = 'gs_om -t stop' + Project.notice('stop openGauss......') + + output = CmdUtil.execCmd(cmd) + + Project.notice('stop openGauss finish.') + Project.log(output) + + @staticmethod + def isOpenGaussAlive(): + pmid = os.path.join(Project.environ.gauss_data, 'postmaster.pid') + + try: + with open(pmid, 'r') as f: + pid = int(f.readlines()[0]) + p = psutil.Process(pid) + Project.notice(f'openGauss is running(pid:{pid} status:{p.status()})') + return True + except: + Project.notice('openGauss is not running.') + return False + + def __init__(self): + assert False, 'Project is just a package of interface.' + + diff --git a/script/impl/perf_config/basic/tuner.py b/script/impl/perf_config/basic/tuner.py new file mode 100644 index 00000000..cf6c5ddb --- /dev/null +++ b/script/impl/perf_config/basic/tuner.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : Provides the abstract logic of the Tuner. +############################################################################# + + +import os +import sys +import json +from base_utils.os.cmd_util import CmdUtil +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.anti import AntiLog + + +class Tuner(object): + """ + This is the basic class of tuner. + """ + + def __init__(self, tuner_name=''): + assert not Project.getTask().tune_target.apply() or AntiLog.init_done, \ + 'Must AntiLog.initAntiLog() first when apply is True.' + assert Project.getGlobalPerfProbe() is not None, 'Must Project.setGlobalPerfProbe() first.' + + Project.log('Tuner construct {1}'.format(self.__class__.__name__, tuner_name)) + + self.tuner_name = tuner_name + + def calculate(self): + """ + calculate the tune point + :return: + """ + assert False, 'Incomplete function: <{0} {1}>.calculate().'.format(self.__class__.__name__, self.tuner_name) + + def explain(self, apply=False): + """ + explain the tune points, generate a tune report. + If apply is True, we will really do the tune and write anti log. + In addition, in this interface, you need to pay attention to the following: + 1. Implement the '_make_report' interface in advance and invoke the interface to + generate a report. + 2. Record the content in logs. + 3. If the tune point is applied, record the anti log. + :param apply: + :return: + """ + assert False, 'Incomplete function: <{0} {1}>.calculate().'.format(self.__class__.__name__, self.tuner_name) + + def _make_report(self): + """ + make a tune report. + Called by self.explain(), the report content will be writed into project-report. + :return: str + """ + assert False, 'Incomplete function: <{0} {1}>.calculate().'.format(self.__class__.__name__, self.tuner_name) + + def _make_alog(self): + """ + make an anti log + Called by self.explain(), the anti log content will be writed into AntiLog. + :return: str + """ + assert False, 'Incomplete function: <{0} {1}>.calculate().'.format(self.__class__.__name__, self.tuner_name) + + @staticmethod + def _parse_alog(alog): + """ + parse an anti log, get information for rollback. + Called by staticmethod rollback() + :param alog: alog str + :return: any data + """ + assert False, 'Incomplete function: <{0} {1}>.calculate().' + + @staticmethod + def rollback(alog): + """ + when exception or recover, rollback the tune point already be applied. + :param alog: + :return: + """ + assert False, 'Incomplete function: <{0} {1}>.calculate().' + + +class ShellTunePoint(Tuner): + """ + This is a common tune class for execute shell. + """ + + def __init__(self, cmd, anti, desc=''): + """ + :param cmd: sh command + :param anti: anti-command, to roll back the tune point + :param desc: description + """ + super(ShellTunePoint, self).__init__('command:' + cmd) + self.cmd = cmd + self.anti = anti + self.desc = desc + + def __str__(self): + return ' {0}: {1}'.format( + self.__class__.__name__, self._make_alog()) + + def calculate(self): + """ + ShellTunePoint does not need calculate. + :return: + """ + assert False + + def explain(self, apply=False): + """ + :return: + """ + Project.log('{0} {1}'.format('Apply' if apply else 'Explain', self.__str__())) + if apply: + AntiLog.write(self.__class__.__name__, self._make_alog()) + output = CmdUtil.execCmd(self.cmd) + Project.log('Output: ' + output) + + Project.report.record(self._make_report()) + + def _make_report(self): + """ + make a tune report + :return: str + """ + report = '**shell tune point** \n' + \ + '{}. \n'.format(self.desc) + \ + 'command: `{}`'.format(self.cmd) + return report + + def _make_alog(self): + """ + make an anti log. + Use json format to record the relevant content. + :return: json str + """ + alog = { + 'cmd': self.cmd, + 'anti': self.anti, + 'desc': self.desc + } + return json.dumps(alog) + + @staticmethod + def _parse_alog(alog): + """ + parse an anti log. + :return: dict + """ + return json.loads(alog) + + @staticmethod + def rollback(alog): + commands = ShellTunePoint._parse_alog(alog) + Project.notice('Rollback: ' + commands['anti']) + output = CmdUtil.execCmd(commands['anti']) + Project.log('Output: ' + output) + + +class TunerGroup(Tuner): + """ + Indicates a tune group. + + Tuner is usually used to adjust a single point, but adjustments for a class of modules + are often many points. Using a tuner group, you can manage multiple Tuners. The Tuners + in the same group belong to the same module or level. + + Tuner group can also manage Tuner group, to form a tree of tune logic. + + Tune group only plays an administrative role and does not carry out actual adjustments, + so it does not need to implement interfaces about alog, report, rollback and so on. + """ + def __init__(self, tuner_name=''): + super(TunerGroup, self).__init__(tuner_name) + self._sub_tuner_groups = [] + + def add(self, sub_tuner): + """ + register a tuner or tuner group. + :param sub_tuner: tuner or tuner group. + :return: NA + """ + self._sub_tuner_groups.append(sub_tuner) + return sub_tuner + + def calculate(self): + """ + Iterate sub tuner and calculate it in turn. + :return: NA + """ + for sub_tuner_group in self._sub_tuner_groups: + sub_tuner_group.calculate() + + def explain(self, apply=False): + """ + Iterate sub tuner and explain it in turn. + :return: NA + """ + for sub_tuner_group in self._sub_tuner_groups: + sub_tuner_group.explain(apply) + diff --git a/script/impl/perf_config/perf_probe.py b/script/impl/perf_config/perf_probe.py new file mode 100644 index 00000000..ca8b805d --- /dev/null +++ b/script/impl/perf_config/perf_probe.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : perf_probe.py setup a information set for configure +############################################################################# + + +from impl.perf_config.basic.probe import Probe, ProbeGroup +from impl.perf_config.probes.business import BusinessProbe +from impl.perf_config.probes.cpu import CPUInfo +from impl.perf_config.probes.db import DBInfo +from impl.perf_config.probes.disk import DiskInfo +from impl.perf_config.probes.memory import MemoryInfo +from impl.perf_config.probes.network import NetworkInfo +from impl.perf_config.probes.os import OSInfo +from impl.perf_config.probes.user import UserInfo + + +class PerfProbe(ProbeGroup): + def __init__(self): + super(PerfProbe, self).__init__() + self.user = self.add(UserInfo()) + self.cpu = self.add(CPUInfo()) + self.memory = self.add(MemoryInfo()) + self.disk = self.add(DiskInfo()) + self.network = self.add(NetworkInfo()) + self.os = self.add(OSInfo()) + self.db = self.add(DBInfo()) + self.business = self.add(BusinessProbe()) + diff --git a/script/impl/perf_config/perf_tuner.py b/script/impl/perf_config/perf_tuner.py new file mode 100644 index 00000000..f8ad7661 --- /dev/null +++ b/script/impl/perf_config/perf_tuner.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : perf_probe.py setup a information set for configure +############################################################################# + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.anti import AntiLog +from impl.perf_config.basic.tuner import Tuner, TunerGroup, ShellTunePoint +from impl.perf_config.tuners.os import OSTuner +from impl.perf_config.tuners.setup import SetupTuner +from impl.perf_config.tuners.guc import GucTuner + + +class PerfTuneTarget(object): + def __init__(self, argc, apply): + self._tune_target = [] + self._apply = apply + + all_module = ['os', 'setup', 'guc', 'suggest'] + argv = argc.split(',') + + for arg in argv: + if arg == 'all': + self._tune_target = all_module + elif arg == 'os': + if 'os' not in self._tune_target: + self._tune_target.append('os') + elif arg == 'setup': + if 'setup' not in self._tune_target: + self._tune_target.append('setup') + elif arg == 'guc': + if 'guc' not in self._tune_target: + self._tune_target.append('guc') + elif arg == 'suggest': + if 'suggest' not in self._tune_target: + self._tune_target.append('suggest') + else: + Project.fatal('unknown param {}.'.format(arg)) + exit(1) + + if not Project.haveRootPrivilege() and self.hasOS(): + Project.warning('no root privilege, ignore os.') + self._tune_target.remove('os') + + def apply(self): + return self._apply + + def noTarget(self): + return len(self._tune_target) == 0 + + def hasOS(self): + return 'os' in self._tune_target + + def hasSetUp(self): + return 'setup' in self._tune_target + + def hasGuc(self): + return 'guc' in self._tune_target + + def hasSuggest(self): + return 'suggest' in self._tune_target + + +class PerfTuner(TunerGroup): + def __init__(self): + super(PerfTuner, self).__init__() + tt = Project.getTask().tune_target + self.os = self.add(OSTuner()) if tt.hasOS() else None + self.setup = self.add(SetupTuner()) if tt.hasSetUp() else None + self.guc = self.add(GucTuner()) if tt.hasGuc() else None + + @staticmethod + def rollback(alog): + AntiLog.register(ShellTunePoint) + AntiLog.register(GucTuner) + + AntiLog.rollback() + diff --git a/script/impl/perf_config/preset/__init__.py b/script/impl/perf_config/preset/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/preset/default.json b/script/impl/perf_config/preset/default.json new file mode 100644 index 00000000..84989136 --- /dev/null +++ b/script/impl/perf_config/preset/default.json @@ -0,0 +1,10 @@ +{ + "desc": "default preset", + "scenario": "OLTP-performance", + "parallel": 600, + "rel_count": 300, + "rel_kind": "heap-table, partition-table", + "part_count": 200, + "index_count": 600, + "data_size": 8192 +} diff --git a/script/impl/perf_config/preset/kunpeng-4P-tpcc.json b/script/impl/perf_config/preset/kunpeng-4P-tpcc.json new file mode 100644 index 00000000..6d8212ed --- /dev/null +++ b/script/impl/perf_config/preset/kunpeng-4P-tpcc.json @@ -0,0 +1,10 @@ +{ + "desc": "kunpeng 4P tpcc", + "scenario": "OLTP-performance", + "parallel": 900, + "rel_count": 20, + "rel_kind": "heap-table, partition-table", + "part_count": 10, + "index_count": 10, + "data_size": 81920 +} diff --git a/script/impl/perf_config/preset/preset.py b/script/impl/perf_config/preset/preset.py new file mode 100644 index 00000000..c754f7d4 --- /dev/null +++ b/script/impl/perf_config/preset/preset.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : preset management. Including reading and resolving preset. +############################################################################# + +import os +import re +import json +from enum import Enum +from impl.perf_config.basic.project import Project + +""" +preset is a set of information about some business attributes. +By writing a preset file, the preset configuration can be directly read during +the tool running, thus skipping the stage of business investigation. + +preset contains Preset information about the service survey, and the configuration +parameters are related to the service survey. + +The preset file is actually a json file. This section describes how to set +configuration items in key-value pairs. +In json, you can write in strings. However, different parameters have different +parsing schemes, such as enumeration, ordinary string, etc., and some parsing rules +are needed to parse and determine whether the preset file is valid. + +The built-in preset is stored in the tool code directory. +Also, preset written by the user can be stored in the $GAUSSLOG/om/pg_perfconfig/preset. +The difference between the two is that built-in files are automatically replaced when +the database is upgrade. + +The preset name is a file name and needs to end with '.json'. +If files with the same name exist in two directories, the internal directory takes +precedence. +""" + +class PsOptionType(Enum): + INT = 'integer' + STR = 'string' + ENUM_VALUE = 'enum value' + ENUM_LIST = 'list of enum' + PATH = 'path' + +class PsOptionRule(object): + @staticmethod + def transformIntValue(value, check_range): + """ + transform an int value. + :param value: origin value. + :param check_range: function to check range. return T/F. + :return: NA + """ + if type(value) != int: + value = int(value) + + if check_range is not None and not check_range(value): + raise ValueError(f'value {value} not in range.') + + return value + + @staticmethod + def transformStrValue(value, check_range): + """ + transform a string value. + :param value: origin value. + :param check_range: function to check range. return T/F. + :return: NA + """ + if type(value) != str: + value = str(value) + + if check_range is not None and not check_range(value): + raise ValueError(f'value {value} not in range.') + + return value + + @staticmethod + def transformEnumValue(value, value_list): + """ + transform an enum value. + :param value: origin value. + :param value_list: enumrate list. + :return: NA + """ + if type(value) != str: + value = str(value) + + value = value.strip() + if value not in value_list: + raise ValueError(f'value {value} not in range.') + + return value + + @staticmethod + def transformEnumValueList(value, value_list): + """ + transform an enum value list. + :param value: origin value. The value can be a string separated + by commas or Spaces. It also could be a list. + :param value_list: enumrate list. + :return: NA + """ + opts = value if type(value) == list else re.split(r'[, ]+', str(value)) + res = set() + + for opt in opts: + if opt not in value_list: + raise ValueError(f'option {opt} not in range.') + res.add(opt) + + return list(res) + + @staticmethod + def transformPath(value, check_exist): + """ + transform a path. + :param value: origin value. + :param check_exist: Checks whether the path exists and is accessible. + :return: NA + """ + path = value if type(value) == str else str(value) + if check_exist and not os.path.access(path, os.F_OK): + raise ValueError(f'Could not access path Option path {path}.') + return path + + def __init__(self, desc, option_type, default, range_info): + self.desc = desc + self.option_type = option_type + self.default = default + self.range_info = range_info + + def regulate(self, value): + if value is None: + return self.default + try: + if self.option_type == PsOptionType.INT: + return PsOptionRule.transformIntValue(value, self.range_info) + if self.option_type == PsOptionType.STR: + return PsOptionRule.transformStrValue(value, self.range_info) + elif self.option_type == PsOptionType.ENUM_VALUE: + return PsOptionRule.transformEnumValue(value, self.range_info) + elif self.option_type == PsOptionType.ENUM_LIST: + return PsOptionRule.transformEnumValueList(value, self.range_info) + elif self.option_type == PsOptionType.PATH: + return PsOptionRule.transformPath(value, self.range_info) + else: + assert False + except ValueError as e: + Project.fatal('Preset error, ' + str(e)) + + +class Preset(object): + + # options list. + options = { + 'desc': PsOptionRule( + 'The description of preset', + PsOptionType.STR, + 'no description', + None + ), + 'scenario': PsOptionRule( + 'Business scenario.', + PsOptionType.ENUM_VALUE, + 'OLTP-performance', + ['OLTP-produce', 'OLTP-performance'] + ), + 'rel_count': PsOptionRule( + 'How many tables do you have?', + PsOptionType.INT, + 300, + lambda x:x > 0 + ), + 'index_count': PsOptionRule( + 'How many indexs do you have?', + PsOptionType.INT, + 600, + lambda x:x > 0 + ), + 'rel_kind': PsOptionRule( + 'What kind of table do you used?', + PsOptionType.ENUM_LIST, + ['heap-table', 'partition-table'], + ['heap-table', 'partition-table', 'column-table', 'column-partition-table'] + ), + 'part_count': PsOptionRule( + 'How many partitions do you have?', + PsOptionType.INT, + 200, + lambda x:x > 0 + ), + 'data_size': PsOptionRule( + 'How much data is there? unit is MB.', + PsOptionType.INT, + 8192, + lambda x:x > 0 + ), + 'parallel': PsOptionRule( + 'How much concurrency is there?', + PsOptionType.INT, + 400, + lambda x:x > 0 + ), + 'isolated_xlog': PsOptionRule( + 'Storing wal on a separate disk.', + PsOptionType.PATH, + None, + True + ) + } + + @staticmethod + def usage(): + """ + how to write preset. + """ + res = 'The preset configure is a file in JSON format that contains the following parameters:\n\n' + for opt in Preset.options: + res += f'Name: {opt}\n' + res += f' Description: {Preset.options[opt].desc}\n' + res += f' Type: {Preset.options[opt].option_type.value}\n' + if Preset.options[opt].option_type in [PsOptionType.ENUM_VALUE, PsOptionType.ENUM_LIST]: + res += f' Range: {str(Preset.options[opt].range_info)}\n' + res += '\n' + return res + + @staticmethod + def get_preset_dir(): + dir1 = os.path.join(Project.environ.workspace1, 'preset') + dir2 = os.path.join(Project.environ.workspace2, 'preset') + return dir1, dir2 + + @staticmethod + def get_all_presets(): + def _read_preset_dir(_dir): + """ + read all '.json' file in _dir. + """ + if _dir is None: + return [] + + if not os.access(_dir, os.F_OK): + Project.warning('Could not access ' + _dir) + return [] + + files = os.listdir(_dir) + res = [] + for file in files: + if file.endswith('.json'): + res.append(file[:-5]) + return res + + dir1, dir2 = Preset.get_preset_dir() + + builtins = _read_preset_dir(dir1) + tmp_usersets = _read_preset_dir(dir2) + # when duplicate name, builtin first. + usersets = [x for x in tmp_usersets if x not in builtins] + + return builtins, usersets + + def __init__(self, preset_name): + builtins, usersets = Preset.get_all_presets() + dir1, dir2 = Preset.get_preset_dir() + file = '' + + if preset_name in builtins: + file = os.path.join(dir1, f'{preset_name}.json') + elif preset_name in usersets: + file = os.path.join(dir2, f'{preset_name}.json') + else: + Project.fatal('Could not find preset: ' + preset_name) + + self.name = preset_name + with open(file, 'r') as f: + config = json.load(f) + for key in config: + if not self.options.__contains__(key): + Project.warning(f'skip unknown option {key} in preset.') + + self.content = {} + for key in self.options: + val = config.get(key) + self.content[key] = self.options[key].regulate(val) + + def __str__(self): + res = f'Preset name: {self.name}\n' + desc = self.content.get('desc') + res += ' {}\n'.format(desc if desc is not None else 'no description') + res += 'Detail:\n' + for k in self.content: + if k == 'desc': + continue + res += ' {0}: {1}\n'.format(k, self.content[k]) + + return res + + def __getitem__(self, item): + return self.content.get(item) diff --git a/script/impl/perf_config/probes/__init__.py b/script/impl/perf_config/probes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/probes/business.py b/script/impl/perf_config/probes/business.py new file mode 100644 index 00000000..9fb44ae2 --- /dev/null +++ b/script/impl/perf_config/probes/business.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : perf_probe.py setup a information set for configure +############################################################################# + +import os +from enum import Enum +from base_utils.common.dialog import DialogUtil +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.probe import Probe +from impl.perf_config.preset.preset import Preset + +""" +The business module is mainly used to investigate the user's business related information. + +The business content will be investigated in the form of questions and answers. + +You can also select a preset and read the configuration in it. +""" + +class BsScenario(Enum): + TP_PRODUCE = 0 + TP_PERFORMANCE = 1 + AP = 2 # just beta or demo + + @staticmethod + def isOLTPScenario(scenario): + return scenario in [BsScenario.TP_PERFORMANCE, BsScenario.TP_PRODUCE] + + +class TblKind(Enum): + COMMON_TBL = 0 # common heap table + PARTITION_TBL = 1 # common heap partition table + COLUMN_TBL = 2 # column table + PART_COLUMN_TBL = 3 # column partition table + + @staticmethod + def isCommonTbl(kind): + return kind in [TblKind.COMMON_TBL] + + @staticmethod + def isPartTbl(kind): + return kind in [TblKind.PARTITION_TBL, TblKind.PART_COLUMN_TBL] + + @staticmethod + def havePartTbl(kinds): + for kind in kinds: + if TblKind.isPartTbl(kind): + return True + return False + + @staticmethod + def isColumnTbl(kind): + return kind in [TblKind.COLUMN_TBL, TblKind.PART_COLUMN_TBL] + + +class BusinessProbe(Probe): + def __init__(self): + super(BusinessProbe, self).__init__() + self._preset = None + + self.scenario = BsScenario.TP_PRODUCE + self.parallel = 200 + self.rel_count = 50 + self.rel_kind = [TblKind.COMMON_TBL, TblKind.PARTITION_TBL] + self.part_count = 100 + self.index_count = 100 + self.data_size = 200 * 1024 # unit is MB + self.isolated_xlog = None + + def __str__(self): + return str({ + '_preset': self._preset, + 'scenario': self.scenario, + 'parallel': self.parallel, + 'rel_count': self.rel_count, + 'rel_kind': self.rel_kind, + 'part_count': self.part_count, + 'index_count': self.index_count, + 'data_size': self.data_size, + 'isolated_xlog': self.isolated_xlog + }) + + def relfilenode_count(self): + """ + Estimate the number of relfilenodes. + """ + rel_count = self.rel_count if self.rel_count is not None else 0 + part_count = self.part_count if self.part_count is not None else 0 + index_count = self.index_count if self.index_count is not None else 0 + return (rel_count + part_count + index_count) * 4 + + def detect(self): + """ + detect the user business by some research questions. + """ + msg = 'Now we need to do some research to understand your business scenario.\n' \ + 'Fields marked with "*" are required.' + Project.msg(msg) + + question = 'What kind of way to choose？' + options = [ + 'Default case', + 'Preset', + 'Customization' + ] + check = DialogUtil.singleAnswerQuestion(question, options) + Project.log('user choose: ' + options[check]) + + if check == 0: + self._load_preset('default') + elif check == 1: + self._load_preset() + + self._do_detect() + + Project.log('business detect res:' + self.__str__()) + + def _load_preset(self, preset_name=None): + if preset_name is None: + question = 'Please select the desired preset.' + builtins, usersets = Preset.get_all_presets() + presets = builtins + usersets + + check = DialogUtil.singleAnswerQuestion(question, presets) + preset_name = presets[check] + + Project.log('business detect research\nquestion: {0}\nanwser: {1}'.format(question, preset_name)) + + self._preset = Preset(preset_name) + + def _do_detect(self): + """ + do detect action. If we had load preset, just read it, otherwise do research. + + Pay attention to the order of detection, some different detection items are + associated with each other. For example: + - whether a partition table is used? + - how many partitions you have. + If you are not using a partition table, you do not need to survey the + number of partitions. + """ + + self._detect_scenario() + self._detect_parallel() + self._detect_rel_count() + self._detect_rel_kind() + + if TblKind.havePartTbl(self.rel_kind): + self._detect_partition_count() + else: + self.part_count = 0 + + self._detect_index_count() + self._detect_data_size() + self._detect_isolated_xlog() + + ############### + # Below are survey questions or preset-load-functions for each option. + ############### + # load or research scenario + def _load_scenario(self): + scenario = self._preset['scenario'] + if scenario is None: + pass + elif scenario == 'OLTP-produce': + self.scenario = BsScenario.TP_PRODUCE + elif scenario == 'OLTP-performance': + self.scenario = BsScenario.TP_PERFORMANCE + else: + assert False + + def _detect_scenario(self): + if self._preset is not None: + self._load_scenario() + return + + question = 'What are the main scenarios for using databases?' + options = [ + 'OLTP performance first', + 'OLTP produce first', + ] + answer = [ + BsScenario.TP_PERFORMANCE, + BsScenario.TP_PRODUCE + ] + check = DialogUtil.singleAnswerQuestion(question, options) + Project.log('business detect research\nquestion:{0}\nanwser:{1}'.format(question, options[check])) + self.scenario = answer[check] + + # load or research parallel + def _load_parallel(self): + parallel = self._preset['parallel'] + self.parallel = parallel if parallel is not None else self.parallel + + def _detect_parallel(self): + if self._preset is not None: + self._load_parallel() + return + + question = 'What is the average number of concurrent transactions?' + num = DialogUtil.askANumber(question, lambda x:'Invalid number, please more than 0.' if x < 0 else None) + Project.log('business detect research\nquestion:{0}\nanwser:{1}'.format(question, num)) + self.parallel = num + + # load or research rel_count + def _load_rel_count(self): + rel_count = self._preset['rel_count'] + self.rel_count = rel_count if rel_count is not None else self.rel_count + + def _detect_rel_count(self): + if self._preset is not None: + self._load_rel_count() + return + + question = 'Approximately how many tables you have?' + num = DialogUtil.askANumber(question, lambda x:'Invalid number, please more than 0.' if x < 0 else None) + Project.log('business detect research\nquestion:{0}\nanwser:{1}'.format(question, num)) + self.rel_count = num + + # load or research rel_kind + def _load_rel_kind(self): + rel_kind = self._preset['rel_kind'] + if rel_kind is None: + pass + else: + options = ['heap-table', 'partition-table', 'column-table', 'column-partition-table'] + checks = [False, False, False, False] + res = [TblKind.COMMON_TBL, TblKind.PARTITION_TBL, TblKind.COLUMN_TBL, TblKind.PART_COLUMN_TBL] + for kind in rel_kind: + assert kind in options + checks[options.index(kind)] = True + self.rel_kind = [res[i] for i in range(0,4) if checks[i]] + + def _detect_rel_kind(self): + if self._preset is not None: + self._load_rel_kind() + return + + question = 'What kind of table you used?' + options = [ + 'common heap table', + 'partition heap table', + 'column table', + 'partition column table' + ] + answer = [ + TblKind.COMMON_TBL, + TblKind.PARTITION_TBL, + TblKind.COLUMN_TBL, + TblKind.PART_COLUMN_TBL + ] + + checks = DialogUtil.multipleAnswerQuestion(question, options) + Project.log('business detect research\nquestion:{0}\nanwser:{1}'.format( + question, + str([options[check] for check in checks])) + ) + self.rel_kind = [answer[check] for check in checks] + + # load or research part_count + def _load_part_count(self): + part_count = self._preset['part_count'] + self.part_count = part_count if part_count is not None else self.part_count + + def _detect_partition_count(self): + if self._preset is not None: + self._load_part_count() + return + + question = 'Approximately how many partitions you have?' + num = DialogUtil.askANumber(question, lambda x:'Invalid number, please more than 0.' if x < 0 else None) + Project.log('business detect research\nquestion:{0}\nanwser:{1}'.format(question, num)) + self.part_count = num + + # load or research index_count + def _load_index_count(self): + index_count = self._preset['index_count'] + self.index_count = index_count if index_count is not None else self.index_count + + def _detect_index_count(self): + if self._preset is not None: + self._load_index_count() + return + + question = 'Approximately how many index you have?' + num = DialogUtil.askANumber(question, lambda x:'Invalid number, please more than 0.' if x < 0 else None) + Project.log('business detect research\nquestion:{0}\nanwser:{1}'.format(question, num)) + self.index_count = num + + def _load_data_size(self): + data_size = self._preset['data_size'] + self.data_size = data_size if data_size is not None else self.data_size + + def _detect_data_size(self): + if self._preset is not None: + self._load_data_size() + return + + question = 'How much data is there, unit by MB?' + num = DialogUtil.askANumber(question, lambda x:'Invalid number, please more than 0.' if x < 0 else None) + Project.log('business detect research\nquestion:{0}\nanwser:{1}'.format(question, num)) + self.data_size = num + + def _load_isolated_xlog(self): + isolated_xlog = self._preset['isolated_xlog'] + if isolated_xlog is None: + self.isolated_xlog = None + return + + if os.path.isdir(isolated_xlog) or not os.access(isolated_xlog): + Project.warning('Could not access ' + isolated_xlog) + self.isolated_xlog = None + return + + self.isolated_xlog = isolated_xlog + + def _detect_isolated_xlog(self): + if self._preset is not None: + self._load_isolated_xlog() + return + + question = 'Storing wal on a separate disk helps improve performance. Do you need to move them?\n' \ + 'Here is some disk information:\n' + infos = Project.getGlobalPerfProbe() + for device in infos.disk: + question += f' {device.simple_info()}\n' + + path = DialogUtil.askAPath(question, check_access=True, required=False) + self.isolated_xlog = path + + diff --git a/script/impl/perf_config/probes/cpu.py b/script/impl/perf_config/probes/cpu.py new file mode 100644 index 00000000..4d6f95d0 --- /dev/null +++ b/script/impl/perf_config/probes/cpu.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : perf_probe.py setup a information set for configure +############################################################################# + + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.probe import Probe +from base_utils.os.cpu_util import CpuArchitecture, CpuUtil +from base_utils.os.dmidecode_util import DmidecodeUtil, DMITypeCategory + + +class CPUInfo(Probe): + def __init__(self): + super(CPUInfo, self).__init__() + self.architecture = lambda : CpuUtil.getCpuArchitecture() + self.vendor = lambda : CpuUtil.getCpuVendor() + self.count = lambda : CpuUtil.getCpuNum() + self.numa = lambda : CpuUtil.getCpuNumaList() + + self.dmi_processor = None + self.dmi_cache = None + + def detect(self): + if Project.haveRootPrivilege(): + self.dmi_processor = DmidecodeUtil.getDmidecodeTableByType(DMITypeCategory.PROCESSOR) + self.dmi_cache = DmidecodeUtil.getDmidecodeTableByType(DMITypeCategory.CACHE) + diff --git a/script/impl/perf_config/probes/db.py b/script/impl/perf_config/probes/db.py new file mode 100644 index 00000000..0fa0d2cb --- /dev/null +++ b/script/impl/perf_config/probes/db.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +import os +import pwd +from enum import Enum +from base_utils.os.cmd_util import CmdUtil +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.probe import Probe + + +class DBSeverMode(Enum): + PRIMARY = 'primary' + STANDBY = 'standby' + + +class DBInfo(Probe): + def __init__(self): + super(DBInfo, self).__init__() + self.ip = None + self.port = None + self.omm = None + self.omm_uid = None + self.omm_gid = None + self.gauss_home = Project.environ.gauss_home + self.gauss_data = Project.environ.gauss_data + self.postgresql_conf = os.path.join(self.gauss_data, 'postgresql.conf') + + def detect(self): + self.gauss_home = Project.environ.gauss_home + self.gauss_data = Project.environ.gauss_data + + self._detect_omm() + self._detect_ip_port() + + def _detect_omm(self): + stat = os.stat(self.gauss_home) + self.omm_uid = stat.st_uid + self.omm_gid = stat.st_gid + self.omm = pwd.getpwuid(stat.st_uid).pw_name + + def _detect_ip_port(self): + listen_addresses = self._read_guc_in_postgresql_conf('listen_addresses') + if listen_addresses is None: + listen_addresses = '*' + Project.log(f'detect database listen_addresses: {listen_addresses}') + self.ip = [ip.strip() for ip in listen_addresses.split(',')] + + port = self._read_guc_in_postgresql_conf('port') + if port is None: + port = 5432 + Project.log(f'detect database port: {port}') + self.port = port + + def _read_guc_in_postgresql_conf(self, guc): + cmd = f'grep "{guc}" {self.postgresql_conf} -i' + output = CmdUtil.execCmd(cmd, noexcept=True) + if output == '': + return + res = None + lines = output.split('\n') + for line in lines: + if line.strip().startswith('#'): + continue + if not line.lower().strip().startswith(guc.lower()): + continue + val = line.split('=')[1].strip() + val = val.split('#')[0].strip() + if val.startswith("'") or val.startswith('"'): + val = val[1:-1] + res = val + return res + + + diff --git a/script/impl/perf_config/probes/disk.py b/script/impl/perf_config/probes/disk.py new file mode 100644 index 00000000..736d0b18 --- /dev/null +++ b/script/impl/perf_config/probes/disk.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : perf_probe.py setup a information set for configure +############################################################################# + +import os +import re +import psutil +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.probe import Probe +from base_utils.os.disk_util import DiskUtil +from base_utils.os.cmd_util import CmdUtil + + +class DiskDeviceInfo(Probe): + def __init__(self, partition): + super(DiskDeviceInfo, self).__init__() + self._partition = partition + self.device = self._partition.device + self.mountpoint = self._partition.mountpoint + self.fstype = self._partition.fstype + self.total_size = lambda : DiskUtil.getTotalSize(self.device) + self.avail_size = lambda : DiskUtil.getAvailSize(self.device) + self.vendor = None + self.r_speed = None + self.w_speed = None + self.opts = {} + + def detect(self): + for opt in self._partition.opts.split(','): + kv = opt.split('=') + if len(kv) == 1: + self.opts[kv[0]] = True + else: + assert len(kv) == 2 + self.opts[kv[0]] = kv[1] + + self._detect_io_speed() + + def simple_info(self): + return 'device={0}, mountpoint={1}, fstype={2} size=(free {3}GB / total {4}GB)'.format( + self.device, self.mountpoint, self.fstype, self.avail_size() / 1024, self.total_size() / 1024 + ) + + def _detect_io_speed(self): + self.r_speed = None + self.w_speed = None + + if not CmdUtil.doesBinExist('fio'): + Project.log('There is no fio.') + return + + fio_file = os.path.join(self.mountpoint, '/gs_perfconfig_fio_test.fiofile') + cmd = f'fio -filename={fio_file} -re=write -size=500M -direct=1 -ioengine=sync | grep WRITE' + try: + Project.log('detect io speed by fio: ' + cmd) + output = CmdUtil.execCmd(cmd) + + CmdUtil.execCmd(f'rm {fio_file} -fr') + Project.log('remove fio file: ' + fio_file) + + nums = re.findall(r'\d+', output) + if len(nums) < 1: + return + self.w_speed = int(nums[0]) + + except Exception: + Project.log('remove fio file: ' + fio_file) + CmdUtil.execCmd(f'rm {fio_file} -fr') + pass + +class DiskInfo(Probe): + def __init__(self): + super(DiskInfo, self).__init__() + self._devices = [] + self._index = 0 + + def get(self, item): + if isinstance(item, int): + if item > len(self._devices): + return + return self._devices[item] + elif isinstance(item, str): + for device in self._devices: + if device.device == item or device.mountpoint == item: + return device + return + else: + return + + def __iter__(self): + self._index = 0 + return self + + def __next__(self): + if self._index >= len(self._devices): + raise StopIteration + value = self._devices[self._index] + self._index += 1 + return value + + def __len__(self): + return len(self._devices) + + def detect(self): + self._devices = [] + partitions = DiskUtil.getMountInfo() + + for partition in partitions: + device = DiskDeviceInfo(partition) + device.detect() + self._devices.append(device) diff --git a/script/impl/perf_config/probes/memory.py b/script/impl/perf_config/probes/memory.py new file mode 100644 index 00000000..17e65eec --- /dev/null +++ b/script/impl/perf_config/probes/memory.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : A probe for memory-related information. +############################################################################# + +import re + +from base_utils.os.cmd_util import CmdUtil +from base_utils.os.memory_util import MemoryUtil +from base_utils.os.dmidecode_util import DmidecodeUtil, DMIType, DMITypeCategory + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.probe import Probe + + +class MemoryInfo(Probe): + def __init__(self): + super(MemoryInfo, self).__init__() + self.total_size = lambda : MemoryUtil.getPhysicalMemTotalSize() + self.avail_size = lambda : MemoryUtil.getMemAvailableSize() + self.page_size = lambda : CmdUtil.execCmd('getconf PAGE_SIZE') + + self.hugepage = None + + self.dmi_physical_mem_array = None + self.dmi_all_devices = None + self.dmi_use_devices = None + + def detect(self): + self._detect_dmidecode() + self._detect_hugepage() + + def _detect_dmidecode(self): + if not Project.haveRootPrivilege(): + return + self.dmi_physical_mem_array = DmidecodeUtil.getDmidecodeTableByType(DMIType.PHYSICAL_MEMORY_ARRAY) + self.dmi_all_devices = DmidecodeUtil.getDmidecodeTableByType(DMIType.MEMORY_DEVICE) + self.dmi_use_devices = [] + for device in self.dmi_all_devices: + if device['Type'] != 'Unknown': + self.dmi_use_devices.append(device) + + def _detect_hugepage(self): + self.hugepage = {'enabled': '', 'defrag': ''} + for key in self.hugepage: + cmd = f'cat /sys/kernel/mm/transparent_hugepage/{key}' + output = CmdUtil.execCmd(cmd) + self.hugepage[key] = re.findall('\[(.*?)\]', output)[0] + diff --git a/script/impl/perf_config/probes/network.py b/script/impl/perf_config/probes/network.py new file mode 100644 index 00000000..1de4a7eb --- /dev/null +++ b/script/impl/perf_config/probes/network.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : perf_probe.py setup a information set for configure +############################################################################# + +import os +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.probe import Probe, ProbeGroup +from base_utils.os.cmd_util import CmdUtil + + +class NetworkGateInfo(Probe): + def __init__(self, ifconfig): + super(NetworkGateInfo, self).__init__() + self._ifconfig = ifconfig + self.name = None + self.ipv4 = None + self.ipv6 = None + self.mac = None + + self.irq_binds = None + self.combined = None + self.tso = None + self.lro = None + self.gro = None + self.gso = None + + def detect(self): + self._parse_ifconfig() + + if self.is_virbr() or self.is_localhost(): + return + + self._detect_irq_binds() + self._detect_combined() + self._detect_ethtool_k() + + def _parse_ifconfig(self): + lines = self._ifconfig.split('\n') + self.name = lines[0].split(':')[0].strip() + for line in lines: + line = line.strip() + if line.startswith('inet6'): + self.ipv6 = line.split(' ')[1] + elif line.startswith('inet'): + self.ipv4 = line.split(' ')[1] + self.netmask = line.split(' ')[3] + elif line.startswith('ether'): + self.mac = line.split(' ')[1] + + def is_localhost(self): + return self.name == 'lo' + + def is_virbr(self): + return self.name.startswith('virbr') + + def _detect_irq_binds(self): + script = Project.environ.get_builtin_script('irq_operate.sh') + cmd = f'sh {script} check {self.name}' + output = CmdUtil.execCmd(cmd) + lines = output.split('\n') + self.irq_binds = [int(num) for num in lines[2:]] + + def _detect_combined(self): + cmd = f'ethtool -l {self.name} | grep Combined' + output = CmdUtil.execCmd(cmd) + parts = output.split() + self.combined = {'maximums': int(parts[1]), + 'current': int(parts[3])} + + def _detect_ethtool_k(self): + cmd = f'ethtool -k {self.name}' + output = CmdUtil.execCmd(cmd) + opt_map = {} + for line in output.split('\n'): + kv = line.split(':') + opt_map[kv[0]] = kv[1].strip() + + self.tso = opt_map.get('tcp-segmentation-offload') + self.lro = opt_map.get('large-receive-offload') + + +class NetworkInfo(ProbeGroup): + def __init__(self): + super(NetworkInfo, self).__init__() + self._gates = [] + + def detect(self): + ifconfig = CmdUtil.execCmd('ifconfig') + lines = ifconfig.split('\n') + assert lines[-1].strip() == '' + s = 0 + for i, line in enumerate(lines): + if line.strip() == '': + device = NetworkGateInfo('\n'.join(lines[s:i])) + self._gates.append(device) + self.add(device) + s = i + 1 + + super(NetworkInfo, self).detect() + + def get_gate(self, argc): + """ + get device in device list + :param argc: device name or ipv4 or mac + :return: + """ + if argc == 'localhost': + argc = '127.0.0.1' + + for gate in self._gates: + if gate.name == argc or gate.ipv4 == argc or gate.mac == argc: + return gate + return diff --git a/script/impl/perf_config/probes/os.py b/script/impl/perf_config/probes/os.py new file mode 100644 index 00000000..b36dbc05 --- /dev/null +++ b/script/impl/perf_config/probes/os.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +from base_utils.os.cmd_util import CmdUtil + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.probe import Probe, ProbeGroup + + +class BiosInfo(Probe): + def __init__(self): + super(BiosInfo).__init__() + self.support_smmu = None + self.cpu_prefetching = None + self.die_interleaving = None + + def detect(self): + # We can't detect it yet, so we're assuming it's all open. + self.support_smmu = 'Enable' + self.cpu_prefetching = 'Enable' + self.die_interleaving = 'Enable' + + +class OSBaseInfo(Probe): + def __init__(self): + super(OSBaseInfo).__init__() + self.is_virtual = False + + def detect(self): + pass + + +class OSServiceInfo(Probe): + def __init__(self): + super(OSServiceInfo).__init__() + self.sysmonitor = None + self.irqbalance = None + + @staticmethod + def is_running(service): + cmd = f'systemctl status {service} | grep "Active:"' + try: + output = CmdUtil.execCmd(cmd) + if output.find('(running)') >= 0: + return True + return False + except Exception: + pass + + def detect(self): + self.sysmonitor = self.is_running('sysmonitor') + self.irqbalance = self.is_running('irqbalance') + + +class OSInfo(ProbeGroup): + def __init__(self): + super(OSInfo, self).__init__() + self.bios = self.add(BiosInfo()) + self.base = self.add(OSBaseInfo()) + self.service = self.add(OSServiceInfo()) diff --git a/script/impl/perf_config/probes/user.py b/script/impl/perf_config/probes/user.py new file mode 100644 index 00000000..f60a6ae3 --- /dev/null +++ b/script/impl/perf_config/probes/user.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +import os +import getpass +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.probe import Probe + + +class UserInfo(Probe): + """ + Information about omm + """ + def __init__(self): + self.name = None + self.uid = None + self.gid = None + + def detect(self): + self.name = Project.role.user_name + self.uid = Project.role.user_uid + self.gid = Project.role.user_gid diff --git a/script/impl/perf_config/scripts/__init__.py b/script/impl/perf_config/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/scripts/irq_operate.sh b/script/impl/perf_config/scripts/irq_operate.sh new file mode 100644 index 00000000..2748f044 --- /dev/null +++ b/script/impl/perf_config/scripts/irq_operate.sh @@ -0,0 +1,56 @@ +function helper +{ +echo ' +RUN LIKE: + sh irq_operate.sh bind enp1s1 "1 2 3 4 5 6" + sh irq_operate.sh check enp1s1 +' +} + +action=$1 +intf=$2 +cpu_qrrqy_irq=($3) + +function bind_irq +{ + cpunum=${#cpu_qrrqy_irq[*]} + + ethtool -L ${intf} combined $cpunum + + irq_list=`cat /proc/interrupts | grep $intf | awk {'print $1'} | tr -d ":"` + irq_array_net=($irq_list) + + for (( i=0;i<$cpunum;i++ )) + do + echo "${cpu_array_irq[$i]}" > /proc/irq/${irq_array_net[$i]}/smp_affinity_list + done + + for j in ${irq_array_net[@]} + do + cat /proc/irq/$j/smp_affinity_list + done +} + + +function check_irq +{ + rx_irq_list=(`cat /proc/interrupts | grep ${intf} | awk -F':' '{print $1}'`) + + echo "check irf of net interface ${intf}" + + echo "rx" + for rx_irq in ${rx_irq_list[@]} + do + echo `cat /proc/irq/$rx_irq/smp_affinity_list` + done +} + + +if [ "$action" = "bind" ]; then + bind_irq +elif [ "$action" = "check" ]; then + check_irq +else + helper + exit 0 +fi diff --git a/script/impl/perf_config/scripts/isolated_xlog.sh b/script/impl/perf_config/scripts/isolated_xlog.sh new file mode 100644 index 00000000..4cbdab32 --- /dev/null +++ b/script/impl/perf_config/scripts/isolated_xlog.sh @@ -0,0 +1,23 @@ +action=$1 +xlog_old=$2 +xlog_new=$3 + + +function isolated +{ + mv $xlog_old $xlog_new # mv not change owner and mod +} + +function recover +{ + mv $xlog_new $xlog_old # mv not change owner and mod +} + +if [ "$action" = "isolated" ]; then + isolated +elif [ "$action" = "recover" ]; then + recover +else + echo 'unknown action', $action + exit 1 +fi \ No newline at end of file diff --git a/script/impl/perf_config/test/__init__.py b/script/impl/perf_config/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/test/data/__init__.py b/script/impl/perf_config/test/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/test/data/test-preset-wrong-format.json b/script/impl/perf_config/test/data/test-preset-wrong-format.json new file mode 100644 index 00000000..8adc8c30 --- /dev/null +++ b/script/impl/perf_config/test/data/test-preset-wrong-format.json @@ -0,0 +1,10 @@ + + "desc": "a err preset", + "scenario": "OLTP-performance", + "parallel": 600, + "rel_count": 300, + "rel_kind": "heap-table, partition-table", + "part_count": 200, + "index_count": 1200, + "data_size": 8192 +} diff --git a/script/impl/perf_config/test/data/test-preset-wrong-format2.conf b/script/impl/perf_config/test/data/test-preset-wrong-format2.conf new file mode 100644 index 00000000..8adc8c30 --- /dev/null +++ b/script/impl/perf_config/test/data/test-preset-wrong-format2.conf @@ -0,0 +1,10 @@ + + "desc": "a err preset", + "scenario": "OLTP-performance", + "parallel": 600, + "rel_count": 300, + "rel_kind": "heap-table, partition-table", + "part_count": 200, + "index_count": 1200, + "data_size": 8192 +} diff --git a/script/impl/perf_config/test/data/test-preset-wrong-param.json b/script/impl/perf_config/test/data/test-preset-wrong-param.json new file mode 100644 index 00000000..7087187c --- /dev/null +++ b/script/impl/perf_config/test/data/test-preset-wrong-param.json @@ -0,0 +1,10 @@ +{ + "desc": "a err preset", + "scenario": "OLTP-performance", + "paralle": 600, + "rel_count": 300, + "rel_kind": "heap-table, partition-table", + "part_count": 200, + "index_count": 1200, + "data_size": 8192 +} diff --git a/script/impl/perf_config/test/data/test-preset-wrong-val.json b/script/impl/perf_config/test/data/test-preset-wrong-val.json new file mode 100644 index 00000000..cf473062 --- /dev/null +++ b/script/impl/perf_config/test/data/test-preset-wrong-val.json @@ -0,0 +1,10 @@ +{ + "desc": "a err preset", + "scenario": "xxxxxxxx", + "parallel": 600, + "rel_count": 300, + "rel_kind": "heap-table, partition-table", + "part_count": 200, + "index_count": 1200, + "data_size": 8192 +} diff --git a/script/impl/perf_config/test/data/test-preset-wrong-val2.json b/script/impl/perf_config/test/data/test-preset-wrong-val2.json new file mode 100644 index 00000000..5d0989ac --- /dev/null +++ b/script/impl/perf_config/test/data/test-preset-wrong-val2.json @@ -0,0 +1,10 @@ +{ + "desc": "a err preset", + "scenario": "OLTP-performance", + "parallel": 600, + "rel_count": 300, + "rel_kind": "heap-table, part-table", + "part_count": 200, + "index_count": 1200, + "data_size": 8192 +} diff --git a/script/impl/perf_config/test/data/test-preset-wrong-val3.json b/script/impl/perf_config/test/data/test-preset-wrong-val3.json new file mode 100644 index 00000000..ec62e2f4 --- /dev/null +++ b/script/impl/perf_config/test/data/test-preset-wrong-val3.json @@ -0,0 +1,10 @@ +{ + "desc": "a err preset", + "scenario": "OLTP-performance", + "parallel": 600, + "rel_count": 300, + "rel_kind": "heap-table, partition-table", + "part_count": 200, + "index_count": 1200, + "data_size": -1 +} diff --git a/script/impl/perf_config/test/test_always_ok.sh b/script/impl/perf_config/test/test_always_ok.sh new file mode 100644 index 00000000..bfa6d498 --- /dev/null +++ b/script/impl/perf_config/test/test_always_ok.sh @@ -0,0 +1,33 @@ + +# user +export GS_PEPRFCONFIG_OPTIONS='always_choose_yes=on' +python3 gs_perfconfig tune --apply +# a + +export GS_PEPRFCONFIG_OPTIONS='always_choose_yes=off' +python3 gs_perfconfig tune --apply +# x x n + +export GS_PEPRFCONFIG_OPTIONS='always_choose_yes=0' +python3 gs_perfconfig tune --apply +# x x y a + +export GS_PEPRFCONFIG_OPTIONS='always_choose_yes=1' +python3 gs_perfconfig tune --apply +# a + +export GS_PEPRFCONFIG_OPTIONS='always_choose_yes=true' +python3 gs_perfconfig tune --apply +# a + +export GS_PEPRFCONFIG_OPTIONS='always_choose_yes=false' +python3 gs_perfconfig tune --apply +# x x y a + +export GS_PEPRFCONFIG_OPTIONS='always_choose_yes=xxx' +python3 gs_perfconfig tune --apply +# x x y a + +export GS_PEPRFCONFIG_OPTIONS='always_choose_yes=ON' +python3 gs_perfconfig tune --apply +# x x y a \ No newline at end of file diff --git a/script/impl/perf_config/test/test_common.sh b/script/impl/perf_config/test/test_common.sh new file mode 100644 index 00000000..8d2f6c87 --- /dev/null +++ b/script/impl/perf_config/test/test_common.sh @@ -0,0 +1,24 @@ + +# ok +python3 gs_perfconfig help +python3 gs_perfconfig --help +python3 gs_perfconfig -h +python3 gs_perfconfig -? + +python3 gs_perfconfig Help +python3 gs_perfconfig --hElp +python3 gs_perfconfig -H +python3 gs_perfconfig -? + +# falied +python3 gs_perfconfig xxx +python3 gs_perfconfig --xxx +python3 gs_perfconfig -x +python3 gs_perfconfig -. + + +# will ignode other param +python3 gs_perfconfig help ddd +python3 gs_perfconfig --help fd +python3 gs_perfconfig -h da +python3 gs_perfconfig -? das \ No newline at end of file diff --git a/script/impl/perf_config/test/test_preset.sh b/script/impl/perf_config/test/test_preset.sh new file mode 100644 index 00000000..08a819cd --- /dev/null +++ b/script/impl/perf_config/test/test_preset.sh @@ -0,0 +1,56 @@ +export GAUSSHOME='' +export GAUSSLOG='' +export PGDATA='' + + +# ok +python3 gs_perfconfig preset +python3 gs_perfconfig preset help +python3 gs_perfconfig preset --help +python3 gs_perfconfig preset -h +python3 gs_perfconfig preset -? +python3 gs_perfconfig preset HELP +python3 gs_perfconfig preset --Help +python3 gs_perfconfig preset -H +python3 gs_perfconfig preset -? +python3 gs_perfconfig preset default + + +# failed +python3 gs_perfconfig preSet +python3 gs_perfconfig preSet help +python3 gs_perfconfig preSet --help +python3 gs_perfconfig preSet -h +python3 gs_perfconfig preSet -? +python3 gs_perfconfig preSet HELP +python3 gs_perfconfig preSet --Help +python3 gs_perfconfig preSet -H +python3 gs_perfconfig preSet -? +python3 gs_perfconfig preset Default # failed + + +# ok +cp ../preset/default.json ../preset/xx1.json +python3 gs_perfconfig preset +python3 gs_perfconfig preset xx1 + +# xx1 = default +cp ../preset/kunpeng-4P-tpcc.json $GAUSSLOG/om/perf_config/preset/xx1.json +python3 gs_perfconfig preset +python3 gs_perfconfig preset xx1 + +# ok +cp ../preset/kunpeng-4P-tpcc.json $GAUSSLOG/om/perf_config/preset/xx2.json +python3 gs_perfconfig preset +python3 gs_perfconfig preset xx2 + + +cp data/test-preset* $GAUSSLOG/om/perf_config/preset/ +python3 gs_perfconfig preset +python3 gs_perfconfig preset test-preset-wrong-format +python3 gs_perfconfig preset test-preset-wrong-format2 +python3 gs_perfconfig preset test-preset-wrong-param +python3 gs_perfconfig preset test-preset-wrong-val +python3 gs_perfconfig preset test-preset-wrong-val32 +python3 gs_perfconfig preset test-preset-wrong-val + diff --git a/script/impl/perf_config/test/test_recover.sh b/script/impl/perf_config/test/test_recover.sh new file mode 100644 index 00000000..af3aacc3 --- /dev/null +++ b/script/impl/perf_config/test/test_recover.sh @@ -0,0 +1,23 @@ + +python3 gs_perfconfig recover +python3 gs_perfconfig recoveR + +# err +python3 gs_perfconfig tune +python3 gs_perfconfig recover + +# ok +python3 gs_perfconfig tune --apply +python3 gs_perfconfig recover + + +# root tune, user recover. err. because os. +python3 gs_perfconfig tune --apply +python3 gs_perfconfig recover + + + +# use tune, root recover. ok +python3 gs_perfconfig tune --apply +python3 gs_perfconfig recover + diff --git a/script/impl/perf_config/test/test_tune.sh b/script/impl/perf_config/test/test_tune.sh new file mode 100644 index 00000000..4084cb60 --- /dev/null +++ b/script/impl/perf_config/test/test_tune.sh @@ -0,0 +1,55 @@ +export GAUSSHOME='' +export GAUSSLOG='' +export PGDATA='' +vim /home/carrot/test.source # GAUSSHOME GAUSSLOG PGDATA. gs_om env + +# root +python3 gs_perfconfig tune +python3 gs_perfconfig tune -t os,guc,os +python3 gs_perfconfig tune -t all +python3 gs_perfconfig tune -t os,guc,os --apply +python3 gs_perfconfig tune -t all --apply +python3 gs_perfconfig tune -t os,guc,os --apply -y +python3 gs_perfconfig tune -t os,guc,os --apply --env /home/carrot/test.source +python3 gs_perfconfig tune -t all --apply --env /home/carrot/test.source -y +python3 gs_perfconfig tune --apply +python3 gs_perfconfig tune --apply --env /home/carrot/test.source + + +# user +python3 gs_perfconfig tune +python3 gs_perfconfig tune -t os +python3 gs_perfconfig tune -t os,guc,os +python3 gs_perfconfig tune -t all +python3 gs_perfconfig tune -t os,guc,os --apply +python3 gs_perfconfig tune -t all --apply +python3 gs_perfconfig tune -t os,guc,os --apply --env /home/carrot/test.source +python3 gs_perfconfig tune -t all --apply --env /home/carrot/test.source +python3 gs_perfconfig tune --apply -y +python3 gs_perfconfig tune --apply --env /home/carrot/test.source -y + + +# mix +# root +python3 gs_perfconfig tune +# user +python3 gs_perfconfig tune -t os +# root +python3 gs_perfconfig tune -t os,guc,os +# user +python3 gs_perfconfig tune -t all +# root +python3 gs_perfconfig tune -t os,guc,os --apply +# user +python3 gs_perfconfig tune -t all --apply +# root +python3 gs_perfconfig tune -t os,guc,os --apply --env /home/carrot/test.source -y +# user +python3 gs_perfconfig tune -t all --apply --env /home/carrot/test.source -y +# root +python3 gs_perfconfig tune --apply -y +# user +python3 gs_perfconfig tune --apply --env /home/carrot/test.source -y + + + diff --git a/script/impl/perf_config/test/test_tune_with_err.sh b/script/impl/perf_config/test/test_tune_with_err.sh new file mode 100644 index 00000000..1875536f --- /dev/null +++ b/script/impl/perf_config/test/test_tune_with_err.sh @@ -0,0 +1,10 @@ + +mv $GAUSSHOME/bin/gs_guc $GAUSSHOME/bin/gs_guc_bak + +# user with env +python3 gs_perfconfig tune -t all --apply + + +mv $GAUSSHOME/bin/gs_guc_bak $GAUSSHOME/bin/gs_guc + + diff --git a/script/impl/perf_config/tuners/__init__.py b/script/impl/perf_config/tuners/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/tuners/guc.py b/script/impl/perf_config/tuners/guc.py new file mode 100644 index 00000000..a5991f95 --- /dev/null +++ b/script/impl/perf_config/tuners/guc.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + + +import os +import json +import shutil + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.anti import AntiLog +from impl.perf_config.basic.tuner import Tuner, TunerGroup +from impl.perf_config.basic.guc import GucRootTuner +from impl.perf_config.tuners.gucs.common import CommonGUC, FileLocationGUC, KernelResourceGUC, MemoryGUC, SysCacheGUC, \ + AlarmGUC, RegionFormatGUC, ThreadPoolGUC, UpgradeGUC, MotGUC, GlobalTempTableGUC, UserDefineFuncGUC, JobScheduleGUC +from impl.perf_config.tuners.gucs.connection import ConnectionGUC, PoolerGUC +from impl.perf_config.tuners.gucs.execute import StmtBehaviorGUC, VersionCompatibilityGUC, EnvCompatibilityGUC +from impl.perf_config.tuners.gucs.ha_cluster import SenderServerGUC, PrimaryServerGUC, StandbyServerGUC +from impl.perf_config.tuners.gucs.ops import StatisticCollectGUC, WorkloadManagerGUC, TrackStmtGUC, WdrAspGUC, LogGUC +from impl.perf_config.tuners.gucs.optimizer import OptNodeCostGUC, OptRewriteGUC, OptPartTableGUC, OptGeqoGUC, \ + OptCodeGenGUC, OptBypassGUC, OptExplainGUC, OptSmpGUC, OptNgrmGUC, OptPbeGUC, OptGlobalPlanCacheGUC, OptOtherGUC +from impl.perf_config.tuners.gucs.other import DoubleDbReplicationGUC, AiGUC, DcfGUC, NvmGUC, FaultToleranceGUC, \ + HyperLogLogGUC, StandbyIUDGuc, DevelopOptionGUC, UndoGUC, OtherDefaultGUC, OtherOptionsGUC +from impl.perf_config.tuners.gucs.security import SecurityGUC, AuditGUC +from impl.perf_config.tuners.gucs.storage import VacuumGUC, CheckpointGUC, BackendWriteThreadGUC, DoubleWriteGUC, \ + AsyncIoGUC, WalGUC, RecoveryGUC, BackoutRecoveryGUC, ArchiveGUC, LockManagerGUC, TransactionGUC, SharedStorageGUC + + +class GucTuner(GucRootTuner): + def __init__(self): + super(GucTuner, self).__init__() + ########################################################################################### + # Register each GUC tune group. Theoretically, the adjustment strategy of different + # guc groups should be independent of each other, but it is inevitable that there + # are special cases. So, we still need to pay attention to the order. + ########################################################################################### + # common + self.common = self.add(CommonGUC()) + self.file_location = self.add(FileLocationGUC()) + self.kernel_resource = self.add(KernelResourceGUC()) + self.memory = self.add(MemoryGUC()) + self.syscache = self.add(SysCacheGUC()) + self.alarm = self.add(AlarmGUC()) + self.region_format = self.add(RegionFormatGUC()) + self.thread_pool = self.add(ThreadPoolGUC()) + self.upgrade = self.add(UpgradeGUC()) + self.mot = self.add(MotGUC()) + self.gtt = self.add(GlobalTempTableGUC()) + self.udf = self.add(UserDefineFuncGUC()) + self.job = self.add(JobScheduleGUC()) + + # security + self.security = self.add(SecurityGUC()) + self.audit = self.add(AuditGUC()) + + # connection + self.connection = self.add(ConnectionGUC()) + self.pooler = self.add(PoolerGUC()) + + # optmizer + self.opt_nodecost = self.add(OptNodeCostGUC()) + self.opt_rewrite = self.add(OptRewriteGUC()) + self.opt_partition = self.add(OptPartTableGUC()) + self.opt_geqo = self.add(OptGeqoGUC()) + self.opt_codegen = self.add(OptCodeGenGUC()) + self.opt_bypass = self.add(OptBypassGUC()) + self.opt_explain = self.add(OptExplainGUC()) + self.opt_smp = self.add(OptSmpGUC()) + self.opt_ngrm = self.add(OptNgrmGUC()) + self.opt_pbe = self.add(OptPbeGUC()) + self.opt_gpc = self.add(OptGlobalPlanCacheGUC()) + self.opt_other = self.add(OptOtherGUC()) + + # execute + self.stmt_behavior = self.add(StmtBehaviorGUC()) + self.version_compa = self.add(VersionCompatibilityGUC()) + self.env_compa = self.add(EnvCompatibilityGUC()) + + # storage + self.vacuum = self.add(VacuumGUC()) + self.checkpoint = self.add(CheckpointGUC()) + self.bgwrite = self.add(BackendWriteThreadGUC()) + self.dw = self.add(DoubleWriteGUC()) + self.async_io = self.add(AsyncIoGUC()) + self.wal = self.add(WalGUC()) + self.recovery = self.add(RecoveryGUC()) + self.backout_recovery = self.add(BackoutRecoveryGUC()) + self.archive = self.add(ArchiveGUC()) + self.lock_mgr = self.add(LockManagerGUC()) + self.transaction = self.add(TransactionGUC()) + self.shared_storage = self.add(SharedStorageGUC()) + + # ha + self.sender_server = self.add(SenderServerGUC()) + self.primary_server = self.add(PrimaryServerGUC()) + self.standby_server = self.add(StandbyServerGUC()) + + # ops + self.statistic = self.add(StatisticCollectGUC()) + self.wlm = self.add(WorkloadManagerGUC()) + self.track_stmt = self.add(TrackStmtGUC()) + self.wdr_asp = self.add(WdrAspGUC()) + self.log = self.add(LogGUC()) + + # other + self.dbrep = self.add(DoubleDbReplicationGUC()) + self.ai = self.add(AiGUC()) + self.dcf = self.add(DcfGUC()) + self.nvm = self.add(NvmGUC()) + self.fault_tolerance = self.add(FaultToleranceGUC()) + self.hll = self.add(HyperLogLogGUC()) + self.standby_iud = self.add(StandbyIUDGuc()) + self.dev_options = self.add(DevelopOptionGUC()) + self.undo = self.add(UndoGUC()) + self.other_default = self.add(OtherDefaultGUC()) + self.other_options = self.add(OtherOptionsGUC()) + diff --git a/script/impl/perf_config/tuners/gucs/__init__.py b/script/impl/perf_config/tuners/gucs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/script/impl/perf_config/tuners/gucs/common.py b/script/impl/perf_config/tuners/gucs/common.py new file mode 100644 index 00000000..2599a92b --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/common.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + + +import math + +from base_utils.os.cpu_util import CpuArchitecture, CpuUtil + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.tuner import Tuner, TunerGroup +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup +from impl.perf_config.probes.business import BsScenario +from impl.perf_config.tuners.os import CPUTuner + +class CommonGUC(GUCTuneGroup): + def __init__(self): + super(CommonGUC, self).__init__() + self.pgxc_node_name = self.bind('pgxc_node_name') + + def calculate(self): + pass + + +class FileLocationGUC(GUCTuneGroup): + def __init__(self): + super(FileLocationGUC, self).__init__() + self.data_directory = self.bind('data_directory') + self.config_file = self.bind('config_file') + self.hba_file = self.bind('hba_file') + self.ident_file = self.bind('ident_file') + self.external_pid_file = self.bind('external_pid_file') + self.enable_default_cfunc_libpath = self.bind('enable_default_cfunc_libpath') + + def calculate(self): + pass + + +class KernelResourceGUC(GUCTuneGroup): + def __init__(self): + super(KernelResourceGUC, self).__init__() + self.max_files_per_process = self.bind('max_files_per_process') + self.shared_preload_libraries = self.bind('shared_preload_libraries') + self.sql_use_spacelimit = self.bind('sql_use_spacelimit') + self.temp_file_limit = self.bind('temp_file_limit') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + + # double maybe ok + max_files_per_process = infos.business.relfilenode_count() * 2 + self.max_files_per_process.set(str(max_files_per_process)) + + +class MemoryGUC(GUCTuneGroup): + def __init__(self): + super(MemoryGUC, self).__init__() + self.memorypool_enable = self.bind('memorypool_enable') + self.memorypool_size = self.bind('memorypool_size') + self.enable_memory_limit = self.bind('enable_memory_limit') + self.max_process_memory = self.bind('max_process_memory') + self.enable_memory_context_control = self.bind('enable_memory_context_control') + self.uncontrolled_memory_context = self.bind('uncontrolled_memory_context') + self.shared_buffers = self.bind('shared_buffers') + self.cstore_buffers = self.bind('cstore_buffers') + self.segment_buffers = self.bind('segment_buffers') + self.temp_buffers = self.bind('temp_buffers') + self.bulk_write_ring_size = self.bind('bulk_write_ring_size') + self.standby_shared_buffers_fraction = self.bind('standby_shared_buffers_fraction') + self.max_prepared_transactions = self.bind('max_prepared_transactions') + self.work_mem = self.bind('work_mem') + self.query_mem = self.bind('query_mem') + self.query_max_mem = self.bind('query_max_mem') + self.maintenance_work_mem = self.bind('maintenance_work_mem') + self.psort_work_mem = self.bind('psort_work_mem') + self.max_loaded_cudesc = self.bind('max_loaded_cudesc') + self.max_stack_depth = self.bind('max_stack_depth') + self.bulk_read_ring_size = self.bind('bulk_read_ring_size') + self.enable_early_free = self.bind('enable_early_free') + self.resilience_memory_reject_percent = self.bind('resilience_memory_reject_percent') + + self.enable_huge_pages = self.bind('enable_huge_pages') + self.huge_page_size = self.bind('huge_page_size') + + self.pca_shared_buffers = self.bind('pca_shared_buffers') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + self.memorypool_enable.turn_off() + + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.enable_memory_limit.turn_off() + self.enable_memory_context_control.turn_off() + + max_process_memory_rate = 0.666 + shared_buffers_rate = 0.4 + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + max_process_memory_rate = 0.7 + shared_buffers_rate = 0.5 + elif infos.business.scenario == BsScenario.AP: + max_process_memory_rate = 0.65 + shared_buffers_rate = 0.5 + + max_process_memory = int(infos.memory.total_size() * max_process_memory_rate / 1024) + if max_process_memory == 0: + Project.fatat('Your machine memory is too small to support configuration.') + self.max_process_memory.set('{}MB'.format(max_process_memory)) + + shared_buffers = int(infos.memory.total_size() * shared_buffers_rate / 1024) + if self.shared_buffers == 0: + Project.fatat('Your machine memory is too small to support configuration.') + self.shared_buffers.set('{}MB'.format(shared_buffers)) + + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.work_mem.set('1MB') + self.maintenance_work_mem.set('2GB') + + if infos.business.scenario == BsScenario.AP: + self.enable_early_free.turn_on() + + +class SysCacheGUC(GUCTuneGroup): + def __init__(self): + super(SysCacheGUC, self).__init__() + self.enable_global_syscache = self.bind('enable_global_syscache') + self.local_syscache_threshold = self.bind('local_syscache_threshold') + self.global_syscache_threshold = self.bind('global_syscache_threshold') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.enable_global_syscache.turn_off() + self.local_syscache_threshold.set('16MB') + + +class AlarmGUC(GUCTuneGroup): + def __init__(self): + super(AlarmGUC, self).__init__() + self.enable_alarm = self.bind('enable_alarm') + self.connection_alarm_rate = self.bind('connection_alarm_rate') + self.alarm_report_interval = self.bind('alarm_report_interval') + self.alarm_component = self.bind('alarm_component') + self.table_skewness_warning_threshold = self.bind('table_skewness_warning_threshold') + self.table_skewness_warning_rows = self.bind('table_skewness_warning_rows') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.enable_alarm.turn_off() + + +class RegionFormatGUC(GUCTuneGroup): + def __init__(self): + super(RegionFormatGUC, self).__init__() + self.DateStyle = self.bind('DateStyle') + self.IntervalStyle = self.bind('IntervalStyle') + self.TimeZone = self.bind('TimeZone') + self.timezone_abbreviations = self.bind('timezone_abbreviations') + self.extra_float_digits = self.bind('extra_float_digits') + self.client_encoding = self.bind('client_encoding') + self.lc_messages = self.bind('lc_messages') + self.lc_monetary = self.bind('lc_monetary') + self.lc_numeric = self.bind('lc_numeric') + self.lc_time = self.bind('lc_time') + self.default_text_search_config = self.bind('default_text_search_config') + + def calculate(self): + self.lc_messages.set('C') + self.lc_monetary.set('C') + self.lc_numeric.set('C') + self.lc_time.set('C') + + +class ThreadPoolGUC(GUCTuneGroup): + def __init__(self): + super(ThreadPoolGUC, self).__init__() + self.enable_thread_pool = self.bind('enable_thread_pool') + self.thread_pool_attr = self.bind('thread_pool_attr') + self.thread_pool_stream_attr = self.bind('thread_pool_stream_attr') + self.resilience_threadpool_reject_cond = self.bind('resilience_threadpool_reject_cond') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + numa_bind_info = infos.cpu.notebook.read('numa_bind_info') + if numa_bind_info is None: + numa_bind_info = CPUTuner.calculate_numa_bind() + if not numa_bind_info['use']: + self.enable_thread_pool.turn_off() + return + + for sug in numa_bind_info['suggestions']: + Project.report.suggest(sug) + + numa_group_count = len(infos.cpu.numa()) + thread_count = math.floor(len(numa_bind_info['threadpool']) * 7.25) + cpubind = 'cpubind:{}'.format(CpuUtil.cpuListToCpuRangeStr(numa_bind_info['threadpool'])) \ + if len(numa_bind_info['threadpool']) != infos.cpu.count() else 'allbind' + + thread_pool_attr = '{0},{1},({2})'.format(numa_group_count, thread_count, cpubind) + self.thread_pool_attr.set(thread_pool_attr) + + +class UpgradeGUC(GUCTuneGroup): + def __init__(self): + super(UpgradeGUC, self).__init__() + self.IsInplaceUpgrade = self.bind('IsInplaceUpgrade') + self.inplace_upgrade_next_system_object_oids = self.bind('inplace_upgrade_next_system_object_oids') + self.upgrade_mode = self.bind('upgrade_mode') + + def calculate(self): + pass + + +class MotGUC(GUCTuneGroup): + def __init__(self): + super(MotGUC, self).__init__() + self.enable_codegen_mot = self.bind('enable_codegen_mot') + self.force_pseudo_codegen_mot = self.bind('force_pseudo_codegen_mot') + self.enable_codegen_mot_print = self.bind('enable_codegen_mot_print') + self.codegen_mot_limit = self.bind('codegen_mot_limit') + self.mot_allow_index_on_nullable_column = self.bind('mot_allow_index_on_nullable_column') + self.mot_config_file = self.bind('mot_config_file') + + def calculate(self): + pass + + +class GlobalTempTableGUC(GUCTuneGroup): + def __init__(self): + super(GlobalTempTableGUC, self).__init__() + self.max_active_global_temporary_table = self.bind('max_active_global_temporary_table') + self.vacuum_gtt_defer_check_age = self.bind('vacuum_gtt_defer_check_age') + self.enable_gtt_concurrent_truncate = self.bind('enable_gtt_concurrent_truncate') + + def calculate(self): + pass + + +class UserDefineFuncGUC(GUCTuneGroup): + def __init__(self): + super(UserDefineFuncGUC, self).__init__() + self.udf_memory_limit = self.bind('udf_memory_limit') + self.FencedUDFMemoryLimit = self.bind('FencedUDFMemoryLimit') + self.UDFWorkerMemHardLimit = self.bind('UDFWorkerMemHardLimit') + self.pljava_vmoptions = self.bind('pljava_vmoptions') + + def calculate(self): + pass + + +class JobScheduleGUC(GUCTuneGroup): + def __init__(self): + super(JobScheduleGUC, self).__init__() + self.job_queue_processes = self.bind('job_queue_processes') + self.enable_prevent_job_task_startup = self.bind('enable_prevent_job_task_startup') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.job_queue_processes.set('0') diff --git a/script/impl/perf_config/tuners/gucs/connection.py b/script/impl/perf_config/tuners/gucs/connection.py new file mode 100644 index 00000000..8ea53bfd --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/connection.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.tuner import Tuner, TunerGroup +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup + + +class ConnectionGUC(GUCTuneGroup): + def __init__(self): + super(ConnectionGUC, self).__init__() + self.light_comm = self.bind('light_comm') + self.listen_addresses = self.bind('listen_addresses') + self.local_bind_address = self.bind('local_bind_address') + self.port = self.bind('port') + self.max_connections = self.bind('max_connections') + self.max_inner_tool_connections = self.bind('max_inner_tool_connections') + self.sysadmin_reserved_connections = self.bind('sysadmin_reserved_connections') + self.unix_socket_directory = self.bind('unix_socket_directory') + self.unix_socket_group = self.bind('unix_socket_group') + self.unix_socket_permissions = self.bind('unix_socket_permissions') + self.application_name = self.bind('application_name') + self.connection_info = self.bind('connection_info') + self.enable_dolphin_proto = self.bind('enable_dolphin_proto') + self.dolphin_server_port = self.bind('dolphin_server_port') + self.tcp_keepalives_idle = self.bind('tcp_keepalives_idle') + self.tcp_keepalives_interval = self.bind('tcp_keepalives_interval') + self.tcp_keepalives_count = self.bind('tcp_keepalives_count') + self.tcp_user_timeout = self.bind('tcp_user_timeout') + self.comm_proxy_attr = self.bind('comm_proxy_attr') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + self.light_comm.set('on') + + max_connections = int(infos.business.parallel) * 4 + self.max_connections.set(str(max_connections)) + + +class PoolerGUC(GUCTuneGroup): + def __init__(self): + super(PoolerGUC, self).__init__() + self.pooler_maximum_idle_time = self.bind('pooler_maximum_idle_time') + self.minimum_pool_size = self.bind('minimum_pool_size') + self.cache_connection = self.bind('cache_connection') + + def calculate(self): + pass + diff --git a/script/impl/perf_config/tuners/gucs/execute.py b/script/impl/perf_config/tuners/gucs/execute.py new file mode 100644 index 00000000..f5933d7f --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/execute.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup + + +class StmtBehaviorGUC(GUCTuneGroup): + def __init__(self): + super(StmtBehaviorGUC, self).__init__() + self.search_path = self.bind('search_path') + self.current_schema = self.bind('current_schema') + self.default_tablespace = self.bind('default_tablespace') + self.temp_tablespaces = self.bind('temp_tablespaces') + self.check_function_bodies = self.bind('check_function_bodies') + self.default_transaction_isolation = self.bind('default_transaction_isolation') + self.default_transaction_read_only = self.bind('default_transaction_read_only') + self.default_transaction_deferrable = self.bind('default_transaction_deferrable') + self.session_replication_role = self.bind('session_replication_role') + self.statement_timeout = self.bind('statement_timeout') + self.bytea_output = self.bind('bytea_output') + self.xmlbinary = self.bind('xmlbinary') + self.xmloption = self.bind('xmloption') + self.max_compile_functions = self.bind('max_compile_functions') + self.gin_pending_list_limit = self.bind('gin_pending_list_limit') + + def calculate(self): + pass + + +class VersionCompatibilityGUC(GUCTuneGroup): + def __init__(self): + super(VersionCompatibilityGUC, self).__init__() + self.array_nulls = self.bind('array_nulls') + self.backslash_quote = self.bind('backslash_quote') + self.escape_string_warning = self.bind('escape_string_warning') + self.lo_compat_privileges = self.bind('lo_compat_privileges') + self.quote_all_identifiers = self.bind('quote_all_identifiers') + self.sql_inheritance = self.bind('sql_inheritance') + self.standard_conforming_strings = self.bind('standard_conforming_strings') + self.synchronize_seqscans = self.bind('synchronize_seqscans') + self.enable_beta_features = self.bind('enable_beta_features') + self.default_with_oids = self.bind('default_with_oids') + + def calculate(self): + self.enable_beta_features.turn_on() + + +class EnvCompatibilityGUC(GUCTuneGroup): + def __init__(self): + super(EnvCompatibilityGUC, self).__init__() + self.convert_string_to_digit = self.bind('convert_string_to_digit') + self.nls_timestamp_format = self.bind('nls_timestamp_format') + self.group_concat_max_len = self.bind('group_concat_max_len') + self.max_function_args = self.bind('max_function_args') + self.transform_null_equals = self.bind('transform_null_equals') + self.support_extended_features = self.bind('support_extended_features') + self.sql_compatibility = self.bind('sql_compatibility') + self.b_format_behavior_compat_options = self.bind('b_format_behavior_compat_options') + self.enable_set_variable_b_format = self.bind('enable_set_variable_b_format') + self.behavior_compat_options = self.bind('behavior_compat_options') + self.plsql_compile_check_options = self.bind('plsql_compile_check_options') + self.td_compatible_truncation = self.bind('td_compatible_truncation') + self.uppercase_attribute_name = self.bind('uppercase_attribute_name') + self.lastval_supported = self.bind('lastval_supported') + self.character_set_connection = self.bind('character_set_connection') + self.collation_connection = self.bind('collation_connection') + + def calculate(self): + pass + diff --git a/script/impl/perf_config/tuners/gucs/ha_cluster.py b/script/impl/perf_config/tuners/gucs/ha_cluster.py new file mode 100644 index 00000000..b7848134 --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/ha_cluster.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup + + +class SenderServerGUC(GUCTuneGroup): + def __init__(self): + super(SenderServerGUC, self).__init__() + self.max_wal_senders = self.bind('max_wal_senders') + self.wal_keep_segments = self.bind('wal_keep_segments') + self.wal_sender_timeout = self.bind('wal_sender_timeout') + self.logical_sender_timeout = self.bind('logical_sender_timeout') + self.max_replication_slots = self.bind('max_replication_slots') + self.enable_slot_log = self.bind('enable_slot_log') + self.max_changes_in_memory = self.bind('max_changes_in_memory') + self.max_cached_tuplebufs = self.bind('max_cached_tuplebufs') + self.enable_wal_shipping_compression = self.bind('enable_wal_shipping_compression') + self.repl_auth_mode = self.bind('repl_auth_mode') + self.repl_uuid = self.bind('repl_uuid') + self.replconninfo1 = self.bind('replconninfo1') + self.replconninfo2 = self.bind('replconninfo2') + self.replconninfo3 = self.bind('replconninfo3') + self.replconninfo4 = self.bind('replconninfo4') + self.replconninfo5 = self.bind('replconninfo5') + self.replconninfo6 = self.bind('replconninfo6') + self.replconninfo7 = self.bind('replconninfo7') + self.replconninfo8 = self.bind('replconninfo8') + self.cross_cluster_replconninfo1 = self.bind('cross_cluster_replconninfo1') + self.cross_cluster_replconninfo2 = self.bind('cross_cluster_replconninfo2') + self.cross_cluster_replconninfo3 = self.bind('cross_cluster_replconninfo3') + self.cross_cluster_replconninfo4 = self.bind('cross_cluster_replconninfo4') + self.cross_cluster_replconninfo5 = self.bind('cross_cluster_replconninfo5') + self.cross_cluster_replconninfo6 = self.bind('cross_cluster_replconninfo6') + self.cross_cluster_replconninfo7 = self.bind('cross_cluster_replconninfo7') + self.cross_cluster_replconninfo8 = self.bind('cross_cluster_replconninfo8') + self.available_zone = self.bind('available_zone') + self.max_keep_log_seg = self.bind('max_keep_log_seg') + self.cluster_run_mode = self.bind('cluster_run_mode') + + def calculate(self): + pass + + +class PrimaryServerGUC(GUCTuneGroup): + def __init__(self): + super(PrimaryServerGUC, self).__init__() + self.synchronous_standby_names = self.bind('synchronous_standby_names') + self.most_available_sync = self.bind('most_available_sync') + self.keep_sync_window = self.bind('keep_sync_window') + self.enable_stream_replication = self.bind('enable_stream_replication') + self.enable_mix_replication = self.bind('enable_mix_replication') + self.vacuum_defer_cleanup_age = self.bind('vacuum_defer_cleanup_age') + self.data_replicate_buffer_size = self.bind('data_replicate_buffer_size') + self.walsender_max_send_size = self.bind('walsender_max_send_size') + self.enable_data_replicate = self.bind('enable_data_replicate') + self.ha_module_debug = self.bind('ha_module_debug') + self.enable_incremental_catchup = self.bind('enable_incremental_catchup') + self.wait_dummy_time = self.bind('wait_dummy_time') + self.catchup2normal_wait_time = self.bind('catchup2normal_wait_time') + self.sync_config_strategy = self.bind('sync_config_strategy') + self.enable_save_confirmed_lsn = self.bind('enable_save_confirmed_lsn') + self.hadr_recovery_time_target = self.bind('hadr_recovery_time_target') + self.hadr_recovery_point_target = self.bind('hadr_recovery_point_target') + self.hadr_super_user_record_path = self.bind('hadr_super_user_record_path') + self.ignore_standby_lsn_window = self.bind('ignore_standby_lsn_window') + self.ignore_feedback_xmin_window = self.bind('ignore_feedback_xmin_window') + + def calculate(self): + pass + + +class StandbyServerGUC(GUCTuneGroup): + def __init__(self): + super(StandbyServerGUC, self).__init__() + self.hot_standby = self.bind('hot_standby') + self.max_standby_archive_delay = self.bind('max_standby_archive_delay') + self.max_standby_streaming_delay = self.bind('max_standby_streaming_delay') + self.wal_receiver_status_interval = self.bind('wal_receiver_status_interval') + self.hot_standby_feedback = self.bind('hot_standby_feedback') + self.wal_receiver_timeout = self.bind('wal_receiver_timeout') + self.wal_receiver_connect_timeout = self.bind('wal_receiver_connect_timeout') + self.wal_receiver_connect_retries = self.bind('wal_receiver_connect_retries') + self.wal_receiver_buffer_size = self.bind('wal_receiver_buffer_size') + self.primary_slotname = self.bind('primary_slotname') + self.max_logical_replication_workers = self.bind('max_logical_replication_workers') + self.max_sync_workers_per_subscription = self.bind('max_sync_workers_per_subscription') + + def calculate(self): + pass + diff --git a/script/impl/perf_config/tuners/gucs/ops.py b/script/impl/perf_config/tuners/gucs/ops.py new file mode 100644 index 00000000..d8b5b326 --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/ops.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.tuner import Tuner, TunerGroup +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup +from impl.perf_config.probes.business import BsScenario + + +class StatisticCollectGUC(GUCTuneGroup): + def __init__(self): + super(StatisticCollectGUC, self).__init__() + self.track_activities = self.bind('track_activities') + self.track_counts = self.bind('track_counts') + self.track_io_timing = self.bind('track_io_timing') + self.track_functions = self.bind('track_functions') + self.track_activity_query_size = self.bind('track_activity_query_size') + self.stats_temp_directory = self.bind('stats_temp_directory') + self.track_thread_wait_status_interval = self.bind('track_thread_wait_status_interval') + self.enable_save_datachanged_timestamp = self.bind('enable_save_datachanged_timestamp') + self.track_sql_count = self.bind('track_sql_count') + + def calculate(self): + self.track_activities.turn_off() + self.track_sql_count.turn_off() + + +class WorkloadManagerGUC(GUCTuneGroup): + def __init__(self): + super(WorkloadManagerGUC, self).__init__() + self.use_workload_manager = self.bind('use_workload_manager') + + self.memory_tracking_mode = self.bind('memory_tracking_mode') + self.memory_detail_tracking = self.bind('memory_detail_tracking') + self.memory_fault_percent = self.bind('memory_fault_percent') + self.disable_memory_protect = self.bind('disable_memory_protect') + self.session_history_memory = self.bind('session_history_memory') + + self.enable_resource_track = self.bind('enable_resource_track') + self.enable_resource_record = self.bind('enable_resource_record') + self.resource_track_level = self.bind('resource_track_level') + self.resource_track_cost = self.bind('resource_track_cost') + self.resource_track_duration = self.bind('resource_track_duration') + + self.enable_logical_io_statistics = self.bind('enable_logical_io_statistics') + self.enable_user_metric_persistent = self.bind('enable_user_metric_persistent') + self.user_metric_retention_time = self.bind('user_metric_retention_time') + self.enable_instance_metric_persistent = self.bind('enable_instance_metric_persistent') + self.instance_metric_retention_time = self.bind('instance_metric_retention_time') + + self.enable_bbox_dump = self.bind('enable_bbox_dump') + self.bbox_dump_count = self.bind('bbox_dump_count') + self.bbox_dump_path = self.bind('bbox_dump_path') + self.bbox_blanklist_items = self.bind('bbox_blanklist_items') + + self.io_limits = self.bind('io_limits') + self.io_priority = self.bind('io_priority') + self.io_control_unit = self.bind('io_control_unit') + self.session_respool = self.bind('session_respool') + self.session_statistics_memory = self.bind('session_statistics_memory') + self.topsql_retention_time = self.bind('topsql_retention_time') + self.transaction_pending_time = self.bind('transaction_pending_time') + self.current_logic_cluster = self.bind('current_logic_cluster') + self.enable_ffic_log = self.bind('enable_ffic_log') + self.cgroup_name = self.bind('cgroup_name') + self.cpu_collect_timer = self.bind('cpu_collect_timer') + self.query_band = self.bind('query_band') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.use_workload_manager.turn_off() + self.enable_logical_io_statistics.turn_off() + self.enable_user_metric_persistent.turn_off() + self.enable_instance_metric_persistent.turn_off() + self.enable_ffic_log.turn_off() + self.enable_bbox_dump.turn_off() + self.enable_resource_track.turn_off() + + +class TrackStmtGUC(GUCTuneGroup): + def __init__(self): + super(TrackStmtGUC, self).__init__() + self.instr_unique_sql_count = self.bind('instr_unique_sql_count') + self.instr_unique_sql_track_type = self.bind('instr_unique_sql_track_type') + self.enable_auto_clean_unique_sql = self.bind('enable_auto_clean_unique_sql') + + self.percentile = self.bind('percentile') + self.enable_instr_cpu_timer = self.bind('enable_instr_cpu_timer') + self.enable_instr_track_wait = self.bind('enable_instr_track_wait') + self.enable_instr_rt_percentile = self.bind('enable_instr_rt_percentile') + self.instr_rt_percentile_interval = self.bind('instr_rt_percentile_interval') + + self.enable_stmt_track = self.bind('enable_stmt_track') + self.track_stmt_session_slot = self.bind('track_stmt_session_slot') + self.track_stmt_details_size = self.bind('track_stmt_details_size') + self.track_stmt_retention_time = self.bind('track_stmt_retention_time') + self.track_stmt_stat_level = self.bind('track_stmt_stat_level') + self.track_stmt_standby_chain_size = self.bind('track_stmt_standby_chain_size') + self.log_min_duration_statement = self.bind('log_min_duration_statement') + + self.time_record_level = self.bind('time_record_level') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.enable_stmt_track.turn_off() + self.instr_unique_sql_count.set('0') + self.time_record_level.set('1') + self.enable_instr_cpu_timer.turn_off() + self.enable_instr_track_wait.turn_off() + self.enable_instr_rt_percentile.turn_off() + + +class WdrAspGUC(GUCTuneGroup): + def __init__(self): + super(WdrAspGUC, self).__init__() + self.enable_wdr_snapshot = self.bind('enable_wdr_snapshot') + self.wdr_snapshot_retention_days = self.bind('wdr_snapshot_retention_days') + self.wdr_snapshot_query_timeout = self.bind('wdr_snapshot_query_timeout') + self.wdr_snapshot_interval = self.bind('wdr_snapshot_interval') + + self.enable_asp = self.bind('enable_asp') + self.asp_flush_mode = self.bind('asp_flush_mode') + self.asp_flush_rate = self.bind('asp_flush_rate') + self.asp_log_filename = self.bind('asp_log_filename') + self.asp_retention_days = self.bind('asp_retention_days') + self.asp_sample_interval = self.bind('asp_sample_interval') + self.asp_sample_num = self.bind('asp_sample_num') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.enable_wdr_snapshot.turn_off() + self.enable_asp.turn_off() + + +class LogGUC(GUCTuneGroup): + def __init__(self): + super(LogGUC, self).__init__() + self.debug_print_parse = self.bind('debug_print_parse') + self.debug_print_rewritten = self.bind('debug_print_rewritten') + self.debug_print_plan = self.bind('debug_print_plan') + self.debug_pretty_print = self.bind('debug_pretty_print') + + self.log_parser_stats = self.bind('log_parser_stats') + self.log_planner_stats = self.bind('log_planner_stats') + self.log_executor_stats = self.bind('log_executor_stats') + self.log_statement_stats = self.bind('log_statement_stats') + self.log_destination = self.bind('log_destination') + self.log_directory = self.bind('log_directory') + self.log_filename = self.bind('log_filename') + self.log_file_mode = self.bind('log_file_mode') + self.log_truncate_on_rotation = self.bind('log_truncate_on_rotation') + self.log_rotation_age = self.bind('log_rotation_age') + self.log_rotation_size = self.bind('log_rotation_size') + self.log_checkpoints = self.bind('log_checkpoints') + self.log_connections = self.bind('log_connections') + self.log_disconnections = self.bind('log_disconnections') + self.log_duration = self.bind('log_duration') + self.log_error_verbosity = self.bind('log_error_verbosity') + self.log_hostname = self.bind('log_hostname') + self.log_line_prefix = self.bind('log_line_prefix') + self.log_lock_waits = self.bind('log_lock_waits') + self.log_statement = self.bind('log_statement') + self.log_temp_files = self.bind('log_temp_files') + self.log_timezone = self.bind('log_timezone') + self.logging_collector = self.bind('logging_collector') + self.logging_module = self.bind('logging_module') + self.log_min_error_statement = self.bind('log_min_error_statement') + self.log_min_messages = self.bind('log_min_messages') + self.client_min_messages = self.bind('client_min_messages') + + self.event_source = self.bind('event_source') + self.enable_debug_vacuum = self.bind('enable_debug_vacuum') + self.backtrace_min_messages = self.bind('backtrace_min_messages') + self.plog_merge_age = self.bind('plog_merge_age') + self.syslog_facility = self.bind('syslog_facility') + self.syslog_ident = self.bind('syslog_ident') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.log_min_messages.set('FATAL') + self.client_min_messages.set('ERROR') + self.log_duration.turn_off() + + diff --git a/script/impl/perf_config/tuners/gucs/optimizer.py b/script/impl/perf_config/tuners/gucs/optimizer.py new file mode 100644 index 00000000..4e267248 --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/optimizer.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.tuner import Tuner, TunerGroup +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup +from impl.perf_config.probes.business import BsScenario + + +class OptNodeCostGUC(GUCTuneGroup): + def __init__(self): + super(OptNodeCostGUC, self).__init__() + self.enable_broadcast = self.bind('enable_broadcast') + self.enable_material = self.bind('enable_material') + self.enable_sort = self.bind('enable_sort') + + self.enable_bitmapscan = self.bind('enable_bitmapscan') + self.enable_indexscan = self.bind('enable_indexscan') + self.enable_indexonlyscan = self.bind('enable_indexonlyscan') + self.enable_seqscan = self.bind('enable_seqscan') + self.enable_tidscan = self.bind('enable_tidscan') + self.force_bitmapand = self.bind('force_bitmapand') + self.cost_weight_index = self.bind('cost_weight_index') + self.effective_cache_size = self.bind('effective_cache_size') + + self.enable_hashagg = self.bind('enable_hashagg') + self.enable_sortgroup_agg = self.bind('enable_sortgroup_agg') + self.enable_sonic_hashagg = self.bind('enable_sonic_hashagg') + + self.enable_hashjoin = self.bind('enable_hashjoin') + self.enable_mergejoin = self.bind('enable_mergejoin') + self.enable_nestloop = self.bind('enable_nestloop') + self.enable_index_nestloop = self.bind('enable_index_nestloop') + self.enable_inner_unique_opt = self.bind('enable_inner_unique_opt') + self.enable_change_hjcost = self.bind('enable_change_hjcost') + self.enable_sonic_hashjoin = self.bind('enable_sonic_hashjoin') + self.enable_sonic_optspill = self.bind('enable_sonic_optspill') + + self.enable_vector_engine = self.bind('enable_vector_engine') + self.enable_vector_targetlist = self.bind('enable_vector_targetlist') + self.enable_force_vector_engine = self.bind('enable_force_vector_engine') + self.try_vector_engine_strategy = self.bind('try_vector_engine_strategy') + + self.seq_page_cost = self.bind('seq_page_cost') + self.random_page_cost = self.bind('random_page_cost') + self.cpu_tuple_cost = self.bind('cpu_tuple_cost') + self.cpu_index_tuple_cost = self.bind('cpu_index_tuple_cost') + self.cpu_operator_cost = self.bind('cpu_operator_cost') + self.allocate_mem_cost = self.bind('allocate_mem_cost') + + self.var_eq_const_selectivity = self.bind('var_eq_const_selectivity') + self.cost_param = self.bind('cost_param') + + self.enable_functional_dependency = self.bind('enable_functional_dependency') + self.default_statistics_target = self.bind('default_statistics_target') + self.constraint_exclusion = self.bind('constraint_exclusion') + + self.cursor_tuple_fraction = self.bind('cursor_tuple_fraction') + self.default_limit_rows = self.bind('default_limit_rows') + self.enable_extrapolation_stats = self.bind('enable_extrapolation_stats') + + def calculate(self): + pass + + +class OptRewriteGUC(GUCTuneGroup): + def __init__(self): + super(OptRewriteGUC, self).__init__() + self.qrw_inlist2join_optmode = self.bind('qrw_inlist2join_optmode') + self.rewrite_rule = self.bind('rewrite_rule') + self.from_collapse_limit = self.bind('from_collapse_limit') + self.join_collapse_limit = self.bind('join_collapse_limit') + + def calculate(self): + pass + + +class OptPartTableGUC(GUCTuneGroup): + def __init__(self): + super(OptPartTableGUC, self).__init__() + self.enable_valuepartition_pruning = self.bind('enable_valuepartition_pruning') + self.partition_page_estimation = self.bind('partition_page_estimation') + self.partition_iterator_elimination = self.bind('partition_iterator_elimination') + self.enable_partitionwise = self.bind('enable_partitionwise') + + def calculate(self): + pass + + +class OptGeqoGUC(GUCTuneGroup): + def __init__(self): + super(OptGeqoGUC, self).__init__() + self.geqo = self.bind('geqo') + self.geqo_threshold = self.bind('geqo_threshold') + self.geqo_effort = self.bind('geqo_effort') + self.geqo_pool_size = self.bind('geqo_pool_size') + self.geqo_generations = self.bind('geqo_generations') + self.geqo_selection_bias = self.bind('geqo_selection_bias') + self.geqo_seed = self.bind('geqo_seed') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.geqo.turn_off() + + +class OptCodeGenGUC(GUCTuneGroup): + def __init__(self): + super(OptCodeGenGUC, self).__init__() + self.enable_codegen = self.bind('enable_codegen') + self.codegen_strategy = self.bind('codegen_strategy') + self.enable_codegen_print = self.bind('enable_codegen_print') + self.codegen_cost_threshold = self.bind('codegen_cost_threshold') + + def calculate(self): + self.enable_codegen.turn_off() + + +class OptBypassGUC(GUCTuneGroup): + def __init__(self): + super(OptBypassGUC, self).__init__() + self.enable_opfusion = self.bind('enable_opfusion') + self.enable_partition_opfusion = self.bind('enable_partition_opfusion') + self.opfusion_debug_mode = self.bind('opfusion_debug_mode') + + def calculate(self): + self.enable_opfusion.turn_on() + self.enable_partition_opfusion.turn_on() + + +class OptExplainGUC(GUCTuneGroup): + def __init__(self): + super(OptExplainGUC, self).__init__() + self.explain_perf_mode = self.bind('explain_perf_mode') + self.explain_dna_file = self.bind('explain_dna_file') + self.enable_hypo_index = self.bind('enable_hypo_index') + self.enable_auto_explain = self.bind('enable_auto_explain') + self.auto_explain_level = self.bind('auto_explain_level') + self.show_fdw_remote_plan = self.bind('show_fdw_remote_plan') + + def calculate(self): + pass + + +class OptSmpGUC(GUCTuneGroup): + def __init__(self): + super(OptSmpGUC, self).__init__() + self.query_dop = self.bind('query_dop') + self.enable_seqscan_dopcost = self.bind('enable_seqscan_dopcost') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.AP: + self.query_dop.set('4') + + +class OptNgrmGUC(GUCTuneGroup): + def __init__(self): + super(OptNgrmGUC, self).__init__() + self.ngram_gram_size = self.bind('ngram_gram_size') + self.ngram_grapsymbol_ignore = self.bind('ngram_grapsymbol_ignore') + self.ngram_punctuation_ignore = self.bind('ngram_punctuation_ignore') + + def calculate(self): + pass + + +class OptPbeGUC(GUCTuneGroup): + def __init__(self): + super(OptPbeGUC, self).__init__() + self.enable_pbe_optimization = self.bind('enable_pbe_optimization') + self.plan_cache_mode = self.bind('plan_cache_mode') + + def calculate(self): + pass + + +class OptGlobalPlanCacheGUC(GUCTuneGroup): + def __init__(self): + super(OptGlobalPlanCacheGUC, self).__init__() + self.enable_global_plancache = self.bind('enable_global_plancache') + self.gpc_clean_timeout = self.bind('gpc_clean_timeout') + + def calculate(self): + pass + + +class OptOtherGUC(GUCTuneGroup): + def __init__(self): + super(OptOtherGUC, self).__init__() + self.enable_startwith_debug = self.bind('enable_startwith_debug') + + self.analysis_options = self.bind('analysis_options') + self.plan_mode_seed = self.bind('plan_mode_seed') + + self.enable_global_stats = self.bind('enable_global_stats') + + self.sql_beta_feature = self.bind('sql_beta_feature') + + self.enable_bloom_filter = self.bind('enable_bloom_filter') + + self.autoanalyze = self.bind('autoanalyze') + self.enable_analyze_check = self.bind('enable_analyze_check') + + self.skew_option = self.bind('skew_option') + self.enable_expr_fusion = self.bind('enable_expr_fusion') + self.enable_indexscan_optimization = self.bind('enable_indexscan_optimization') + self.enable_default_index_deduplication = self.bind('enable_default_index_deduplication') + + # some option + self.hashagg_table_size = self.bind('hashagg_table_size') + self.check_implicit_conversions = self.bind('check_implicit_conversions') + self.max_recursive_times = self.bind('max_recursive_times') + + self.enable_absolute_tablespace = self.bind('enable_absolute_tablespace') + self.enable_kill_query = self.bind('enable_kill_query') + self.enforce_a_behavior = self.bind('enforce_a_behavior') + + def calculate(self): + self.sql_beta_feature.set('partition_opfusion') diff --git a/script/impl/perf_config/tuners/gucs/other.py b/script/impl/perf_config/tuners/gucs/other.py new file mode 100644 index 00000000..cfb0ed89 --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/other.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup + + +class DoubleDbReplicationGUC(GUCTuneGroup): + def __init__(self): + super(DoubleDbReplicationGUC, self).__init__() + self.RepOriginId = self.bind('RepOriginId') + + def calculate(self): + pass + +class AiGUC(GUCTuneGroup): + def __init__(self): + super(AiGUC, self).__init__() + self.db4ai_snapshot_mode = self.bind('db4ai_snapshot_mode') + self.db4ai_snapshot_version_delimiter = self.bind('db4ai_snapshot_version_delimiter') + self.db4ai_snapshot_version_separator = self.bind('db4ai_snapshot_version_separator') + self.enable_ai_stats = self.bind('enable_ai_stats') + self.enable_cachedplan_mgr = self.bind('enable_cachedplan_mgr') + + def calculate(self): + pass + + +class DcfGUC(GUCTuneGroup): + def __init__(self): + super(DcfGUC, self).__init__() + self.enable_dcf = self.bind('enable_dcf') + self.dcf_ssl = self.bind('dcf_ssl') + self.dcf_config = self.bind('dcf_config') + self.dcf_data_path = self.bind('dcf_data_path') + self.dcf_log_path = self.bind('dcf_log_path') + self.dcf_node_id = self.bind('dcf_node_id') + self.dcf_max_workers = self.bind('dcf_max_workers') + self.dcf_truncate_threshold = self.bind('dcf_truncate_threshold') + self.dcf_election_timeout = self.bind('dcf_election_timeout') + self.dcf_enable_auto_election_priority = self.bind('dcf_enable_auto_election_priority') + self.dcf_election_switch_threshold = self.bind('dcf_election_switch_threshold') + self.dcf_run_mode = self.bind('dcf_run_mode') + self.dcf_log_level = self.bind('dcf_log_level') + self.dcf_log_backup_file_count = self.bind('dcf_log_backup_file_count') + self.dcf_max_log_file_size = self.bind('dcf_max_log_file_size') + self.dcf_socket_timeout = self.bind('dcf_socket_timeout') + self.dcf_connect_timeout = self.bind('dcf_connect_timeout') + self.dcf_mec_fragment_size = self.bind('dcf_mec_fragment_size') + self.dcf_stg_pool_max_size = self.bind('dcf_stg_pool_max_size') + self.dcf_stg_pool_init_size = self.bind('dcf_stg_pool_init_size') + self.dcf_mec_pool_max_size = self.bind('dcf_mec_pool_max_size') + self.dcf_flow_control_disk_rawait_threshold = self.bind('dcf_flow_control_disk_rawait_threshold') + self.dcf_flow_control_net_queue_message_num_threshold = self.bind('dcf_flow_control_net_queue_message_num_threshold') + self.dcf_flow_control_cpu_threshold = self.bind('dcf_flow_control_cpu_threshold') + self.dcf_mec_batch_size = self.bind('dcf_mec_batch_size') + self.dcf_mem_pool_max_size = self.bind('dcf_mem_pool_max_size') + self.dcf_mem_pool_init_size = self.bind('dcf_mem_pool_init_size') + self.dcf_compress_algorithm = self.bind('dcf_compress_algorithm') + self.dcf_compress_level = self.bind('dcf_compress_level') + self.dcf_mec_channel_num = self.bind('dcf_mec_channel_num') + self.dcf_rep_append_thread_num = self.bind('dcf_rep_append_thread_num') + self.dcf_mec_agent_thread_num = self.bind('dcf_mec_agent_thread_num') + self.dcf_mec_reactor_thread_num = self.bind('dcf_mec_reactor_thread_num') + self.dcf_log_file_permission = self.bind('dcf_log_file_permission') + self.dcf_log_path_permission = self.bind('dcf_log_path_permission') + self.dcf_majority_groups = self.bind('dcf_majority_groups') + + def calculate(self): + self.enable_dcf.turn_off() + + +class NvmGUC(GUCTuneGroup): + def __init__(self): + super(NvmGUC, self).__init__() + self.enable_nvm = self.bind('enable_nvm') + self.nvm_buffers = self.bind('nvm_buffers') + self.nvm_file_path = self.bind('nvm_file_path') + self.bypass_nvm = self.bind('bypass_nvm') + self.bypass_dram = self.bind('bypass_dram') + + def calculate(self): + self.enable_nvm.turn_off() + + +class FaultToleranceGUC(GUCTuneGroup): + def __init__(self): + super(FaultToleranceGUC, self).__init__() + self.exit_on_error = self.bind('exit_on_error') + self.restart_after_crash = self.bind('restart_after_crash') + self.omit_encoding_error = self.bind('omit_encoding_error') + self.cn_send_buffer_size = self.bind('cn_send_buffer_size') + self.max_cn_temp_file_size = self.bind('max_cn_temp_file_size') + self.retry_ecode_list = self.bind('retry_ecode_list') + self.data_sync_retry = self.bind('data_sync_retry') + self.remote_read_mode = self.bind('remote_read_mode') + + def calculate(self): + pass + + +class HyperLogLogGUC(GUCTuneGroup): + def __init__(self): + super(HyperLogLogGUC, self).__init__() + self.hll_default_log2m = self.bind('hll_default_log2m') + self.hll_default_log2explicit = self.bind('hll_default_log2explicit') + self.hll_default_log2sparse = self.bind('hll_default_log2sparse') + self.hll_duplicate_check = self.bind('hll_duplicate_check') + + def calculate(self): + pass + + +class StandbyIUDGuc(GUCTuneGroup): + def __init__(self): + super(StandbyIUDGuc, self).__init__() + self.enable_remote_excute = self.bind('enable_remote_excute') + + def calculate(self): + pass + + +class DevelopOptionGUC(GUCTuneGroup): + def __init__(self): + super(DevelopOptionGUC, self).__init__() + self.allow_system_table_mods = self.bind('allow_system_table_mods') + self.debug_assertions = self.bind('debug_assertions') + self.ignore_checksum_failure = self.bind('ignore_checksum_failure') + self.ignore_system_indexes = self.bind('ignore_system_indexes') + self.post_auth_delay = self.bind('post_auth_delay') + self.pre_auth_delay = self.bind('pre_auth_delay') + self.trace_notify = self.bind('trace_notify') + self.trace_recovery_messages = self.bind('trace_recovery_messages') + self.trace_sort = self.bind('trace_sort') + self.zero_damaged_pages = self.bind('zero_damaged_pages') + self.remotetype = self.bind('remotetype') + self.max_user_defined_exception = self.bind('max_user_defined_exception') + self.enable_fast_numeric = self.bind('enable_fast_numeric') + self.enable_compress_spill = self.bind('enable_compress_spill') + self.resource_track_log = self.bind('resource_track_log') + self.show_acce_estimate_detail = self.bind('show_acce_estimate_detail') + self.support_batch_bind = self.bind('support_batch_bind') + self.numa_distribute_mode = self.bind('numa_distribute_mode') + self.log_pagewriter = self.bind('log_pagewriter') + self.advance_xlog_file_num = self.bind('advance_xlog_file_num') + self.enable_beta_opfusion = self.bind('enable_beta_opfusion') + self.enable_csqual_pushdown = self.bind('enable_csqual_pushdown') + self.string_hash_compatible = self.bind('string_hash_compatible') + self.pldebugger_timeout = self.bind('pldebugger_timeout') + self.plsql_show_all_error = self.bind('plsql_show_all_error') + + def calculate(self): + pass + + +class UndoGUC(GUCTuneGroup): + def __init__(self): + super(UndoGUC, self).__init__() + self.undo_space_limit_size = self.bind('undo_space_limit_size') + self.undo_limit_size_per_transaction = self.bind('undo_limit_size_per_transaction') + self.max_undo_workers = self.bind('max_undo_workers') + + self.enable_recyclebin = self.bind('enable_recyclebin') + self.recyclebin_retention_time = self.bind('recyclebin_retention_time') + self.version_retention_age = self.bind('version_retention_age') + self.undo_retention_time = self.bind('undo_retention_time') + + def calculate(self): + pass + + +class OtherDefaultGUC(GUCTuneGroup): + def __init__(self): + super(OtherDefaultGUC, self).__init__() + self.dynamic_library_path = self.bind('dynamic_library_path') + self.gin_fuzzy_search_limit = self.bind('gin_fuzzy_search_limit') + self.local_preload_libraries = self.bind('local_preload_libraries') + + def calculate(self): + pass + + +class OtherOptionsGUC(GUCTuneGroup): + def __init__(self): + super(OtherOptionsGUC, self).__init__() + self.enable_default_ustore_table = self.bind('enable_default_ustore_table') + self.enable_ustore = self.bind('enable_ustore') + self.reserve_space_for_nullable_atts = self.bind('reserve_space_for_nullable_atts') + self.ustore_attr = self.bind('ustore_attr') + self.server_version = self.bind('server_version') + self.server_version_num = self.bind('server_version_num') + self.block_size = self.bind('block_size') + self.segment_size = self.bind('segment_size') + self.max_index_keys = self.bind('max_index_keys') + self.integer_datetimes = self.bind('integer_datetimes') + self.lc_collate = self.bind('lc_collate') + self.lc_ctype = self.bind('lc_ctype') + self.max_identifier_length = self.bind('max_identifier_length') + self.server_encoding = self.bind('server_encoding') + self.enable_upgrade_merge_lock_mode = self.bind('enable_upgrade_merge_lock_mode') + self.transparent_encrypted_string = self.bind('transparent_encrypted_string') + self.transparent_encrypt_kms_url = self.bind('transparent_encrypt_kms_url') + self.transparent_encrypt_kms_region = self.bind('transparent_encrypt_kms_region') + self.basebackup_timeout = self.bind('basebackup_timeout') + self.datanode_heartbeat_interval = self.bind('datanode_heartbeat_interval') + self.max_concurrent_autonomous_transactions = self.bind('max_concurrent_autonomous_transactions') + self.enable_seqscan_fusion = self.bind('enable_seqscan_fusion ') + self.sql_ignore_strategy = self.bind('sql_ignore_strategy') + + def calculate(self): + pass + diff --git a/script/impl/perf_config/tuners/gucs/security.py b/script/impl/perf_config/tuners/gucs/security.py new file mode 100644 index 00000000..a6cc57f7 --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/security.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.tuner import Tuner, TunerGroup +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup +from impl.perf_config.probes.business import BsScenario + + +class SecurityGUC(GUCTuneGroup): + def __init__(self): + super(SecurityGUC, self).__init__() + # connect and timeout + self.authentication_timeout = self.bind('authentication_timeout') + self.auth_iteration_count = self.bind('auth_iteration_count') + self.session_authorization = self.bind('session_authorization') + self.session_timeout = self.bind('session_timeout') + self.idle_in_transaction_session_timeout = self.bind('idle_in_transaction_session_timeout') + # ssl + self.ssl = self.bind('ssl') + self.require_ssl = self.bind('require_ssl') + self.ssl_ciphers = self.bind('ssl_ciphers') + self.ssl_renegotiation_limit = self.bind('ssl_renegotiation_limit') + self.ssl_cert_file = self.bind('ssl_cert_file') + self.ssl_key_file = self.bind('ssl_key_file') + self.ssl_ca_file = self.bind('ssl_ca_file') + self.ssl_crl_file = self.bind('ssl_crl_file') + # krb + self.krb_server_keyfile = self.bind('krb_server_keyfile') + self.krb_srvname = self.bind('krb_srvname') + self.krb_caseins_users = self.bind('krb_caseins_users') + # password + self.password_policy = self.bind('password_policy') + self.password_reuse_time = self.bind('password_reuse_time') + self.password_reuse_max = self.bind('password_reuse_max') + self.password_lock_time = self.bind('password_lock_time') + self.password_encryption_type = self.bind('password_encryption_type') + self.password_min_length = self.bind('password_min_length') + self.password_max_length = self.bind('password_max_length') + self.password_min_uppercase = self.bind('password_min_uppercase') + self.password_min_lowercase = self.bind('password_min_lowercase') + self.password_min_digital = self.bind('password_min_digital') + self.password_min_special = self.bind('password_min_special') + self.password_effect_time = self.bind('password_effect_time') + self.password_notify_time = self.bind('password_notify_time') + self.modify_initial_password = self.bind('modify_initial_password') + # config + self.failed_login_attempts = self.bind('failed_login_attempts') + self.elastic_search_ip_addr = self.bind('elastic_search_ip_addr') + self.enable_security_policy = self.bind('enable_security_policy') + self.use_elastic_search = self.bind('use_elastic_search') + self.is_sysadmin = self.bind('is_sysadmin') + self.enable_tde = self.bind('enable_tde') + self.tde_cmk_id = self.bind('tde_cmk_id') + self.block_encryption_mode = self.bind('block_encryption_mode') + self.enableSeparationOfDuty = self.bind('enableSeparationOfDuty') + self.enable_nonsysadmin_execute_direct = self.bind('enable_nonsysadmin_execute_direct') + self.enable_access_server_directory = self.bind('enable_access_server_directory') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.ssl.turn_off() + self.enable_security_policy.turn_off() + self.use_elastic_search.turn_off() + self.enable_tde.turn_off() + self.enableSeparationOfDuty.turn_off() + + +class AuditGUC(GUCTuneGroup): + def __init__(self): + super(AuditGUC, self).__init__() + self.audit_enabled = self.bind('audit_enabled') + self.audit_directory = self.bind('audit_directory') + self.audit_data_format = self.bind('audit_data_format') + self.audit_rotation_interval = self.bind('audit_rotation_interval') + self.audit_rotation_size = self.bind('audit_rotation_size') + self.audit_resource_policy = self.bind('audit_resource_policy') + self.audit_file_remain_time = self.bind('audit_file_remain_time') + self.audit_space_limit = self.bind('audit_space_limit') + self.audit_file_remain_threshold = self.bind('audit_file_remain_threshold') + self.audit_thread_num = self.bind('audit_thread_num') + + self.audit_login_logout = self.bind('audit_login_logout') + self.audit_database_process = self.bind('audit_database_process') + self.audit_user_locked = self.bind('audit_user_locked') + self.audit_user_violation = self.bind('audit_user_violation') + self.audit_grant_revoke = self.bind('audit_grant_revoke') + self.full_audit_users = self.bind('full_audit_users') + self.no_audit_client = self.bind('no_audit_client') + + self.audit_system_object = self.bind('audit_system_object') + self.audit_dml_state = self.bind('audit_dml_state') + self.audit_dml_state_select = self.bind('audit_dml_state_select') + self.audit_function_exec = self.bind('audit_function_exec') + self.audit_system_function_exec = self.bind('audit_system_function_exec') + self.audit_copy_exec = self.bind('audit_copy_exec') + self.audit_set_parameter = self.bind('audit_set_parameter') + self.audit_xid_info = self.bind('audit_xid_info') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario == BsScenario.TP_PERFORMANCE: + self.audit_enabled.turn_off() + + diff --git a/script/impl/perf_config/tuners/gucs/storage.py b/script/impl/perf_config/tuners/gucs/storage.py new file mode 100644 index 00000000..0ec3127e --- /dev/null +++ b/script/impl/perf_config/tuners/gucs/storage.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : gs_perfconfg is a utility to optimize system and database configure about openGauss +############################################################################# + +import math +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.guc import GucMap, GUCTuneGroup +from impl.perf_config.probes.business import BsScenario + + +class VacuumGUC(GUCTuneGroup): + def __init__(self): + super(VacuumGUC, self).__init__() + self.vacuum_cost_delay = self.bind('vacuum_cost_delay') + self.vacuum_cost_page_hit = self.bind('vacuum_cost_page_hit') + self.vacuum_cost_page_miss = self.bind('vacuum_cost_page_miss') + self.vacuum_cost_page_dirty = self.bind('vacuum_cost_page_dirty') + self.vacuum_cost_limit = self.bind('vacuum_cost_limit') + self.vacuum_freeze_min_age = self.bind('vacuum_freeze_min_age') + self.vacuum_freeze_table_age = self.bind('vacuum_freeze_table_age') + + self.autovacuum = self.bind('autovacuum') + self.autovacuum_mode = self.bind('autovacuum_mode') + self.autovacuum_io_limits = self.bind('autovacuum_io_limits') + self.autovacuum_max_workers = self.bind('autovacuum_max_workers') + self.autovacuum_naptime = self.bind('autovacuum_naptime') + self.autovacuum_vacuum_threshold = self.bind('autovacuum_vacuum_threshold') + self.autovacuum_analyze_threshold = self.bind('autovacuum_analyze_threshold') + self.autovacuum_vacuum_scale_factor = self.bind('autovacuum_vacuum_scale_factor') + self.autovacuum_analyze_scale_factor = self.bind('autovacuum_analyze_scale_factor') + self.autovacuum_freeze_max_age = self.bind('autovacuum_freeze_max_age') + self.autovacuum_vacuum_cost_delay = self.bind('autovacuum_vacuum_cost_delay') + self.autovacuum_vacuum_cost_limit = self.bind('autovacuum_vacuum_cost_limit') + self.autoanalyze_timeout = self.bind('autoanalyze_timeout') + self.defer_csn_cleanup_time = self.bind('defer_csn_cleanup_time') + self.log_autovacuum_min_duration = self.bind('log_autovacuum_min_duration') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario in [BsScenario.TP_PERFORMANCE, BsScenario.TP_PRODUCE]: + self.autovacuum.turn_on() + self.autovacuum_mode.set('vacuum') + + autovacuum_max_workers = min(max(math.floor(infos.cpu.count() * 0.15), 1), 20) + self.autovacuum_max_workers.set(str(autovacuum_max_workers)) + + autovacuum_naptime = 5 + self.autovacuum_naptime.set(f'{autovacuum_naptime}s') + + self.autovacuum_vacuum_cost_delay.set('10') + self.autovacuum_vacuum_scale_factor.set('0.1') + self.autovacuum_analyze_scale_factor.set('0.02') + + +class CheckpointGUC(GUCTuneGroup): + def __init__(self): + super(CheckpointGUC, self).__init__() + self.checkpoint_segments = self.bind('checkpoint_segments') + self.checkpoint_timeout = self.bind('checkpoint_timeout') + self.checkpoint_completion_target = self.bind('checkpoint_completion_target') + self.checkpoint_warning = self.bind('checkpoint_warning') + self.checkpoint_wait_timeout = self.bind('checkpoint_wait_timeout') + self.enable_incremental_checkpoint = self.bind('enable_incremental_checkpoint') + self.incremental_checkpoint_timeout = self.bind('incremental_checkpoint_timeout') + self.enable_xlog_prune = self.bind('enable_xlog_prune') + self.max_redo_log_size = self.bind('max_redo_log_size') + self.max_size_for_xlog_prune = self.bind('max_size_for_xlog_prune') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario in [BsScenario.TP_PERFORMANCE, BsScenario.TP_PRODUCE]: + + self.enable_incremental_checkpoint.turn_on() + self.incremental_checkpoint_timeout.set('5min') + self.enable_xlog_prune.turn_off() + + +class BackendWriteThreadGUC(GUCTuneGroup): + def __init__(self): + super(BackendWriteThreadGUC, self).__init__() + self.candidate_buf_percent_target = self.bind('candidate_buf_percent_target') + self.bgwriter_delay = self.bind('bgwriter_delay') + self.bgwriter_lru_maxpages = self.bind('bgwriter_lru_maxpages') + self.bgwriter_lru_multiplier = self.bind('bgwriter_lru_multiplier') + self.bgwriter_flush_after = self.bind('bgwriter_flush_after') + self.dirty_page_percent_max = self.bind('dirty_page_percent_max') + self.pagewriter_thread_num = self.bind('pagewriter_thread_num') + self.pagewriter_sleep = self.bind('pagewriter_sleep') + self.max_io_capacity = self.bind('max_io_capacity') + self.enable_consider_usecount = self.bind('enable_consider_usecount') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.business.scenario in [BsScenario.TP_PERFORMANCE, BsScenario.TP_PRODUCE]: + self.bgwriter_delay.set('5s') + self.bgwriter_flush_after.set('32') + self.candidate_buf_percent_target.set('0.7') + + +class DoubleWriteGUC(GUCTuneGroup): + def __init__(self): + super(DoubleWriteGUC, self).__init__() + self.enable_double_write = self.bind('enable_double_write') + self.dw_file_num = self.bind('dw_file_num') + self.dw_file_size = self.bind('dw_file_size') + + def calculate(self): + self.enable_double_write.turn_on() + + +class AsyncIoGUC(GUCTuneGroup): + def __init__(self): + super(AsyncIoGUC, self).__init__() + self.enable_adio_debug = self.bind('enable_adio_debug') + self.enable_adio_function = self.bind('enable_adio_function') + self.enable_fast_allocate = self.bind('enable_fast_allocate') + self.prefetch_quantity = self.bind('prefetch_quantity') + self.backwrite_quantity = self.bind('backwrite_quantity') + self.cstore_prefetch_quantity = self.bind('cstore_prefetch_quantity') + self.cstore_backwrite_quantity = self.bind('cstore_backwrite_quantity') + self.cstore_backwrite_max_threshold = self.bind('cstore_backwrite_max_threshold') + self.fast_extend_file_size = self.bind('fast_extend_file_size') + self.effective_io_concurrency = self.bind('effective_io_concurrency') + self.checkpoint_flush_after = self.bind('checkpoint_flush_after') + self.backend_flush_after = self.bind('backend_flush_after') + + def calculate(self): + pass + + +class WalGUC(GUCTuneGroup): + def __init__(self): + super(WalGUC, self).__init__() + self.wal_level = self.bind('wal_level') + self.fsync = self.bind('fsync') + self.synchronous_commit = self.bind('synchronous_commit') + self.wal_sync_method = self.bind('wal_sync_method') + self.full_page_writes = self.bind('full_page_writes') + self.wal_log_hints = self.bind('wal_log_hints') + self.wal_buffers = self.bind('wal_buffers') + self.wal_writer_delay = self.bind('wal_writer_delay') + self.commit_delay = self.bind('commit_delay') + self.commit_siblings = self.bind('commit_siblings') + self.wal_block_size = self.bind('wal_block_size') + self.wal_segment_size = self.bind('wal_segment_size') + self.walwriter_cpu_bind = self.bind('walwriter_cpu_bind') + self.walwriter_sleep_threshold = self.bind('walwriter_sleep_threshold') + self.wal_file_init_num = self.bind('wal_file_init_num') + self.xlog_file_path = self.bind('xlog_file_path') + self.xlog_file_size = self.bind('xlog_file_size') + self.xlog_lock_file_path = self.bind('xlog_lock_file_path') + self.force_promote = self.bind('force_promote') + self.wal_flush_timeout = self.bind('wal_flush_timeout') + self.wal_flush_delay = self.bind('wal_flush_delay') + self.autocommit = self.bind('autocommit') + + def calculate(self): + pass + + +class RecoveryGUC(GUCTuneGroup): + def __init__(self): + super(RecoveryGUC, self).__init__() + self.recovery_time_target = self.bind('recovery_time_target') + self.recovery_max_workers = self.bind('recovery_max_workers') + self.recovery_parse_workers = self.bind('recovery_parse_workers') + self.recovery_redo_workers = self.bind('recovery_redo_workers') + self.recovery_parallelism = self.bind('recovery_parallelism') + self.enable_page_lsn_check = self.bind('enable_page_lsn_check') + self.recovery_min_apply_delay = self.bind('recovery_min_apply_delay') + self.redo_bind_cpu_attr = self.bind('redo_bind_cpu_attr') + + def calculate(self): + pass + + +class BackoutRecoveryGUC(GUCTuneGroup): + def __init__(self): + super(BackoutRecoveryGUC, self).__init__() + self.operation_mode = self.bind('operation_mode') + self.enable_cbm_tracking = self.bind('enable_cbm_tracking') + self.hadr_max_size_for_xlog_receiver = self.bind('hadr_max_size_for_xlog_receiver') + + def calculate(self): + pass + + +class ArchiveGUC(GUCTuneGroup): + def __init__(self): + super(ArchiveGUC, self).__init__() + self.archive_mode = self.bind('archive_mode') + self.archive_command = self.bind('archive_command') + self.archive_dest = self.bind('archive_dest') + self.archive_timeout = self.bind('archive_timeout') + self.archive_interval = self.bind('archive_interval') + + def calculate(self): + pass + + +class LockManagerGUC(GUCTuneGroup): + def __init__(self): + super(LockManagerGUC, self).__init__() + self.deadlock_timeout = self.bind('deadlock_timeout') + self.lockwait_timeout = self.bind('lockwait_timeout') + self.update_lockwait_timeout = self.bind('update_lockwait_timeout') + self.max_locks_per_transaction = self.bind('max_locks_per_transaction') + self.max_pred_locks_per_transaction = self.bind('max_pred_locks_per_transaction') + self.gs_clean_timeout = self.bind('gs_clean_timeout') + self.partition_lock_upgrade_timeout = self.bind('partition_lock_upgrade_timeout') + self.fault_mon_timeout = self.bind('fault_mon_timeout') + self.enable_online_ddl_waitlock = self.bind('enable_online_ddl_waitlock') + self.xloginsert_locks = self.bind('xloginsert_locks') + self.num_internal_lock_partitions = self.bind('num_internal_lock_partitions') + + def calculate(self): + self.update_lockwait_timeout.set('40min') + + +class TransactionGUC(GUCTuneGroup): + def __init__(self): + super(TransactionGUC, self).__init__() + self.transaction_isolation = self.bind('transaction_isolation') + self.transaction_read_only = self.bind('transaction_read_only') + self.xc_maintenance_mode = self.bind('xc_maintenance_mode') + self.allow_concurrent_tuple_update = self.bind('allow_concurrent_tuple_update') + self.transaction_deferrable = self.bind('transaction_deferrable') + self.enable_show_any_tuples = self.bind('enable_show_any_tuples') + self.replication_type = self.bind('replication_type') + self.enable_defer_calculate_snapshot = self.bind('enable_defer_calculate_snapshot') + + def calculate(self): + pass + + +class SharedStorageGUC(GUCTuneGroup): + def __init__(self): + super(SharedStorageGUC, self).__init__() + self.ss_enable_dss = self.bind('ss_enable_dss') + self.ss_enable_dms = self.bind('ss_enable_dms') + self.ss_enable_ssl = self.bind('ss_enable_ssl') + self.ss_enable_catalog_centralized = self.bind('ss_enable_catalog_centralized') + self.ss_instance_id = self.bind('ss_instance_id') + self.ss_dss_vg_name = self.bind('ss_dss_vg_name') + self.ss_dss_conn_path = self.bind('ss_dss_conn_path') + self.ss_interconnect_channel_count = self.bind('ss_interconnect_channel_count') + self.ss_work_thread_count = self.bind('ss_work_thread_count') + self.ss_recv_msg_pool_size = self.bind('ss_recv_msg_pool_size') + self.ss_interconnect_type = self.bind('ss_interconnect_type') + self.ss_interconnect_url = self.bind('ss_interconnect_url') + self.ss_rdma_work_config = self.bind('ss_rdma_work_config') + self.ss_ock_log_path = self.bind('ss_ock_log_path') + self.ss_enable_scrlock = self.bind('ss_enable_scrlock ') + self.ss_enable_scrlock_sleep_mode = self.bind('ss_enable_scrlock_sleep_mode ') + self.ss_scrlock_server_port = self.bind('ss_scrlock_server_port ') + self.ss_scrlock_worker_count = self.bind('ss_scrlock_worker_count ') + self.ss_scrlock_worker_bind_core = self.bind('ss_scrlock_worker_bind_core ') + self.ss_scrlock_server_bind_core = self.bind('ss_scrlock_server_bind_core ') + self.ss_log_level = self.bind('ss_log_level') + self.ss_log_backup_file_count = self.bind('ss_log_backup_file_count') + self.ss_log_max_file_size = self.bind('ss_log_max_file_size') + self.ss_enable_aio = self.bind('ss_enable_aio') + self.ss_enable_verify_page = self.bind('ss_enable_verify_page') + self.ss_enable_reform = self.bind('ss_enable_reform') + self.ss_parallel_thread_count = self.bind('ss_parallel_thread_count') + self.ss_enable_ondemand_recovery = self.bind('ss_enable_ondemand_recovery') + self.ss_ondemand_recovery_mem_size = self.bind('ss_ondemand_recovery_mem_size') + self.ss_enable_bcast_snapshot = self.bind('ss_enable_bcast_snapshot') + + def calculate(self): + pass diff --git a/script/impl/perf_config/tuners/os.py b/script/impl/perf_config/tuners/os.py new file mode 100644 index 00000000..dda78aa5 --- /dev/null +++ b/script/impl/perf_config/tuners/os.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : perf_probe.py setup a information set for configure +############################################################################# + +import math +from base_utils.os.cpu_util import CpuArchitecture, CpuUtil +from base_utils.os.disk_util import DiskUtil + +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.anti import AntiLog +from impl.perf_config.basic.tuner import Tuner, TunerGroup, ShellTunePoint +from impl.perf_config.probes.business import BsScenario + + +################################################ +# BIOS +# - BiosTuner +################################################ +class BiosTuner(TunerGroup): + def __init__(self): + super(BiosTuner, self).__init__() + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.os.bios.support_smmu == 'Enable': + sug = 'modify BIOS->Advances->MISC Config->Support Smmu to Disable.' + Project.report.suggest(sug) + Project.log('SUGGESTION: ' + sug) + if infos.os.bios.cpu_prefetching == 'Enable': + sug = 'modify BIOS->Advances->MISC Config->CPU Prefetching Configuration to Disable.' + Project.report.suggest(sug) + Project.log('SUGGESTION: ' + sug) + if infos.os.bios.die_interleaving == 'Enable': + sug = 'modify BIOS->Advances->MISC Config->Die Interleaving to Disable.' + Project.report.suggest(sug) + Project.log('SUGGESTION: ' + sug) + + +################################################ +# CPU +# - CPUTuner +################################################ +class CPUTuner(TunerGroup): + def __init__(self): + super(CPUTuner, self).__init__() + + @staticmethod + def calculate_numa_bind(): + infos = Project.getGlobalPerfProbe() + session_per_core = infos.business.parallel / infos.cpu.count() + numa_bind = (infos.cpu.architecture() == CpuArchitecture.AARCH64 and + infos.cpu.count() >= 32 and + BsScenario.isOLTPScenario(infos.business.scenario) and + session_per_core > 1) + + numa_bind_info = {'use': False, 'threadpool': [], 'network': '', 'walwriter': '', 'suggestions': []} + if numa_bind: + numa_bind_info['use'] = True + ratio_for_net = 0.1 # cpu num for irq-bind / all cpu, 0.1 is ref tpcc 2P 4P. + threadpool = [] + network = [] + for cpu_list in infos.cpu.numa(): + cpu_count = len(cpu_list) + count_for_net = max(math.floor(cpu_count * ratio_for_net), 1) + threadpool += cpu_list[:cpu_count - count_for_net] + network += cpu_list[cpu_count - count_for_net:] + walwriter = [0] + numactl_param = CpuUtil.cpuListToCpuRangeStr(threadpool) + threadpool.remove(0) + numa_bind_info['walwriter'] = CpuUtil.cpuListToCpuRangeStr(walwriter) + numa_bind_info['threadpool'] = threadpool + numa_bind_info['network'] = CpuUtil.cpuListToCpuRangeStr(network) + + suggestion = "If you use numactl for startup, you are advised to add param '-C {0}'{1}.".format( + numactl_param, + '' if infos.cpu.count() < 256 or len(infos.cpu.numa) < 4 else "and '--preferred=0'." + ) + numa_bind_info['suggestions'].append(suggestion) + return numa_bind_info + return numa_bind_info + + def calculate(self): + infos = Project.getGlobalPerfProbe() + numa_bind_info = self.calculate_numa_bind() + infos.cpu.notebook.write('numa_bind_info', numa_bind_info) + + def explain(self, apply=False): + # tune threadpool in dbtuner and network in nerwork tuner + pass + + +################################################ +# MEMORY +# - MemoryTuner +# - MemHugePageTuner +############################################### +class MemHugePageTuner(TunerGroup): + def __init__(self): + super(MemHugePageTuner, self).__init__() + + def calculate(self): + infos = Project.getGlobalPerfProbe() + if infos.memory.hugepage is None: + return + + self._disable_huge_mem(infos) + + def _disable_huge_mem(self, infos): + tune_enabled = (infos.memory.hugepage['enabled'] != 'never') + if tune_enabled: + cmd = "echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled" + anti = "echo '{}' > /sys/kernel/mm/transparent_hugepage/enabled".format( + infos.memory.hugepage['enabled']) + desc = 'Tune hugepage enabled to never, old value is {}'.format(infos.memory.hugepage['enabled']) + self.add(ShellTunePoint(cmd, anti, desc)) + + tune_defrag = (infos.memory.hugepage['defrag'] != 'never') + if tune_defrag: + cmd = "echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag" + anti = "echo '{}' > /sys/kernel/mm/transparent_hugepage/defrag".format( + infos.memory.hugepage['enabled']) + desc = 'Tune hugepage defrag to never, old value is {}'.format(infos.memory.hugepage['enabled']) + self.add(ShellTunePoint(cmd, anti, desc)) + + +class MemoryTuner(TunerGroup): + def __init__(self): + super(MemoryTuner, self).__init__() + self.hugepage = self.add(MemHugePageTuner()) + + +################################################ +# DISK +# - DiskTuner +############################################### +class DiskTuner(TunerGroup): + def __init__(self): + super(DiskTuner, self).__init__() + + def calculate(self): + infos = Project.getGlobalPerfProbe() + gausshome_disk = DiskUtil.getMountPathByDataDir(infos.db.gauss_data) + device = infos.disk.get(gausshome_disk) + if device.fstype != 'xfs' or device.bsize != 8192: + sug = "Disk {0} is {1} format, you are advised to set the disk format to xfs ".format( + device.device, device.fstype + ) + Project.report.suggest(sug) + Project.log('SUGGESTION: ' + sug) + + +################################################ +# DISK +# - NetworkTuner +# - NetworkIRQTuner +# - EthtoolTuner +# - NetConfigTuner +############################################### +class NetworkIRQTuner(TunerGroup): + def __init__(self): + super(NetworkIRQTuner, self).__init__() + self.script = Project.environ.get_builtin_script('irq_operate.sh') + + def calculate(self): + infos = Project.getGlobalPerfProbe() + numa_bind_info = infos.cpu.notebook.read('numa_bind_info') + if numa_bind_info is None or numa_bind_info.get('network') is None: + return + for ip in infos.db.ip: + if ip == '*': + continue + gate = infos.network.get_gate(ip) + if gate is None or gate.is_localhost() or gate.is_virbr(): + continue + + bind_list = CpuUtil.cpuRangeStrToCpuList(numa_bind_info.get('network')) + roll_list = gate.irq_binds + cmd = 'sh {0} bind "{1}"'.format(self.script, ' '.join([str(cpuid) for cpuid in bind_list])) + anti = 'sh {0} bind "{1}"'.format(self.script, ' '.join([str(cpuid) for cpuid in roll_list])) + desc = 'bind irq' + self.add(ShellTunePoint(cmd, anti, desc)) + + +class OffloadingTuner(TunerGroup): + def __init__(self): + super(OffloadingTuner, self).__init__() + + def calculate(self): + infos = Project.getGlobalPerfProbe() + for ip in infos.db.ip: + if ip == '*': + continue + gate = infos.network.get_gate(ip) + if gate is None or gate.is_localhost() or gate.is_virbr(): + continue + + tune_tso = (gate.tso == 'off') + if tune_tso: + cmd = 'ethtool -K {} tso on'.format(gate.name) + anti = 'ethtool -K {} tso off'.format(gate.name) + desc = 'open offloading tso' + self.add(ShellTunePoint(cmd, anti, desc)) + + tune_lro = (gate.lro == 'off') + if tune_lro: + cmd = 'ethtool -K {} lro on'.format(gate.name) + anti = 'ethtool -K {} lro off'.format(gate.name) + desc = 'open offloading lro' + self.add(ShellTunePoint(cmd, anti, desc)) + + +class NetConfigTuner(TunerGroup): + def __init__(self): + super(NetConfigTuner, self).__init__() + + def calculate(self): + pass + + +class NetworkTuner(TunerGroup): + def __init__(self): + super(NetworkTuner, self).__init__() + self.network_irq = self.add(NetworkIRQTuner()) + self.offloading = self.add(OffloadingTuner()) + self.net_config = self.add(NetConfigTuner()) + + +################################################ +# OS CONFIGURE AND SERVICE +# - OSServiceTuner +################################################ +class OSServiceTuner(TunerGroup): + def __init__(self): + super(OSServiceTuner, self).__init__() + + def calculate(self): + infos = Project.getGlobalPerfProbe() + + if infos.os.service.sysmonitor is not None and infos.os.service.sysmonitor: + cmd = "service sysmonitor stop" + anti = "service sysmonitor start" + desc = "stop sysmonitor" + self.add(ShellTunePoint(cmd, anti, desc)) + + if infos.os.service.irqbalance is not None and infos.os.service.irqbalance: + cmd = "service irqbalance stop" + anti = "service irqbalance start" + desc = "stop irqbalance" + self.add(ShellTunePoint(cmd, anti, desc)) + + +################################################ +# OS TUNER +# - OSTuner +# - BiosTuner +# - CPUTuner +# - MemoryTuner +# - DiskTuner +# - NetworkTuner +# - OSServiceTuner +############################################### +class OSTuner(TunerGroup): + def __init__(self): + super(OSTuner, self).__init__() + self.bios = self.add(BiosTuner()) + self.cpu = self.add(CPUTuner()) + self.memory = self.add(MemoryTuner()) + self.disk = self.add(DiskTuner()) + self.network = self.add(NetworkTuner()) + self.service = self.add(OSServiceTuner()) diff --git a/script/impl/perf_config/tuners/setup.py b/script/impl/perf_config/tuners/setup.py new file mode 100644 index 00000000..d7c4965d --- /dev/null +++ b/script/impl/perf_config/tuners/setup.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +############################################################################# +# Copyright (c) 2023 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- +# Description : perf_probe.py setup a information set for configure +############################################################################# + +import os +from impl.perf_config.basic.project import Project +from impl.perf_config.basic.tuner import Tuner, TunerGroup, ShellTunePoint +from impl.perf_config.probes.db import DBInfo +from impl.perf_config.probes.business import BusinessProbe + + +class SetupTuner(TunerGroup): + def __init__(self): + super(SetupTuner, self).__init__() + + def calculate(self): + self._calculate_isolated_xlog() + + def _calculate_isolated_xlog(self): + infos = Project.getGlobalPerfProbe() + if infos.business.isolated_xlog is None: + return + + script = Project.environ.get_builtin_script('isolated_xlog.sh') + old_path = os.path.join(infos.db.gauss_data, 'pg_xlog') + new_path = os.path.join(infos.business.isolated_xlog, 'pg_xlog') + + cmd = f'sh {script} isolated {old_path} {new_path}' + anti = f'sh {script} recover {old_path} {new_path}' + desc = 'Storing wal on a separate disk.' + + self.add(ShellTunePoint(cmd, anti, desc)) + diff --git a/script/impl/preinstall/OLAP/PreinstallImplOLAP.py b/script/impl/preinstall/OLAP/PreinstallImplOLAP.py index 5c9e05d1..4fcae5b3 100644 --- a/script/impl/preinstall/OLAP/PreinstallImplOLAP.py +++ b/script/impl/preinstall/OLAP/PreinstallImplOLAP.py @@ -314,7 +314,7 @@ class PreinstallImplOLAP(PreinstallImpl): """ self.context.clusterInfo.corePath = \ self.context.clusterInfo.readClustercorePath(self.context.xmlFile) - if not self.context.clusterInfo.corePath: + if not self.context.clusterInfo.corePath or not self.context.current_user_root: return self.context.logger.log("Setting Core file", "addStep") try: @@ -362,7 +362,8 @@ class PreinstallImplOLAP(PreinstallImpl): psshlib_path = os.path.join( os.path.dirname(__file__), "../../../gspylib/pssh/bin/TaskPool.py") - dest_path = "/usr/bin/" + # 将pssh放到om tools的bin目录下 + dest_path = os.path.join(self.context.clusterToolPath, "bin") secbox_path = "/var/chroot/usr/bin/" cmd = "cp %s %s %s %s" % ( pssh_path, pscp_path, psshlib_path, dest_path) @@ -399,13 +400,15 @@ class PreinstallImplOLAP(PreinstallImpl): self.context.logger.log("Setting host ip env", "addStep") try: # remove HOST_IP info with /etc/profile and environ - cmd = "sed -i '/^export[ ]*HOST_IP=/d' /etc/profile" - CmdExecutor.execCommandWithMode( - cmd, - self.context.sshTool, - self.context.localMode or self.context.isSingle) - if "HOST_IP" in os.environ.keys(): - os.environ.pop("HOST_IP") + if self.context.current_user_root: + # remove HOST_IP info with /etc/profile and environ + cmd = "sed -i '/^export[ ]*HOST_IP=/d' /etc/profile" + CmdExecutor.execCommandWithMode( + cmd, + self.context.sshTool, + self.context.localMode or self.context.isSingle) + if "HOST_IP" in os.environ.keys(): + os.environ.pop("HOST_IP") except Exception as e: raise Exception(str(e)) self.context.logger.log("Successfully set host ip env.", "constant") @@ -416,6 +419,8 @@ class PreinstallImplOLAP(PreinstallImpl): input: NA output: NA """ + if self.context.skip_cgroup_set or not self.context.current_user_root: + return self.context.logger.log("Setting Cgroup.", "addStep") try: # set the cgroup @@ -483,6 +488,8 @@ class PreinstallImplOLAP(PreinstallImpl): input: NA output: NA """ + if not self.context.current_user_root: + return # the flag for whether the virtual IP exists flag = 0 # all virtual IPs list diff --git a/script/impl/preinstall/PreinstallImpl.py b/script/impl/preinstall/PreinstallImpl.py index e1f0f622..84ec292b 100644 --- a/script/impl/preinstall/PreinstallImpl.py +++ b/script/impl/preinstall/PreinstallImpl.py @@ -171,7 +171,7 @@ class PreinstallImpl: try: # save the sshIps Ips = [] - # create trust for root + # # create trust for root or current user # get the user name username = pwd.getpwuid(os.getuid()).pw_name # get the user sshIps @@ -193,6 +193,12 @@ class PreinstallImpl: """ if self.context.preMode: return + if not self.context.current_user_root: + # Create user mutual trust, and after execution, no mutual trust will be created + self.context.create_user_ssh_trust = True + self.createTrustForCommonUser() + self.context.create_user_ssh_trust = False + return # Ask to create trust for root flag = input("Are you sure you want to create trust for root (yes/no)?") while True: @@ -235,7 +241,7 @@ class PreinstallImpl: return if self.context.preMode or not self.context.root_ssh_agent_flag: return - if not self.context.root_delete_flag: + if not self.context.root_delete_flag or not self.context.current_user_root: return self.context.logger.debug("Start Delete root mutual trust") @@ -342,8 +348,6 @@ class PreinstallImpl: self.context.logger.log("Distributing package.", "addStep") try: - PackageInfo.makeCompressedToolPackage(self.context.clusterToolPath) - # get the all node names in xml file hosts = self.context.clusterInfo.getClusterNodeNames() # remove the local node name @@ -554,7 +558,10 @@ class PreinstallImpl: cmd_type="python") # check enter permission - cmd = "su - %s -c 'cd '%s''" % (self.context.user, packageDir) + if self.context.current_user_root: + cmd = "su - %s -c 'cd '%s''" % (self.context.user, packageDir) + else: + cmd = "cd '%s'" % packageDir (status, output) = subprocess.getstatusoutput(cmd) # if cmd failed, then exit if status != 0: @@ -574,7 +581,10 @@ class PreinstallImpl: retry_time=15, waite_time=1, link=True) # check enter permission - cmd = "su - %s -c 'cd '%s''" % (self.context.user, user_dir) + if self.context.current_user_root: + cmd = "su - %s -c 'cd '%s''" % (self.context.user, user_dir) + else: + cmd = "cd '%s'" % user_dir (status, output) = subprocess.getstatusoutput(cmd) # if cmd failed, then exit if status != 0: @@ -590,7 +600,10 @@ class PreinstallImpl: # check enter permission log_file_dir = os.path.dirname(self.context.logger.logFile) - cmd = "su - %s -c 'cd '%s''" % (self.context.user, log_file_dir) + if self.context.current_user_root: + cmd = "su - %s -c 'cd '%s''" % (self.context.user, log_file_dir) + else: + cmd = "cd '%s'" % log_file_dir (status, output) = subprocess.getstatusoutput(cmd) # if cmd failed, then exit if status != 0: @@ -703,6 +716,8 @@ class PreinstallImpl: if createTrustFlag: return + if not self.context.current_user_root and not self.context.create_user_ssh_trust: + return self.context.logger.log( "Creating SSH trust for [%s] user." % self.context.user) try: @@ -763,11 +778,10 @@ class PreinstallImpl: output:NA hiden:NA """ + if self.context.localMode or not self.context.current_user_root: + return # single cluster also need to create user without local mode self.context.logger.debug("Creating OS user and create trust for user") - if self.context.localMode: - return - global createTrustFlag try: # check the interactive mode @@ -995,7 +1009,7 @@ class PreinstallImpl: namelist = ",".join(NodeNames) # check skip-os-set parameter - if self.context.skipOSSet: + if self.context.skipOSSet or not self.context.current_user_root: # check the OS parameters self.checkOSParameter(namelist) else: @@ -1088,15 +1102,15 @@ class PreinstallImpl: raise Exception( ErrorCode.GAUSS_524["GAUSS_52400"] + "\nPlease get more details by \"%s " - "-i A -h %s --detail\"." - % (gs_checkos_path, namelist)) + "-i A -h %s %s --detail\"." + % (gs_checkos_path, namelist, self.context.xmlFile)) if warning_num > 0: self.context.logger.log( "Warning: Installation environment " "contains some warning messages." + \ "\nPlease get more details by \"%s " - "-i A -h %s --detail\"." - % (gs_checkos_path, namelist)) + "-i A -h %s %s --detail\"." + % (gs_checkos_path, namelist, self.context.xmlFile)) except Exception as e: raise Exception(str(e)) @@ -1191,6 +1205,14 @@ class PreinstallImpl: self.context.logger.log("Successfully set the dynamic link library.", "constant") + def setArmOptimization(self): + """ + function: set arm optimization + input: NA + output: NA + """ + pass + def setCgroup(self): """ function: setting Cgroup @@ -1271,7 +1293,10 @@ class PreinstallImpl: os.path.join(dirName, "./../../../")) + "/" # check enter permission - cmd = "su - %s -c 'cd '%s''" % (self.context.user, packageDir) + if self.context.current_user_root: + cmd = "su - %s -c 'cd '%s''" % (self.context.user, packageDir) + else: + cmd = "cd '%s'" % packageDir (status, output) = subprocess.getstatusoutput(cmd) # if cmd failed, then exit if status != 0: @@ -1285,7 +1310,10 @@ class PreinstallImpl: # so we need check its exists if os.path.exists(user_dir): # check enter permission - cmd = "su - %s -c 'cd '%s''" % (self.context.user, user_dir) + if self.context.current_user_root: + cmd = "su - %s -c 'cd '%s''" % (self.context.user, user_dir) + else: + cmd = "cd '%s'" % user_dir (status, output) = subprocess.getstatusoutput(cmd) # if cmd failed, then exit if status != 0: @@ -1295,7 +1323,10 @@ class PreinstallImpl: # check enter permission log_file_dir = os.path.dirname(self.context.logger.logFile) - cmd = "su - %s -c 'cd '%s''" % (self.context.user, log_file_dir) + if self.context.current_user_root: + cmd = "su - %s -c 'cd '%s''" % (self.context.user, log_file_dir) + else: + cmd = "cd '%s'" % log_file_dir (status, output) = subprocess.getstatusoutput(cmd) # if cmd failed, then exit if status != 0: @@ -1383,7 +1414,8 @@ class PreinstallImpl: with open(source_file, 'r') as f: env_list = f.readlines() new_env_list = [] - if not self.context.mpprcFile: + # mpprcFile not exist + if not self.context.mpprcFile and self.context.current_user_root: with open(os.path.join("/etc", "profile"), "r") as etc_file: gp_home_env = etc_file.readlines() gphome_env_list = [env.replace('\n', '') for env in gp_home_env] @@ -1537,7 +1569,7 @@ class PreinstallImpl: # failed to Check software raise Exception(str(e)) # Successfully Check software - self.context.logger.log("Successfully check os software.", + self.context.logger.log("Successfully check OS software.", "constant") def get_package_path(self): @@ -1553,19 +1585,19 @@ class PreinstallImpl: """ :return: """ - if not self.context.user_ssh_agent_flag: - return self.context.logger.debug("Start set cron for %s" %self.context.user) tmp_path = ClusterConfigFile.readClusterTmpMppdbPath( self.context.user, self.context.xmlFile) gaussdb_tool_path = ClusterDir.getPreClusterToolPath(self.context.xmlFile) cron_file = "%s/gauss_cron_%s" % (tmp_path, self.context.user) - set_cron_cmd = "crontab -u %s -l > %s && " % (self.context.user, cron_file) + if self.context.current_user_root: + set_cron_cmd = "crontab -u %s -l > %s && " % (self.context.user, cron_file) + else: + set_cron_cmd = "crontab -l > %s && " % cron_file set_cron_cmd += "sed -i '/CheckSshAgent.py/d' %s;" % cron_file set_cron_cmd += "echo '*/1 * * * * source ~/.bashrc;python3 %s/script/local/CheckSshAgent.py >>/dev/null 2>&1 &' >> %s;" % (gaussdb_tool_path, cron_file) - - set_cron_cmd += "crontab -u %s %s;service cron restart;" % (self.context.user, cron_file) set_cron_cmd += "rm -f '%s'" % cron_file + set_cron_cmd += "crontab %s " % cron_file self.context.logger.debug("Command for setting CRON: %s" % set_cron_cmd) CmdExecutor.execCommandWithMode(set_cron_cmd, self.context.sshTool, @@ -1573,6 +1605,19 @@ class PreinstallImpl: self.context.mpprcFile) self.context.logger.debug("Successfully to set cron for %s" %self.context.user) + def do_perf_config(self): + """ + run gs_perfconfig to tune os configure. + """ + if not self.context.enable_perf_config: + return + self.context.logger.log("gs_preinstall has finished, start gs_perfconfig now.") + + cmd = 'gs_perfconfig tune -t os,suggest --apply -y' + if self.context.mpprcFile: + cmd += (' --env ' + self.context.mpprcFile) + CmdExecutor.execCommandLocally(cmd) + def doPreInstall(self): """ function: the main process of preinstall @@ -1636,14 +1681,12 @@ class PreinstallImpl: self.setPssh() # set cgroup self.setCgroup() - + # set arm optimization self.setArmOptimization() # fix server package mode self.fixServerPackageOwner() - # unreg the disk of the dss and about self.dss_init() - # set user cron self.set_user_crontab() # set user env and a flag, @@ -1654,6 +1697,9 @@ class PreinstallImpl: self.context.logger.log("Preinstallation succeeded.") + # gs_perfconfig is not a step in the preinstall, so do it after log succeeded. + self.do_perf_config() + def run(self): """ function: run method diff --git a/script/impl/upgrade/UpgradeImpl.py b/script/impl/upgrade/UpgradeImpl.py index 27671cbb..cff42392 100644 --- a/script/impl/upgrade/UpgradeImpl.py +++ b/script/impl/upgrade/UpgradeImpl.py @@ -2391,6 +2391,7 @@ class UpgradeImpl: self.recordNodeStepInplace(const.ACTION_INPLACE_UPGRADE, const.BINARY_UPGRADE_STEP_UPGRADE_APP) self.installNewBin() + self.cpDolphinUpgradeScript() # 11. restore the cluster config. including this: # cluster_static_config @@ -6908,10 +6909,13 @@ END;""" self.context.logger.debug("Start to copy dolphin upgrade script.") try: - cmd = "if ls %s/share/postgresql/extension/ | grep -qE \"dolphin--(.*)--(.*)sql\" ; " \ + cmd = "if ls %s/share/postgresql/extension/ | grep -qE 'dolphin--(.*)--(.*)sql' ; " \ "then cp -f %s/share/postgresql/extension/dolphin--*--*sql %s/share/postgresql/extension/; fi" % \ (self.context.oldClusterAppPath, self.context.oldClusterAppPath, self.context.newClusterAppPath) - CmdExecutor.execCommandLocally(cmd) + CmdExecutor.execCommandWithMode(cmd, + self.context.sshTool, + self.context.isSingle, + self.context.mpprcFile) self.context.logger.debug("Successfully copy dolphin upgrade script.") except Exception as e: raise Exception("Failed to copy dolphin upgrade script.") diff --git a/script/local/PreInstallUtility.py b/script/local/PreInstallUtility.py index 69e59c32..91e5491a 100644 --- a/script/local/PreInstallUtility.py +++ b/script/local/PreInstallUtility.py @@ -134,6 +134,15 @@ class PreInstall(LocalBaseOM): self.clusterAppPath = "" self.white_list = {} self.logger = None + self.current_user_root = False + + def get_current_user(self): + """ + get current user + """ + user_info = UserUtil.getUserInfo() + if user_info['uid'] == 0: + self.current_user_root = True def initGlobals(self): """ @@ -305,7 +314,7 @@ Common options: if self.logFile == "": self.logFile = ClusterLog.getOMLogPath( - ClusterConstants.LOCAL_LOG_FILE, self.user, "") + ClusterConstants.LOCAL_LOG_FILE, self.user, "", self.clusterConfig) def prepareMpprcFile(self): """ @@ -576,7 +585,10 @@ Common options: # report permisson error about GPHOME FileUtil.checkPathandChangeOwner(originalPath, self.user, DefaultValue.KEY_DIRECTORY_MODE) - cmd = "su - %s -c \"cd '%s'\"" % (username, originalPath) + if self.current_user_root: + cmd = "su - %s -c \"cd '%s'\"" % (username, originalPath) + else: + cmd = "cd '%s'" % originalPath status = subprocess.getstatusoutput(cmd)[0] if status != 0: return False @@ -585,14 +597,21 @@ Common options: return True testFile = os.path.join(originalPath, "touch.tst") - cmd = "su - %s -c 'touch %s && chmod %s %s' >/dev/null 2>&1" % ( - username, testFile, DefaultValue.KEY_FILE_MODE, testFile) + if self.current_user_root: + cmd = "su - %s -c 'touch %s && chmod %s %s' >/dev/null 2>&1" % ( + username, testFile, DefaultValue.KEY_FILE_MODE, testFile) + else: + cmd = "touch %s && chmod %s %s >/dev/null 2>&1" % ( + testFile, DefaultValue.KEY_FILE_MODE, testFile) status = subprocess.getstatusoutput(cmd)[0] if status != 0: return False - cmd = "su - %s -c 'echo aaa > %s' >/dev/null 2>&1" \ - % (username, testFile) + if self.current_user_root: + cmd = "su - %s -c 'echo test > %s' >/dev/null 2>&1" \ + % (username, testFile) + else: + cmd = "echo test > %s >/dev/null 2>&1" % testFile (status, output) = subprocess.getstatusoutput(cmd) if status != 0: cmd = "rm -f '%s' >/dev/null 2>&1" % testFile @@ -612,6 +631,8 @@ Common options: input : NA output: NA """ + if not self.current_user_root: + return self.logger.debug("Checking hostname mapping.") try: self.logger.debug("Change file[/etc/hosts] mode.") @@ -851,6 +872,7 @@ Common options: needCheckEmpty = True self.initNodeInfo() + self.prepare_om_tools_bin_path() self.prepareGaussLogPath() self.prepareInstallPath(needCheckEmpty) self.prepareTmpPath(needCheckEmpty) @@ -971,6 +993,25 @@ Common options: break self.logger.debug("Successfully created dss disk link.") + def prepare_om_tools_bin_path(self): + """ + function: Prepare om tools bin path + input : NA + output: NA + """ + self.logger.debug("Creating om tools bin path.") + bin_path = os.path.join(self.clusterToolPath, "bin") + # check bin path + self.logger.debug("Checking %s directory [%s]." % (VersionInfo.PRODUCT_NAME, bin_path)) + if not os.path.exists(bin_path): + self.makeDirsInRetryMode(bin_path, DefaultValue.MAX_DIRECTORY_MODE) + try: + FileUtil.checkLink(bin_path) + FileUtil.changeMode(DefaultValue.MAX_DIRECTORY_MODE, bin_path, False, "shell") + FileUtil.changeOwner(self.user, bin_path, False, "shell", link=True) + except Exception as e: + raise Exception(str(e)) + def prepareGaussLogPath(self): """ function: Prepare Gausslog Path @@ -1128,8 +1169,11 @@ Common options: if not needCheckEmpty: return upperDir = os.path.dirname(installPath) - cmd = "su - %s -c \"if [ -w %s ];then echo 1; else echo 0;fi\"" % ( - self.user, upperDir) + if self.current_user_root: + cmd = "su - %s -c \"if [ -w %s ];then echo 1; else echo 0;fi\"" % ( + self.user, upperDir) + else: + cmd = "if [ -w %s ];then echo 1; else echo 0;fi" % upperDir self.logger.debug( "Command to check if we have write permission for upper path:" " %s" % cmd) @@ -1168,6 +1212,8 @@ Common options: input : NA output: NA """ + if not self.current_user_root: + return self.logger.debug("Preparing user cron service.") ##1.set crontab file permission crontabFile = "/usr/bin/crontab" @@ -1226,6 +1272,8 @@ Common options: input : NA output: NA """ + if not self.current_user_root: + return self.logger.debug("Preparing user SSHD service.") sshd_config_file = "/etc/ssh/sshd_config" paramName = "MaxStartups" @@ -1387,8 +1435,10 @@ Common options: FileUtil.changeOwner(self.user, omLogPath, True, "shell", retry_flag=True, retry_time=15, waite_time=1, link=True) self.logger.debug("Checking the permission of GPHOME: %s." % user_dir) - cmd = g_file.SHELL_CMD_DICT["checkUserPermission"] % ( - self.user, user_dir) + if self.current_user_root: + cmd = "su - %s -c 'cd %s'" % (self.user, user_dir) + else: + cmd = "cd %s" % user_dir self.logger.debug("The command of check permission is: %s." % cmd) (status, output) = subprocess.getstatusoutput(cmd) if status != 0: @@ -1399,8 +1449,11 @@ Common options: # get the value of GAUSS_ENV self.logger.debug("Setting finish flag.") - cmd = "su - %s -c 'source %s;echo $GAUSS_ENV' 2>/dev/null" % ( - self.user, userProfile) + if self.current_user_root: + cmd = "su - %s -c 'source %s;echo $GAUSS_ENV' 2>/dev/null" % ( + self.user, userProfile) + else: + cmd = "source %s;echo $GAUSS_ENV 2>/dev/null" % userProfile (status, output) = subprocess.getstatusoutput(cmd) if status != 0: raise Exception(ErrorCode.GAUSS_514[ @@ -1421,11 +1474,17 @@ Common options: output: True/False """ if self.mpprcFile != "": - cmd = "su - root -c 'source %s;echo $GAUSS_ENV' 2>/dev/null" \ - % self.mpprcFile + if self.current_user_root: + cmd = "su - %s -c 'source %s;echo $GAUSS_ENV' 2>/dev/null" \ + % (self.user, self.mpprcFile) + else: + cmd = "source %s;echo $GAUSS_ENV 2>/dev/null" % self.mpprcFile else: - cmd = "su - %s -c 'source ~/.bashrc;echo $GAUSS_ENV' 2>/dev/null" \ - % self.user + if self.current_user_root: + cmd = "su - %s -c 'source ~/.bashrc;echo $GAUSS_ENV' 2>/dev/null" \ + % self.user + else: + cmd = "source ~/.bashrc;echo $GAUSS_ENV 2>/dev/null" status, output = subprocess.getstatusoutput(cmd) if status != 0: self.logger.debug( @@ -1503,16 +1562,16 @@ Common options: if self.mpprcFile != "": # have check its exists when check parameters, # so it should exist here - userProfile = self.mpprcFile + user_profile = self.mpprcFile else: # check if os profile exist - userProfile = ClusterConstants.ETC_PROFILE - if not os.path.exists(userProfile): + user_profile = ProfileFile.get_user_bashrc(self.user) + if not os.path.exists(user_profile): self.logger.debug( - "Profile does not exist. Please create %s." % userProfile) - FileUtil.createFile(userProfile) - FileUtil.changeMode(DefaultValue.DIRECTORY_MODE, userProfile) - return userProfile + "Profile does not exist. Please create %s." % user_profile) + FileUtil.createFile(user_profile) + FileUtil.changeMode(DefaultValue.DIRECTORY_MODE, user_profile) + return user_profile def setDBUerProfile(self): """ @@ -1556,7 +1615,7 @@ Common options: # clean GPHOME FileUtil.deleteLine(userProfile, "^\\s*export\\s*GPHOME=.*$") - # clean GPHOME + # clean UNPACKPATH FileUtil.deleteLine(userProfile, "^\\s*export\\s*UNPACKPATH=.*$") self.logger.debug( "Deleting crash GPHOME in user environment variables.") @@ -1673,6 +1732,8 @@ Common options: input : NA output: NA """ + if not self.current_user_root: + return self.logger.debug("Setting Library.") config_file_dir = "/etc/ld.so.conf" alreadySet = False @@ -2588,7 +2649,7 @@ Common options: """ package_path = get_package_path() om_root_path = os.path.dirname(package_path) - if om_root_path == DefaultValue.ROOT_SCRIPTS_PATH: + if om_root_path == DefaultValue.ROOT_SCRIPTS_PATH or not self.current_user_root: return self.logger.log("Separate om root scripts.") @@ -2674,15 +2735,15 @@ Common options: toolPath = self.clusterToolPath self.logger.log("change '%s' files permission and owner." % toolPath) FileUtil.changeOwner(self.user, toolPath, recursive=True, link=True) - FileUtil.changeMode(DefaultValue.KEY_DIRECTORY_MODE, + FileUtil.changeMode(DefaultValue.MAX_DIRECTORY_MODE, toolPath, recursive=True) FileUtil.changeMode(DefaultValue.SPE_FILE_MODE, "%s/script/gs_*" % toolPath) - FileUtil.changeMode(DefaultValue.MIN_FILE_MODE, "%s/*.sha256" % toolPath) - FileUtil.changeMode(DefaultValue.MIN_FILE_MODE, "%s/*.tar.gz" % toolPath) - FileUtil.changeMode(DefaultValue.MIN_FILE_MODE, "%s/*.tar.bz2" % + FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, "%s/*.sha256" % toolPath) + FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, "%s/*.tar.gz" % toolPath) + FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, "%s/*.tar.bz2" % toolPath) - FileUtil.changeMode(DefaultValue.MIN_FILE_MODE, "%s/version.cfg" % + FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, "%s/version.cfg" % toolPath) def fixop_package_path(self): @@ -2695,17 +2756,17 @@ Common options: gsom_path = os.path.dirname(package_path) if gsom_path != DefaultValue.ROOT_SCRIPTS_PATH: self.logger.log("Change file mode in path %s" % package_path) - FileUtil.changeOwner("root", package_path, recursive=True, link=True) + FileUtil.changeOwner(self.user, package_path, recursive=True, link=True) FileUtil.changeMode(DefaultValue.MAX_DIRECTORY_MODE, package_path) FileUtil.changeMode(DefaultValue.KEY_DIRECTORY_MODE, "%s/script" % package_path, recursive=True) - FileUtil.changeMode(DefaultValue.MIN_FILE_MODE, "%s/*.sha256" % + FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, "%s/*.sha256" % package_path) - FileUtil.changeMode(DefaultValue.MIN_FILE_MODE, "%s/*.tar.gz" % + FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, "%s/*.tar.gz" % package_path) - FileUtil.changeMode(DefaultValue.MIN_FILE_MODE, "%s/*.tar.bz2" % + FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, "%s/*.tar.bz2" % package_path) - FileUtil.changeMode(DefaultValue.MIN_FILE_MODE, "%s/version.cfg" % + FileUtil.changeMode(DefaultValue.BIN_FILE_MODE, "%s/version.cfg" % package_path) def fix_dss_cap_permission(self): @@ -2941,6 +3002,7 @@ Common options: output : NA """ try: + self.get_current_user() self.parseCommandLine() self.checkParameter() self.initGlobals() -- Gitee