diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 89fb2dbad82975cec03ac002f50f7196b2b6daa0..0000000000000000000000000000000000000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index 02db4d00b2c815b72582d251162a74111d7ad8f9..31aec5cdf13fc2e970998ec30b8989c9cf4e790f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ build .vscode +.idea +.venv __pycache__ -tools/install_dependency/src/component \ No newline at end of file +tools/install_dependency/src/component +component/LkpTests/compatibility_help/compatibility_testing +component/LkpTests/compatibility_help/compatibility_testing.tar.gz +component/LkpTests/lkp_help/lkp-tests.tar.gz +component/DevkitDistribute/devkit_distribute/config/log.ini +component/DevkitDistribute/devkit_distribute/config/devkit_distribute.yaml \ No newline at end of file diff --git a/common/devkit_utils/devkit_client.py b/common/devkit_utils/devkit_client.py index e508f3889dc75f0ca5584a885e5087d296a4cffe..7e806c9ff41a584f8c193687c0ed9d3bbc198b37 100644 --- a/common/devkit_utils/devkit_client.py +++ b/common/devkit_utils/devkit_client.py @@ -2,7 +2,9 @@ import logging import os.path import requests -from urllib3 import encode_multipart_formdata +import urllib3 + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) class DevKitClient: @@ -69,7 +71,7 @@ class DevKitClient: except OSError as e: logging.exception(e) raise - encoded_data = encode_multipart_formdata(data) + encoded_data = urllib3.encode_multipart_formdata(data) _header = self.header.copy() _header.update({"Content-Type": encoded_data[1]}) url = f"https://{self.ip}:{self.port}/plugin/api/v1.0/java_perf/api/records/actions/upload/" diff --git a/component/DevkitDistribute/devkit_distribute/bin/entrance.py b/component/DevkitDistribute/devkit_distribute/bin/entrance.py index 4c3194c6c1a58e6236c840dbc41e7d621a002843..85e3634daeac25da1939ba1240025f5821b6c8aa 100644 --- a/component/DevkitDistribute/devkit_distribute/bin/entrance.py +++ b/component/DevkitDistribute/devkit_distribute/bin/entrance.py @@ -2,9 +2,12 @@ import argparse import datetime import logging import os +import threading +import typing import uuid from devkit_utils import file_utils +from devkit_utils import shell_tools from devkit_utils.devkit_client import DevKitClient from devkit_utils.error_coce import ErrorCodeEnum, ErrorCodeMsg from devkit_utils.log_config import config_log_ini @@ -15,6 +18,61 @@ from report.report import Report ROOT_PATH = os.path.dirname(os.path.dirname(__file__)) +class JmeterCommand: + def __init__(self, origin_command): + self.origin_command: str = origin_command + self.csv_file = None + self.result_dir = None + self.jmx_file = None + + def check_and_init_jmeter_cmd(self): + if not self.origin_command: + return + jmeter_commands: typing.List[str] = self.origin_command.split() + command_length = len(jmeter_commands) + index = 0 + while index < command_length: + if jmeter_commands[index] in ["sh", "bash", "/bin/bash", "/bin/sh"]: + index = index + 1 + continue + elif jmeter_commands[index].endswith("jmeter.sh"): + index = index + 1 + continue + elif jmeter_commands[index] == "-n": + index = index + 1 + continue + elif jmeter_commands[index] == "-e": + index = index + 1 + continue + elif jmeter_commands[index].endswith("t") and index + 1 < command_length: + self.jmx_file = jmeter_commands[index + 1] + index = index + 2 + continue + elif jmeter_commands[index].endswith("l") and index + 1 < command_length: + self.csv_file = jmeter_commands[index + 1] + index = index + 2 + continue + elif jmeter_commands[index].endswith("o") and index + 1 < command_length: + self.result_dir = jmeter_commands[index + 1] + index = index + 2 + continue + else: + break + else: + return self.__check_param_resource() + raise Exception( + "The command line is not supported。example: sh jmeter.sh -n -t /home/demo.jmt " + "-l /home/jmeter/result.csv -e -o /home/jmeter/empty_dir/") + + def __check_param_resource(self): + if os.path.exists(self.csv_file): + raise Exception(f"the file {self.csv_file} is exist") + if os.path.exists(self.result_dir) and os.path.getsize(self.result_dir) > 0: + raise Exception(f"the directory {self.result_dir} is exist or not empty") + if not os.path.exists(self.jmx_file): + raise Exception(f"the jmx file {self.jmx_file} is not exist") + + class Distributor: def __init__(self, args): self.ips_list = args.ips_list.split(",") @@ -34,13 +92,25 @@ class Distributor: self.devkit_user = args.devkit_user self.devkit_password = args.devkit_password file_utils.create_dir(self.data_path) - self.template_path=os.path.join(self.root_path, "config") + self.template_path = os.path.join(self.root_path, "config") self.git_path = args.git_path + self.jmeter_command: JmeterCommand = JmeterCommand(args.jmeter_command) + self.jmeter_thread: typing.Optional[threading.Thread] = None + self.enable_jmeter_command = True if args.jmeter_command else False def distribute(self): + # jmeter 命令校验 + self.jmeter_command.check_and_init_jmeter_cmd() + # 校验联通性 + self.__check_ips_connected() + # 启动jmeter + if self.enable_jmeter_command: + self.__start_jmeter_thread() task_id = str(uuid.uuid4()) # 分发采集任务 self.distribute_to_sample_task(task_id) + if self.enable_jmeter_command: + self.jmeter_thread.join() # 获取jfr文件，删除任务文件 local_jfrs = list() self.obtain_jfrs(local_jfrs, task_id) @@ -53,12 +123,31 @@ class Distributor: client.logout() # 清空本地jfr文件 file_utils.clear_dir(self.data_path) - report = Report(report_path=self.data_path,template_path=self.template_path, - git_path=self.git_path, devkit_tool_ip=self.devkit_ip, - devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) + # 等待jmeter完成 + if self.enable_jmeter_command: + report = Report(report_path=self.data_path, template_path=self.template_path, + jmeter_report_path=self.jmeter_command.csv_file, + git_path=self.git_path, devkit_tool_ip=self.devkit_ip, + devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) + else: + report = Report(report_path=self.data_path, template_path=self.template_path, + git_path=self.git_path, devkit_tool_ip=self.devkit_ip, + devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) report.report() self.__print_result(jfr_names) + def __start_jmeter_thread(self): + self.jmeter_command.check_and_init_jmeter_cmd() + self.jmeter_thread = threading.Thread(target=self.__jmeter_start, args=(self.jmeter_command.origin_command,)) + self.jmeter_thread.start() + + @staticmethod + def __jmeter_start(command): + outcome = shell_tools.exec_shell(command, is_shell=True, timeout=None) + logging.info("return_code: %s", outcome.return_code) + logging.info("error: %s", outcome.err) + logging.info("out: %s", outcome.out) + def __print_result(self, jfr_names): print("=============================================================") print("The following files have been uploaded to the DevKit server:") @@ -68,6 +157,16 @@ class Distributor: f"https://{self.devkit_ip}:{self.devkit_port}") print(f"user :{self.devkit_user}, password: {self.devkit_password}") + def __check_ips_connected(self): + for ip in self.ips_list: + factory = SSHClientFactory(ip=ip, user=self.user, port=self.port, password=self.password, + pkey_file=self.pkey_file, pkey_content=self.pkey_content, + pkey_password=self.pkey_password) + ssh_client = factory.create_ssh_client() + ssh_client.close() + client = DevKitClient(self.devkit_ip, self.devkit_port, self.devkit_user, self.devkit_password) + client.logout() + def obtain_jfrs(self, local_jfrs, task_id): # 顺序获取 for ip in self.ips_list: @@ -194,6 +293,8 @@ def main(): help="the time of the sample") parser.add_argument("--git-path", required=True, dest="git_path", type=str, help="git path") + parser.add_argument("--jmeter-command", dest="jmeter_command", type=str, + help="the command that start jmeter tests") parser.set_defaults(root_path=obtain_root_path(ROOT_PATH)) parser.set_defaults(password="") args = parser.parse_args() diff --git a/component/DevkitDistribute/devkit_distribute/bin/report/report.py b/component/DevkitDistribute/devkit_distribute/bin/report/report.py index 39eea96abbf562340eeb8c1adaf85da9c5a849f7..31a88181ed2a5742acc410cf106885154839fb22 100644 --- a/component/DevkitDistribute/devkit_distribute/bin/report/report.py +++ b/component/DevkitDistribute/devkit_distribute/bin/report/report.py @@ -1,10 +1,8 @@ import csv import json import os -import subprocess -import time import re - +import subprocess PORT_SUB_PATTERN = re.compile(r':[0-9]+') GIT_REMOTE_URL_COMMAND = """ @@ -19,10 +17,15 @@ JMETER_REPORT_NAME = "result.csv" HTML_TEMPLATE_NAME = "perf_report.html" DEVKIT_REPORT_DATA_LINE_NUM = 32 GIT_REPORT_DATA_LINE_NUM = 41 +REPORT_VALID_LINE = 28 +JMETER_REPORT_DATA_HEADER_LEN = 35 +JMETER_REPORT_DATA_LINE_NUM = 36 class Report: - def __init__(self, report_path="./", template_path="./", git_path="./", jmeter_report_path="./", devkit_tool_ip="", devkit_tool_port="8086", devkit_user_name="devadmin"): + def __init__(self, report_path="./", template_path="./", git_path="./", + jmeter_report_path=None, devkit_tool_ip="", + devkit_tool_port="8086", devkit_user_name="devadmin"): if not os.path.isdir(report_path): raise Exception(f"Report path:{report_path} illegal.") self.report_dir = report_path @@ -32,6 +35,7 @@ class Report: self.devkit_tool_ip = devkit_tool_ip self.devkit_tool_port = devkit_tool_port self.devkit_user_name = devkit_user_name + self.jmeter_report_data_cols = 17 def report(self): html_lines = self.read_template() @@ -40,6 +44,11 @@ class Report: html_lines[DEVKIT_REPORT_DATA_LINE_NUM] = "report_tb_data: {}".format(devkit_report_json) html_lines[GIT_REPORT_DATA_LINE_NUM] = "git_tb_data: {},".format(git_log) + if self.jmeter_report_path: + html_lines[REPORT_VALID_LINE] = "const valid_pages = ['report', 'trend', 'git'];\n" + jmeter_report_data = self.jmeter_report_to_html() + html_lines[JMETER_REPORT_DATA_HEADER_LEN] = "trend_tb_cols: {},\n".format(self.jmeter_report_data_cols) + html_lines[JMETER_REPORT_DATA_LINE_NUM] = "trend_tb_data: {},\n".format(jmeter_report_data) final_report = os.path.join(self.report_dir, "devkit_performance_report.html") with open(final_report, "w") as file: @@ -67,10 +76,14 @@ class Report: git_log = json.dumps(data) return git_log - def jmeter_report_to_html(self, jmeter_report_path): - all_data= [] - jmeter_report = os.path.join(jmeter_report_path, JMETER_REPORT_NAME) - with open(jmeter_report, newline='') as csvfile: + def jmeter_report_to_html(self): + all_data = [] + with open(self.jmeter_report_path) as csvfile: + reader = csv.reader(csvfile) + for row in reader: + self.jmeter_report_data_cols = len(row) + break + with open(self.jmeter_report_path, newline='') as csvfile: reader = csv.reader(csvfile) for row in reader: all_data.extend(row) diff --git a/component/DevkitDistribute/devkit_distribute/config/devkit_distribute_template.yaml b/component/DevkitDistribute/devkit_distribute/config/devkit_distribute_template.yaml index 2bcc15bd3592813ec9d82d62a8f16e87d35bb624..09a23fc6e3bae95466a5e321bc7ce637c27d899c 100644 --- a/component/DevkitDistribute/devkit_distribute/config/devkit_distribute_template.yaml +++ b/component/DevkitDistribute/devkit_distribute/config/devkit_distribute_template.yaml @@ -15,4 +15,5 @@ devkit_password: ${devkit_password} applications: ${applications} duration: ${duration} git_path: ${git_path} +jmeter_command: "${jmeter_command}" diff --git a/component/DevkitDistribute/devkit_distribute/config/log.ini.template b/component/DevkitDistribute/devkit_distribute/config/log.ini.template index ac3abb556249a0151a0bfc077baa6dfad229dc07..2490d706c25bdf5f256b8ae763a59e95409d568d 100644 --- a/component/DevkitDistribute/devkit_distribute/config/log.ini.template +++ b/component/DevkitDistribute/devkit_distribute/config/log.ini.template @@ -3,7 +3,7 @@ keys=root [handlers] # handlers 对象列表 -keys=consoleHandler,fileHandler +keys=consoleHandler [formatters] # formatters 对象列表 keys=fmt @@ -19,11 +19,6 @@ level = INFO formatter = fmt args = (sys.stdout,) -[handler_fileHandler]# fileHandler 控制器输出方向、级别、输出格式、参数 -class = FileHandler -level = INFO -formatter = fmt -args = ("LOG_PATH/LOG_NAME.log", "a") [formatter_fmt] format=[%(asctime)s] [%(levelname)s] [processID:%(process)d][%(threadName)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s diff --git a/component/DevkitDistribute/devkit_distribute/config/perf_report.html b/component/DevkitDistribute/devkit_distribute/config/perf_report.html index ba5c1bc23f0cc59fb1b3e49238ae9686a06270f7..e530a71340e5457e909e03867ba1a70d7eeab06b 100644 --- a/component/DevkitDistribute/devkit_distribute/config/perf_report.html +++ b/component/DevkitDistribute/devkit_distribute/config/perf_report.html @@ -35,7 +35,7 @@ trend: { trend_tb_cols: 4, trend_tb_data: [], // must be countable, idx_0 represents x-axis - trend_chart_labels: [], + trend_chart_labels: ["TimeStamp", "Elapsed"], }, git: { git_tb_cols: 5, diff --git a/component/DevkitDistribute/devkit_distribute/script/generate_lkptest_config.sh b/component/DevkitDistribute/devkit_distribute/script/generate_lkptest_config.sh index ecd950c675d96ba14dc44c51fc53d51b7d45abfa..b35fd2d528ff55ce14e74bf33e1f2be0f62d6d3b 100644 --- a/component/DevkitDistribute/devkit_distribute/script/generate_lkptest_config.sh +++ b/component/DevkitDistribute/devkit_distribute/script/generate_lkptest_config.sh @@ -22,26 +22,35 @@ local devkit_password="admin100" local applications="" local duration=10 local git_path="" +local jmeter_command="" -while getopts "i:u:f:a:d:D:g:" opts; do - case $opts in - i) - ips_list=$OPTARG ;; - u) - user=$OPTARG ;; - f) - pkey_file=$OPTARG ;; - a) - applications=$OPTARG ;; - d) - duration=$OPTARG ;; - D) - devkit_ip=$OPTARG ;; - g) - git_path=$OPTARG ;; - ?) - echo "not recogize paramters";; - esac +while getopts "i:u:f:a:d:D:P:U:W:g:j:" opts; do + case $opts in + i) + ips_list=$OPTARG ;; + u) + user=$OPTARG ;; + f) + pkey_file=$OPTARG ;; + a) + applications=$OPTARG ;; + d) + duration=$OPTARG ;; + D) + devkit_ip=$OPTARG ;; + P) + devkit_port=$OPTARG ;; + U) + devkit_user=$OPTARG ;; + W) + devkit_password=$OPTARG ;; + g) + git_path=$OPTARG ;; + j) + jmeter_command=$OPTARG ;; + ?) + echo "The Parameters are not recognized";; + esac done sed -i "s?\${root_path}?${root_path}?g" "${root_path}/config/devkit_distribute.yaml" @@ -57,6 +66,7 @@ sed -i "s/\${devkit_password}/${devkit_password}/g" "${root_path}/config/devkit_ sed -i "s?\${applications}?${applications}?g" "${root_path}/config/devkit_distribute.yaml" sed -i "s?\${git_path}?${git_path}?g" "${root_path}/config/devkit_distribute.yaml" sed -i "s/\${duration}/${duration}/g" "${root_path}/config/devkit_distribute.yaml" +sed -i "s?\${jmeter_command}?${jmeter_command}?g" "${root_path}/config/devkit_distribute.yaml" } diff --git a/component/DevkitDistribute/devkit_distribute/script/start.sh b/component/DevkitDistribute/devkit_distribute/script/start.sh index a2967ee9451ff5554d281908daaab669b4c70a47..6ea230c191ac3b3b2a3fb39bee5e8dc715271568 100644 --- a/component/DevkitDistribute/devkit_distribute/script/start.sh +++ b/component/DevkitDistribute/devkit_distribute/script/start.sh @@ -5,4 +5,4 @@ # shellcheck disable=SC2154 "${root_path}/bin/entrance" -i "${ips_list}" -u "${user}" -P "${port}" -f "${pkey_file}" --duration "${duration}" --app "${applications}" \ --devkit-ip "${devkit_ip}" --devkit-port "${devkit_port}" --devkit-password "${devkit_password}" --devkit-user "${devkit_user}" \ - --pkey-password "${pkey_password}" --git-path "${git_path}" \ No newline at end of file + --pkey-password "${pkey_password}" --git-path "${git_path}" --jmeter-command "${jmeter_command}" \ No newline at end of file diff --git a/tools/.DS_Store b/tools/.DS_Store deleted file mode 100644 index 5e2bd9676140361212eb36c42bc41ed5c88e6fac..0000000000000000000000000000000000000000 Binary files a/tools/.DS_Store and /dev/null differ diff --git a/tools/distribute/.DS_Store b/tools/distribute/.DS_Store deleted file mode 100644 index ca4fa5e6bd8895a471e5b31eefd0d0e28db7cd7a..0000000000000000000000000000000000000000 Binary files a/tools/distribute/.DS_Store and /dev/null differ diff --git a/tools/distribute/devkit_distribute/.DS_Store b/tools/distribute/devkit_distribute/.DS_Store deleted file mode 100644 index f943ffb8727b3ca9f06ba17a4521f713d0d1f8db..0000000000000000000000000000000000000000 Binary files a/tools/distribute/devkit_distribute/.DS_Store and /dev/null differ diff --git a/tools/distribute/devkit_distribute/bin/.DS_Store b/tools/distribute/devkit_distribute/bin/.DS_Store deleted file mode 100644 index 4a74d9fe6b65a63810df1058ff8371ca55e24ad9..0000000000000000000000000000000000000000 Binary files a/tools/distribute/devkit_distribute/bin/.DS_Store and /dev/null differ diff --git a/tools/distribute/devkit_distribute/bin/report/.DS_Store b/tools/distribute/devkit_distribute/bin/report/.DS_Store deleted file mode 100644 index 514b652a5aa6777ddd1fe06b6f72cac9ff987832..0000000000000000000000000000000000000000 Binary files a/tools/distribute/devkit_distribute/bin/report/.DS_Store and /dev/null differ