From 1a1774b6a9b1cb37a00dbfa2042ad6da45b13b48 Mon Sep 17 00:00:00 2001 From: pan <601760354@163.com> Date: Tue, 26 Mar 2024 19:16:07 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E6=8C=89=E9=9C=80=E4=B8=8B=E8=BD=BD=E7=BB=84=E4=BB=B6=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/download_dependency/src/download.py | 11 ++-- .../download_dependency/src/download_utils.py | 2 + .../src/handler/connect_check.py | 4 +- .../src/handler/gather_package.py | 55 +++++++------------ .../src/machine/local_machine.py | 3 + .../install_dependency/src/machine/machine.py | 3 + 6 files changed, 34 insertions(+), 44 deletions(-) diff --git a/tools/download_dependency/src/download.py b/tools/download_dependency/src/download.py index 8a33211..6ab7276 100644 --- a/tools/download_dependency/src/download.py +++ b/tools/download_dependency/src/download.py @@ -26,6 +26,7 @@ AUTO_PREFETCH = "auto_prefetch.sh" SPLIT_JSON_PY = "split_json.py" FILE_LIST = (A_FOT, A_FOT_INI, AUTO_FDO_SH, AUTO_BOLT_SH, AUTO_PREFETCH, SPLIT_JSON_PY) + component_collection_map = { component.get("component_name"): { "download file": @@ -56,9 +57,7 @@ lkp_collection_map = { URL: f"{download_config.LkpTests.get('gem dependency')}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'gem_dependencies.zip')}", }, - }, - "CompatibilityTesting": { - "download file": { + "download CompatibilityTesting": { URL: f"{download_config.CompatibilityTesting.get(FILE)}", SAVE_PATH: f"{os.path.join(DEFAULT_PATH, 'compatibility_testing.tar.gz')}", } @@ -66,18 +65,18 @@ lkp_collection_map = { } -def download_dependence(): +def download_dependence(component_list): if not os.path.exists(DEFAULT_PATH): os.mkdir(DEFAULT_PATH) elif os.path.isfile(DEFAULT_PATH): print(f"[ERROR] The file {DEFAULT_PATH} exists. Please rename or remove this file.") return False - else: - pass ret = True component_collection_map.update(lkp_collection_map) for component_name in component_collection_map: + if component_name not in component_list: + continue shell_dict = component_collection_map.get(component_name) ret = ret and download_dependence_handler(shell_dict) return ret diff --git a/tools/download_dependency/src/download_utils.py b/tools/download_dependency/src/download_utils.py index 6f42217..b51ca8b 100644 --- a/tools/download_dependency/src/download_utils.py +++ b/tools/download_dependency/src/download_utils.py @@ -14,6 +14,8 @@ def download_dependence_handler(shell_dict): def download_dependence_file(shell_cmd, shell_dict): ret = True url_and_save_path = shell_dict.get(shell_cmd) + if os.path.exists(url_and_save_path.get('save_path')) and os.path.isfile(url_and_save_path.get('save_path')): + return ret try: print(f"Downloading from {url_and_save_path.get('url')}") download_result = wget.download( diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index fbe8e87..623afbf 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -17,11 +17,10 @@ ROLE_COMPONENT = { class ConnectCheck(Handler): - def handle(self, data) -> bool: LOGGER.debug("ConnectCheck start!") local_ip = ConnectCheck.get_local_ip() - + data[constant.MACHINE] = dict() ret = True for role in (set(KLASS_DICT.keys()) & set(data.keys())): ret = ret and ConnectCheck.machine_role_check(data, role, local_ip) @@ -31,7 +30,6 @@ class ConnectCheck(Handler): def machine_role_check(data, role, local_ip): builder_list = data.get(role) klass = KLASS_DICT.get(role) - data[constant.MACHINE] = dict() for ip in builder_list: if ip == local_ip or ip == "127.0.0.1": ip = "127.0.0.1" diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/install_dependency/src/handler/gather_package.py index edaed4e..714525c 100644 --- a/tools/install_dependency/src/handler/gather_package.py +++ b/tools/install_dependency/src/handler/gather_package.py @@ -9,32 +9,28 @@ LOGGER = logging.getLogger("install_dependency") class GatherPackage(Handler): + def __init__(self): + super(GatherPackage, self).__init__() + self.component_list = list() - def handle(self, data) -> bool: - instruction_to_func_dict = { - "deploy_iso": GatherPackage.deploy_iso_handle, - "default": GatherPackage.default_handle, - } - return instruction_to_func_dict.get(data.get(constant.INSTRUCTION, "default"))() - - @staticmethod - def deploy_iso_handle(): - LOGGER.info("Iso file already checked.") - return GatherPackage.default_handle() + def generate_component_list(self, data): + component_list = [] + for _, machine in data[constant.MACHINE].items(): + component_list.extend(machine.get_components()) + self.component_list = list(set(component_list)) - @staticmethod - def default_handle(): + def handle(self, data) -> bool: LOGGER.debug("GatherPackage start!") - if GatherPackage.check_default_path_available(): - LOGGER.info("Dependencies ready.") - return True + component_collection_map.update(lkp_collection_map) + self.generate_component_list(data) - if os.path.isfile(constant.DEPENDENCY_DIR): - LOGGER.error(f"The file {constant.DEPENDENCY_DIR} exists. Please rename or remove this file.") + try: + self.check_dependency() + except Exception as e: return False try: - ret = download_dependence() + ret = download_dependence(self.component_list) except Exception as e: LOGGER.error(f"Download dependencies failed. {str(e)}. Please execute download tool.") return False @@ -45,7 +41,7 @@ class GatherPackage(Handler): return True @staticmethod - def check_default_path_available(): + def check_dependency(): if os.path.exists(constant.DEPENDENCY_FILE): try: print(f"Now extract files from {constant.DEPENDENCY_FILE}:") @@ -55,18 +51,7 @@ class GatherPackage(Handler): LOGGER.warning(f"{constant.DEPENDENCY_FILE} may already extracted.") except Exception as e: LOGGER.error(f"Extract {constant.DEPENDENCY_FILE} failed. {str(e)}") - return False - - if not os.path.isdir(constant.DEPENDENCY_DIR): - LOGGER.warning(f"The directory {constant.DEPENDENCY_DIR} not exists.") - return False - component_collection_map.update(lkp_collection_map) - for component_name in component_collection_map: - shell_dict = component_collection_map.get(component_name) - for shell_cmd in shell_dict: - url_and_save_path = shell_dict.get(shell_cmd) - component = url_and_save_path.get("save_path") - if not os.path.isfile(component): - LOGGER.warning(f"The file {component} not exists.") - return False - return True + raise + if os.path.isfile(constant.DEPENDENCY_DIR): + LOGGER.error(f"The file {constant.DEPENDENCY_DIR} exists. Please rename or remove this file.") + raise Exception diff --git a/tools/install_dependency/src/machine/local_machine.py b/tools/install_dependency/src/machine/local_machine.py index 8e46f6f..f12f9c7 100644 --- a/tools/install_dependency/src/machine/local_machine.py +++ b/tools/install_dependency/src/machine/local_machine.py @@ -28,6 +28,9 @@ class LocalMachine: self.component_list.extend(component) self.component_list = list(set(self.component_list)) + def get_components(self): + return self.component_list.copy() + def install_components(self): if self.mirrors: self.install_component("OpenEulerMirrorISO") diff --git a/tools/install_dependency/src/machine/machine.py b/tools/install_dependency/src/machine/machine.py index 31300fc..a08d9d9 100644 --- a/tools/install_dependency/src/machine/machine.py +++ b/tools/install_dependency/src/machine/machine.py @@ -34,6 +34,9 @@ class Machine: self.component_list.extend(component) self.component_list = list(set(self.component_list)) + def get_components(self): + return self.component_list.copy() + def set_mirror(self): self.mirrors = True -- Gitee From 196134ddcabc1be64b73ced8e068a403166ec478 Mon Sep 17 00:00:00 2001 From: pan <601760354@163.com> Date: Wed, 27 Mar 2024 11:15:03 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=B8=8B=E8=BC=89=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E9=81=A9=E9=85=8D=E6=8C=89=E9=9C=80=E4=B8=8B=E8=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/download_dependency/src/download.py | 41 ++++++++++++++++++- .../src/download_command_line.py | 5 +++ .../src/handler/connect_check.py | 7 +--- tools/install_dependency/src/install.py | 19 +-------- 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/tools/download_dependency/src/download.py b/tools/download_dependency/src/download.py index 6ab7276..6baba46 100644 --- a/tools/download_dependency/src/download.py +++ b/tools/download_dependency/src/download.py @@ -5,6 +5,7 @@ import sys import shutil import tarfile import wget +import yaml import download_config from download_utils import download_dependence_handler, download_dependence_file from download_command_line import process_command_line, CommandLine @@ -63,6 +64,33 @@ lkp_collection_map = { } }, } +SCANNER = "scanner" +BUILDER = "builder" +EXECUTOR = "executor" + +ROLE_COMPONENT = { + SCANNER: ["BiShengJDK17"], + BUILDER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], + EXECUTOR: ["BiShengJDK17", "LkpTests"] +} + +ROLE_LIST = [SCANNER, BUILDER, EXECUTOR] + + +def read_yaml_file(yaml_path): + try: + with open(yaml_path, "r") as file: + yaml_dict = yaml.safe_load(file) + except (FileNotFoundError, IsADirectoryError) as e: + print(f"[ERROR] Yaml file is not in specified path. Error: {str(e)}") + sys.exit(1) + except (yaml.parser.ParserError, + yaml.scanner.ScannerError, + yaml.composer.ComposerError, + yaml.constructor.ConstructorError) as e: + print(f"[ERROR] Incorrect yaml file. Error: {str(e)}") + sys.exit(1) + return yaml_dict def download_dependence(component_list): @@ -162,6 +190,15 @@ def download_iso(): return download_dependence_file("download file", shell_dict) +def generate_component_list(yaml_dict): + component_list = list() + for role in ROLE_LIST: + if role not in yaml_dict: + continue + component_list.extend(ROLE_COMPONENT[role]) + return list(set(component_list)) + + if __name__ == '__main__': try: process_command_line(program="download_dependency", description="devkit-pipeline download_dependency tool", @@ -172,8 +209,8 @@ if __name__ == '__main__': else: print("Download iso failed.") sys.exit(0) - - ret = download_dependence() + config_dict = read_yaml_file(CommandLine.yaml_path) + ret = download_dependence(generate_component_list(config_dict)) if ret: print(f"Now compress dependencies to {DEPENDENCY_FILE}...") with tarfile.open(DEPENDENCY_FILE, "w:gz") as tar: diff --git a/tools/download_dependency/src/download_command_line.py b/tools/download_dependency/src/download_command_line.py index 12cd1f5..0140818 100644 --- a/tools/download_dependency/src/download_command_line.py +++ b/tools/download_dependency/src/download_command_line.py @@ -1,12 +1,17 @@ import argparse import download_config +DEFAULT_YAML_PATH = "./machine.yaml" + class CommandLine: download_iso = None + yaml_path = DEFAULT_YAML_PATH @classmethod def add_options(cls, parser): + parser.add_argument("-f", "--config", action="store", dest="yaml_path", default=DEFAULT_YAML_PATH, + help="Assign yaml config file path. Default path is 'machine.yaml' in current directory.") parser.add_argument("-iso", action="store", dest="download_iso", default="", choices=[ component.get("component_name") for component in ( diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py index 623afbf..e1e852f 100644 --- a/tools/install_dependency/src/handler/connect_check.py +++ b/tools/install_dependency/src/handler/connect_check.py @@ -6,15 +6,10 @@ from handler.handler_and_node import Handler from machine.local_machine import LocalMachine from machine.klass_dict import KLASS_DICT from exception.connect_exception import ConnectException +from download import ROLE_COMPONENT LOGGER = logging.getLogger("install_dependency") -ROLE_COMPONENT = { - "scanner": ["BiShengJDK17"], - "builder": ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "BiShengJDK8"], - "executor": ["BiShengJDK17", "LkpTests"] -} - class ConnectCheck(Handler): def handle(self, data) -> bool: diff --git a/tools/install_dependency/src/install.py b/tools/install_dependency/src/install.py index 62b77b5..ca320da 100644 --- a/tools/install_dependency/src/install.py +++ b/tools/install_dependency/src/install.py @@ -2,7 +2,7 @@ import os import subprocess import sys import logging -import yaml + import constant from log import config_logging @@ -13,28 +13,13 @@ from handler.base_yaml_check import BaseCheck from handler.connect_check import ConnectCheck from handler.gather_package import GatherPackage from handler.install_package import InstallPackage +from download import read_yaml_file LOGGER = logging.getLogger("install_dependency") PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] ISO_VERIFY_FLAG_STRING = "ISO 9660 CD-ROM filesystem data" -def read_yaml_file(yaml_path): - try: - with open(yaml_path, "r") as file: - config_dict = yaml.safe_load(file) - except (FileNotFoundError, IsADirectoryError) as e: - LOGGER.error(f"Yaml file is not in specified path. Error: {str(e)}") - sys.exit(1) - except (yaml.parser.ParserError, - yaml.scanner.ScannerError, - yaml.composer.ComposerError, - yaml.constructor.ConstructorError) as e: - LOGGER.error(f"Incorrect yaml file. Error: {str(e)}") - sys.exit(1) - return config_dict - - def check_iso_available(iso_path): if not os.path.isfile(iso_path): LOGGER.error(f"ISO file is not in specified path. Error: {iso_path} file not found.") -- Gitee