From a0327fe67d16eb8a525f21be2b3dee2c2094d2a8 Mon Sep 17 00:00:00 2001 From: ZT <1450026690@qq.com> Date: Mon, 20 Jul 2020 20:55:47 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86build=5Fdepend?= =?UTF-8?q?=E7=9A=84=E8=BF=94=E5=9B=9E=E7=BB=93=E6=9E=9C=E4=B8=ADdb=5Fname?= =?UTF-8?q?=E4=B8=8D=E6=98=AF=E6=9C=80=E9=AB=98=E4=BC=98=E5=85=88=E7=BA=A7?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98,=E4=BF=AE=E6=94=B9=E4=BA=86searchdb?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E6=9F=A5=E8=AF=A2=E6=96=B9=E5=BC=8F,?= =?UTF-8?q?=E9=98=B2=E6=AD=A2=E5=87=BA=E7=8E=B0=E5=8F=82=E6=95=B0=E8=BF=87?= =?UTF-8?q?=E5=A4=9A=E7=9A=84=E9=94=99=E8=AF=AF,=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E4=BA=86=E8=8E=B7=E5=8F=96=E6=9C=80=E9=AB=98=E4=BC=98=E5=85=88?= =?UTF-8?q?=E7=BA=A7=E7=9A=84=E5=87=BD=E6=95=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apps/package/function/build_depend.py | 27 +- .../apps/package/function/searchdb.py | 260 +++++++++--------- 2 files changed, 152 insertions(+), 135 deletions(-) diff --git a/packageship/packageship/application/apps/package/function/build_depend.py b/packageship/packageship/application/apps/package/function/build_depend.py index 672cbe60..b65464b9 100644 --- a/packageship/packageship/application/apps/package/function/build_depend.py +++ b/packageship/packageship/application/apps/package/function/build_depend.py @@ -80,9 +80,9 @@ class BuildDepend(): res_status, build_list = self.search_db.get_build_depend(pkg_list) if not build_list: - return res_status if res_status == \ - ResponseCode.DIS_CONNECTION_DB else \ + return res_status if res_status == ResponseCode.DIS_CONNECTION_DB else \ ResponseCode.PACK_NAME_NOT_FOUND + # create root node and get next search list search_list = self._create_node_and_get_search_list(build_list, pkg_list) @@ -153,7 +153,7 @@ class BuildDepend(): self.result_dict[obj.bin_name] = [ obj.source_name, obj.version, - obj.db_name, + self.search_db.binary_search_database_for_first_time(obj.bin_name), [ [obj.search_name, 'build'] ] @@ -191,19 +191,21 @@ class BuildDepend(): return # generate data content + search_name_set = set() for obj in bin_info_lis: + search_name_set.add(obj.search_name) + if obj.search_name not in self.source_dict: + self.source_dict[obj.search_name] = [obj.db_name, obj.search_version] + if not obj.bin_name: continue - # for first loop, init the source_dict - if not self.source_dict: - for src_name in pkg_name_li: - self.source_dict[src_name] = [obj.db_name, obj.search_version] + if obj.bin_name not in self.result_dict: self.result_dict[obj.bin_name] = [ obj.source_name if obj.source_name else None, obj.version if obj.version else None, - obj.db_name if obj.db_name else "NOT_FOUND", + self.search_db.binary_search_database_for_first_time(obj.bin_name), [ [obj.search_name, "build"] ] @@ -216,11 +218,14 @@ class BuildDepend(): if obj.source_name and \ obj.source_name not in self.source_dict and \ - obj.source_name not in self.history_dicts: - self.source_dict[obj.source_name] = [obj.db_name, - obj.version] + obj.source_name not in self.history_dicts: next_src_set.add(obj.source_name) + not_found_pkg = set(pkg_name_li) - search_name_set + for pkg_name in not_found_pkg: + if pkg_name not in self.source_dict: + self.source_dict[pkg_name] = ['NOT FOUND', 'NOT FOUND'] + not_found_pkg.clear() self.self_build(next_src_set) return diff --git a/packageship/packageship/application/apps/package/function/searchdb.py b/packageship/packageship/application/apps/package/function/searchdb.py index bba2994b..e7a4f3de 100644 --- a/packageship/packageship/application/apps/package/function/searchdb.py +++ b/packageship/packageship/application/apps/package/function/searchdb.py @@ -11,6 +11,7 @@ from flask import current_app from sqlalchemy import text from sqlalchemy.exc import SQLAlchemyError, DisconnectionError from sqlalchemy.sql import literal_column +from sqlalchemy import exists from packageship.libs.dbutils import DBHelper from packageship.libs.log import Log @@ -30,6 +31,7 @@ class SearchDB(): db_object_dict:A dictionary for storing database connection objects changeLog: """ + def __new__(cls, *args, **kwargs): # pylint: disable=w0613 if not hasattr(cls, "_instance"): @@ -209,7 +211,7 @@ class SearchDB(): (return_tuple(None, None, search_name), 'NOT_FOUND')) return ResponseCode.SUCCESS, result_list - def get_binary_in_other_database(self, not_found_binary, db_): + def _get_binary_in_other_database(self, not_found_binary): """ Description: Binary package name data not found in the current database, go to other databases to try @@ -235,57 +237,53 @@ class SearchDB(): "version", "db_name", "search_version", - "req_name" ]) - src_req_map = {req_: src for src, req_ in not_found_binary} - - local_search_set = {req_ for _, req_ in not_found_binary} - - local_dict = {k: v for k, v in self.db_object_dict.items() if k != db_} - res = [] - - for db_name, data_base in local_dict.items(): - try: - sql_string = text(""" - SELECT - t3.NAME AS source_name, - t1.NAME AS bin_name, - t1.version, - t3.version as search_version, - t2.NAME AS req_name - FROM - bin_pack t1, - pack_provides t2, - src_pack t3 - WHERE - t2.{} - AND t1.id = t2.binIDkey - AND t1.srcIDkey = t3.id; - """.format(literal_column('name').in_(local_search_set))) - build_set_2 = data_base.session. \ - execute(sql_string, {'name_{}'.format(i): v - for i, v in enumerate(local_search_set, 1)}).fetchall() - if not build_set_2: - continue - - res.extend([return_tuple( - src_req_map.get(bin_pack.req_name), - bin_pack.source_name, - bin_pack.bin_name, - bin_pack.version, - db_name, - bin_pack.search_version, - bin_pack.req_name - ) for bin_pack in build_set_2 if bin_pack.bin_name]) - - for obj in res: - local_search_set.remove(obj.req_name) - - except AttributeError as attr_error: - current_app.logger.error(attr_error) - except SQLAlchemyError as sql_error: - current_app.logger.error(sql_error) - return res + search_list = [] + result_list = [] + for db_name, data_base in self.db_object_dict.items(): + for key, _ in not_found_binary.items(): + search_list.append(key) + + search_set = set(search_list) + search_list.clear() + sql_string = text(""" + SELECT DISTINCT + t1.src_name AS source_name, + t1.NAME AS bin_name, + t1.version, + t2.NAME AS req_name + FROM + bin_pack t1, + bin_provides t2 + WHERE + t2.{} + AND t1.pkgKey = t2.pkgKey; + """.format(literal_column('name').in_(search_set))) + bin_set = data_base.session. \ + execute(sql_string, {'name_{}'.format(i): v + for i, v in enumerate(search_set, 1)}).fetchall() + if bin_set: + for result in bin_set: + if result.req_name not in not_found_binary: + LOGGER.logger.warning(result.req_name + " contains in two rpm packages!!!") + else: + for source_info in not_found_binary[result.req_name]: + obj = return_tuple( + source_info[0], + result.source_name, + result.bin_name, + result.version, + db_name, + source_info[1] + ) + result_list.append(obj) + del not_found_binary[result.req_name] + if not not_found_binary: + return result_list + if not_found_binary: + for key, values in not_found_binary.items(): + LOGGER.logger.warning("CANNOT FOUND THE component" + key + " in all database") + return result_list def get_build_depend(self, source_name_li): """ @@ -315,89 +313,103 @@ class SearchDB(): if not s_name_set: return ResponseCode.PARAM_ERROR, None - not_found_binary = set() + provides_not_found = dict() build_list = [] + build_set = [] for db_name, data_base in self.db_object_dict.items(): - try: - sql_com = text("""SELECT DISTINCT - src.NAME AS search_name, - src.version AS search_version, - s2.NAME AS source_name, - pack_provides.binIDkey AS bin_id, - pack_requires.NAME AS req_name, - bin_pack.version AS version, - bin_pack.NAME AS bin_name - FROM - ( SELECT id, NAME,version FROM src_pack WHERE {} ) src - LEFT JOIN pack_requires ON src.id = pack_requires.srcIDkey - LEFT JOIN pack_provides ON pack_provides.id = pack_requires.depProIDkey - LEFT JOIN bin_pack ON bin_pack.id = pack_provides.binIDkey - LEFT JOIN src_pack s1 ON s1.id = pack_requires.srcIDkey - LEFT JOIN src_pack s2 ON bin_pack.srcIDkey = s2.id; - """.format(literal_column("name").in_(s_name_set))) - - build_set = data_base.session. \ - execute(sql_com, {'name_{}'.format(i): v - for i, v in enumerate(s_name_set, 1)}).fetchall() - - if not build_set: - continue - # When processing source package without compilation dependency - to_remove_obj_index = [] - for index, b_pack in enumerate(build_set): - if not b_pack.source_name and not b_pack.req_name: - obj = return_tuple( - b_pack.search_name, - b_pack.source_name, - b_pack.bin_name, - b_pack.version, - db_name, - b_pack.search_version + temp_list = list(s_name_set) + for input_name_li in [temp_list[i:i + 900] for i in range(0, len(temp_list), 900)]: + sql_com = text(""" + SELECT DISTINCT + src.NAME AS search_name, + src.version AS search_version, + bin_pack.src_name AS source_name, + bin_provides.pkgKey AS bin_id, + src_requires.NAME AS req_name, + bin_pack.version AS version, + bin_pack.NAME AS bin_name + FROM + ( SELECT pkgKey, NAME, version FROM src_pack WHERE {} ) src + LEFT JOIN src_requires ON src.pkgKey = src_requires.pkgKey + LEFT JOIN bin_provides ON bin_provides.NAME = src_requires.NAME + LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey; + """.format(literal_column("name").in_(input_name_li))) + res = data_base.session.execute( + sql_com, + {'name_{}'.format(i): v + for i, v in enumerate(input_name_li, 1)} + ).fetchall() + + build_set.extend(res) + + if not build_set: + continue + + # When processing source package without compilation dependency + get_list = [] + for result in build_set: + get_list.append(result.search_name) + if not result.bin_name and result.req_name: + if result.req_name in provides_not_found: + provides_not_found[result.req_name].append( + [result.search_name, result.search_version, db_name] ) - - build_list.append(obj) - to_remove_obj_index.append(index) - - for i in reversed(to_remove_obj_index): - build_set.pop(i) - - if not build_set: - continue - - build_list.extend([ - return_tuple( - bin_pack.search_name, - bin_pack.source_name, - bin_pack.bin_name, - bin_pack.version, + else: + provides_not_found[result.req_name] = [ + [result.search_name, result.search_version, db_name] + ] + else: + obj = return_tuple( + result.search_name, + result.source_name, + result.bin_name, + result.version, db_name, - bin_pack.search_version - ) for bin_pack in build_set if bin_pack.bin_id and bin_pack.bin_name - ]) - # Component name can't find its binary package name - not_found_binary.update([(bin_pack.search_name, bin_pack.req_name) - for bin_pack in build_set if not bin_pack.bin_id]) - - s_name_set -= {bin_pack.search_name for bin_pack in build_set - if bin_pack.bin_id} - - if not not_found_binary and not s_name_set: - return ResponseCode.SUCCESS, build_list - - for obj in self.get_binary_in_other_database(not_found_binary, db_name): + result.search_version + ) build_list.append(obj) - not_found_binary.clear() - - except AttributeError as attr_error: - current_app.logger.error(attr_error) - except SQLAlchemyError as sql_error: - current_app.logger.error(sql_error) - return ResponseCode.DIS_CONNECTION_DB, None + get_set = set(get_list) + get_list.clear() + s_name_set.symmetric_difference_update(get_set) + if not s_name_set: + build_result = self._get_binary_in_other_database(provides_not_found) + build_list.extend(build_result) + return ResponseCode.SUCCESS, build_list + + if s_name_set: + build_result = self._get_binary_in_other_database(provides_not_found) + build_list.extend(build_result) + for source in s_name_set: + LOGGER.logger.warning("CANNOT FOUND THE source " + source + " in all database") return ResponseCode.SUCCESS, build_list + def binary_search_database_for_first_time(self, binary_name): + """ + Args: + binary_name: a binary package name + + Returns: + The name of the first database + in which the binary package appears according to priority + If it does not exist or exception occurred , return 'NOT FOUND' + + """ + try: + for db_name, data_base in self.db_object_dict.items(): + if data_base.session.query( + exists().where(bin_pack.name == binary_name) + ).scalar(): + return db_name + except AttributeError as attr_err: + current_app.logger.error(attr_err) + except SQLAlchemyError as sql_err: + current_app.logger.err(sql_err) + + return 'NOT FOUND' + def db_priority(): """ -- Gitee From 6895fa17f9a9d4fbfaca1cd3b66e473198d9ce72 Mon Sep 17 00:00:00 2001 From: stutao <1450026690@qq.com> Date: Thu, 23 Jul 2020 15:14:16 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86try=20catch=20?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E7=9A=84=E6=8D=95=E8=8E=B7=E5=86=85=E5=AE=B9?= =?UTF-8?q?,=E4=BF=AE=E6=94=B9=E4=BA=86=E8=AE=B0=E5=BD=95=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E6=97=B6=E5=80=99=E7=9A=84error=E6=8B=BC=E5=86=99?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apps/package/function/searchdb.py | 131 ++++++++++-------- 1 file changed, 71 insertions(+), 60 deletions(-) diff --git a/packageship/packageship/application/apps/package/function/searchdb.py b/packageship/packageship/application/apps/package/function/searchdb.py index e7a4f3de..62c4a86b 100644 --- a/packageship/packageship/application/apps/package/function/searchdb.py +++ b/packageship/packageship/application/apps/package/function/searchdb.py @@ -246,40 +246,46 @@ class SearchDB(): search_set = set(search_list) search_list.clear() - sql_string = text(""" - SELECT DISTINCT - t1.src_name AS source_name, - t1.NAME AS bin_name, - t1.version, - t2.NAME AS req_name - FROM - bin_pack t1, - bin_provides t2 - WHERE - t2.{} - AND t1.pkgKey = t2.pkgKey; - """.format(literal_column('name').in_(search_set))) - bin_set = data_base.session. \ - execute(sql_string, {'name_{}'.format(i): v - for i, v in enumerate(search_set, 1)}).fetchall() - if bin_set: - for result in bin_set: - if result.req_name not in not_found_binary: - LOGGER.logger.warning(result.req_name + " contains in two rpm packages!!!") - else: - for source_info in not_found_binary[result.req_name]: - obj = return_tuple( - source_info[0], - result.source_name, - result.bin_name, - result.version, - db_name, - source_info[1] - ) - result_list.append(obj) - del not_found_binary[result.req_name] - if not not_found_binary: - return result_list + try: + sql_string = text(""" + SELECT DISTINCT + t1.src_name AS source_name, + t1.NAME AS bin_name, + t1.version, + t2.NAME AS req_name + FROM + bin_pack t1, + bin_provides t2 + WHERE + t2.{} + AND t1.pkgKey = t2.pkgKey; + """.format(literal_column('name').in_(search_set))) + bin_set = data_base.session. \ + execute(sql_string, {'name_{}'.format(i): v + for i, v in enumerate(search_set, 1)}).fetchall() + if bin_set: + for result in bin_set: + if result.req_name not in not_found_binary: + LOGGER.logger.warning(result.req_name + " contains in two rpm packages!!!") + else: + for source_info in not_found_binary[result.req_name]: + obj = return_tuple( + source_info[0], + result.source_name, + result.bin_name, + result.version, + db_name, + source_info[1] + ) + result_list.append(obj) + del not_found_binary[result.req_name] + if not not_found_binary: + return result_list + except AttributeError as attr_err: + current_app.logger.error(attr_err) + except SQLAlchemyError as sql_err: + current_app.logger.error(sql_err) + if not_found_binary: for key, values in not_found_binary.items(): LOGGER.logger.warning("CANNOT FOUND THE component" + key + " in all database") @@ -316,33 +322,38 @@ class SearchDB(): provides_not_found = dict() build_list = [] - build_set = [] for db_name, data_base in self.db_object_dict.items(): - temp_list = list(s_name_set) - for input_name_li in [temp_list[i:i + 900] for i in range(0, len(temp_list), 900)]: - sql_com = text(""" - SELECT DISTINCT - src.NAME AS search_name, - src.version AS search_version, - bin_pack.src_name AS source_name, - bin_provides.pkgKey AS bin_id, - src_requires.NAME AS req_name, - bin_pack.version AS version, - bin_pack.NAME AS bin_name - FROM - ( SELECT pkgKey, NAME, version FROM src_pack WHERE {} ) src - LEFT JOIN src_requires ON src.pkgKey = src_requires.pkgKey - LEFT JOIN bin_provides ON bin_provides.NAME = src_requires.NAME - LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey; - """.format(literal_column("name").in_(input_name_li))) - res = data_base.session.execute( - sql_com, - {'name_{}'.format(i): v - for i, v in enumerate(input_name_li, 1)} - ).fetchall() - - build_set.extend(res) + build_set = [] + try: + temp_list = list(s_name_set) + for input_name_li in [temp_list[i:i + 900] for i in range(0, len(temp_list), 900)]: + sql_com = text(""" + SELECT DISTINCT + src.NAME AS search_name, + src.version AS search_version, + bin_pack.src_name AS source_name, + bin_provides.pkgKey AS bin_id, + src_requires.NAME AS req_name, + bin_pack.version AS version, + bin_pack.NAME AS bin_name + FROM + ( SELECT pkgKey, NAME, version FROM src_pack WHERE {} ) src + LEFT JOIN src_requires ON src.pkgKey = src_requires.pkgKey + LEFT JOIN bin_provides ON bin_provides.NAME = src_requires.NAME + LEFT JOIN bin_pack ON bin_pack.pkgKey = bin_provides.pkgKey; + """.format(literal_column("name").in_(input_name_li))) + res = data_base.session.execute( + sql_com, + {'name_{}'.format(i): v + for i, v in enumerate(input_name_li, 1)} + ).fetchall() + + build_set.extend(res) + except AttributeError as attr_err: + current_app.logger.error(attr_err) + except SQLAlchemyError as sql_err: + current_app.logger.error(sql_err) if not build_set: continue @@ -406,7 +417,7 @@ class SearchDB(): except AttributeError as attr_err: current_app.logger.error(attr_err) except SQLAlchemyError as sql_err: - current_app.logger.err(sql_err) + current_app.logger.error(sql_err) return 'NOT FOUND' -- Gitee