From c4320de4881df952bcf9062c69c01a194fe28e6d Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Fri, 25 Oct 2024 18:35:30 +0800 Subject: [PATCH 1/5] update oe_review for easy handle --- advisors/gitee.py | 45 ++++++++++++++ advisors/oe_review.py | 134 +++++++++++++++++++++++++++++++++++------- 2 files changed, 158 insertions(+), 21 deletions(-) diff --git a/advisors/gitee.py b/advisors/gitee.py index b73f84ac..12b5152a 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -153,6 +153,14 @@ class Gitee(): url = url_template.format(owner=owner, pkg=repo) return self.__get_gitee(url) + def list_pr(self, repo, owner="src-openeuler"): + """ + List all prs of repo + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls?state=open&per_page=100&page=1" + url = url_template.format(owner=owner, repo=repo) + return self.__get_gitee_json(url) + def create_pr(self, repo, version, branch, owner="src-openeuler"): """ Create PR in gitee @@ -351,7 +359,44 @@ class Gitee(): return my_dir['sha'] return '' + def get_sigs(self): + """ + Get list of SIGs in openEuler + """ + sig_sha = self.__get_community_sha('master', 'sig') + + sig_tree = self.__get_community_tree(sig_sha) + sigs = {} + for sig in sig_tree: + if sig['path'] == 'sig-template': + continue + else: + sigs[sig['path']] = sig['sha'] + return sigs + + def get_openeuler_repos_by_sig(self, sig): + """ + Get openEuler repos by SIG + """ + repo_list = [] + sigs = self.get_sigs() + if sig not in sigs.keys(): + return repo_list + + openeuler_sha = self.__get_community_sha(sigs[sig], 'openeuler') + if not openeuler_sha: + return '' + + initials_tree = self.__get_community_tree(openeuler_sha) + for initials_dir in initials_tree: + openeuler_repo_tree = self.__get_community_tree(initials_dir['sha']) + for my_repo_dir in openeuler_repo_tree: + repo_name = my_repo_dir['path'] + repo_name = repo_name[:-5] + repo_list.append(repo_name) + return repo_list + def get_repos_by_sig(self, sig): """ Get repos list by sig diff --git a/advisors/oe_review.py b/advisors/oe_review.py index 45b2ff50..d9d9eb2b 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -45,7 +45,14 @@ You are a code reviewer of a openEuler Pull Request, providing feedback on the c The diff to be reviewed, which is get from the Gitee source code repo, as following: """ -def generate_review_from_ollama(pr_content, prompt, model="llama3.1:70b"): +OE_REVIEW_RATING_PROMPT=""" +Assume you are managing a software project and need sort out important patches from careless ones, +quality patches from careless patches, relevant/impactful patches from trivial patches, difficult +patches from easy patches, core changes from leaf changes. Please evaluate the below patch and give +a rating number in range 1-100. +""" + +def generate_review_from_ollama(pr_content, prompt, model="llama3.1:8b"): base_url = "http://localhost:11434/api" json_resp = [] resp = None @@ -101,16 +108,14 @@ def args_parser(): """ pars = argparse.ArgumentParser() pars.add_argument("-q", "--quiet", action='store_true', default=False, help="No log print") + pars.add_argument("-a", "--active_user", action='store_true', default=False, help="Review all PRs in repositories as maintainer or committer") pars.add_argument("-n", "--repo", type=str, help="Repository name that include group") pars.add_argument("-p", "--pull", type=str, help="Number ID of Pull Request") pars.add_argument("-u", "--url", type=str, help="URL of Pull Request") + pars.add_argument("-s", "--sig", type=str, default="TC", help="When active_user is set, review all PRs in specified SIG") pars.add_argument("-m", "--model", type=str, help="Model of selection to generate review") - pars.add_argument("-w", "--workdir", type=str, default=os.getcwd(), - help="Work directory.Default is current directory.") - pars.add_argument("-e", "--editor", type=str, default="/opt/homebrew/bin/nvim", + pars.add_argument("-e", "--editor", type=str, default="nvim", help="Editor of choice to edit content, default to nvim") - pars.add_argument("-c", "--clean", help="Clean environment", action="store_true") - pars.add_argument("-l", "--local", help="Using local checklist", action="store_true") return pars.parse_args() @@ -123,6 +128,89 @@ def edit_content(text, editor): text_new = open(path).read() return text_new + +def review_pr(user_gitee, repo_name, pull_id, group, editor): + """ + Review Pull Request + """ + review_content = "" + pull_request = user_gitee.get_pr(repo_name, pull_id, group) + + review_content += "# !{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) + if not pull_request["mergeable"]: + review_content += "# This PR is not mergeable!\n" + suggest_action = "/close" + suggest_reason = "存在冲突" + + review_content += "# This PR has following labels:\n" + for label in pull_request["labels"]: + if label['name'] == "ci_failed": + suggest_action = "/close" + suggest_reason = "CI 失败" + elif label['name'] == 'openeuler-cla/no': + suggest_action = "/close" + suggest_reason = "CLA 未签署" + elif label['name'] == 'ci_processing': + suggest_action = "暂不处理" + suggest_reason = "等待 CI 处理结果" + elif label['name'] == 'kind/wait_for_update': + suggest_action = "暂不处理" + suggest_reason = "等待提交人更新" + elif label['name'] == 'wait_confirm': + suggest_action = "暂不处理" + suggest_reason = "等待相关开发者确认" + else: + pass + review_content += f"{label['name']} " + review_content += "\n\n" + + + pr_diff = user_gitee.get_diff(repo_name, pull_id, group) + if not pr_diff: + print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo_name)) + return 1 + + if suggest_action == "/close": + review = "/close" + review_comment_raw = review + "\n" + suggest_reason + elif suggest_action == "暂不处理": + review = "暂不处理" + review_comment_raw = review + "\n" + suggest_reason + else: + review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) + review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT) + review_comment_raw = edit_content(review_content + '\n\n' + review + '\n\n' + review_rating + '\n\n' + pr_diff, editor) + + review_comment = "" + # remove lines start with hash + for l in review_comment_raw: + if l.startswith("#"): + continue + else: + review_comment += l + + user_gitee.create_pr_comment(repo_name, pull_id, review_comment, group) + if suggest_action == "/close": + print("!{id}: {title} is closed due to {reason}.".format(id=pull_id, title=pull_request["title"], reason=suggest_reason)) + elif suggest_action == "暂不处理": + print("!{id}: {title} is skipped due to {reason}.".format(id=pull_id, title=pull_request["title"], reason=suggest_reason)) + else: + print("push review list finish.") + +def review_repo(user_gitee, owner, repo, editor): + """" + Review PRs in give repo, or doing nothing if no PR + """ + result = f'{owner}/{repo}'.format(owner=owner, repo=repo) + PRs = user_gitee.list_pr(repo, owner) + if not PRs: + return + else: + for pr in PRs: + result += '\n' + "!{number}: {title}".format(number=pr["number"], title=pr["title"]) + review_pr(user_gitee, repo, pr['number'], owner, editor) + print(result) + def main(): """ Main entrance of the functionality @@ -131,28 +219,32 @@ def main(): if args.quiet: sys.stdout = open('/dev/null', 'w') sys.stderr = sys.stdout - work_dir = os.path.realpath(args.workdir) - params = extract_params(args) - if not params: - return 1 - group = params[0] - repo_name = params[1] - pull_id = params[2] + try: user_gitee = gitee.Gitee() except NameError: sys.exit(1) - pr_diff = user_gitee.get_diff(repo_name, pull_id, group) - if not pr_diff: - print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo_name)) - return 1 - review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) - review_comment = edit_content(review + '\n\n' + pr_diff, args.editor) + if args.active_user: + src_oe_repos = user_gitee.get_repos_by_sig(args.sig) + for repo in src_oe_repos: + review_repo(user_gitee, 'src-openeuler', repo, args.editor) - user_gitee.create_pr_comment(repo_name, pull_id, review_comment, group) + oe_repos = user_gitee.get_openeuler_repos_by_sig(args.sig) + for repo in oe_repos: + review_repo(user_gitee, 'openeuler', repo, args.editor) + # 获取当前用户所有担任 maintainer 的 sig，以及所有担任 committer 的 代码仓 + # 此处应考虑建立本地缓存，保留PR的标签，最近更新，如果和本地缓存相比没有更新就忽略，否则更新 review + # 然后根据不同的模型生成 review + else: + params = extract_params(args) + if not params: + return 1 + group = params[0] + repo_name = params[1] + pull_id = params[2] + review_pr(user_gitee, repo_name, pull_id, group, args.editor) - print("push review list finish.") return 0 -- Gitee From b6f79b7ecb3e8cf69f9e25e441106e3d9cbab863 Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Fri, 25 Oct 2024 18:35:30 +0800 Subject: [PATCH 2/5] update oe_review for easy handle --- advisors/oe_review.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/advisors/oe_review.py b/advisors/oe_review.py index d9d9eb2b..fcb3ddd5 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -136,6 +136,13 @@ def review_pr(user_gitee, repo_name, pull_id, group, editor): review_content = "" pull_request = user_gitee.get_pr(repo_name, pull_id, group) +<<<<<<< HEAD +======= + suggest_action = "" + + result = "" + +>>>>>>> e05f663 (update oe_review for easy handle) review_content += "# !{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) if not pull_request["mergeable"]: review_content += "# This PR is not mergeable!\n" @@ -191,11 +198,20 @@ def review_pr(user_gitee, repo_name, pull_id, group, editor): user_gitee.create_pr_comment(repo_name, pull_id, review_comment, group) if suggest_action == "/close": +<<<<<<< HEAD print("!{id}: {title} is closed due to {reason}.".format(id=pull_id, title=pull_request["title"], reason=suggest_reason)) elif suggest_action == "暂不处理": print("!{id}: {title} is skipped due to {reason}.".format(id=pull_id, title=pull_request["title"], reason=suggest_reason)) else: print("push review list finish.") +======= + result = " is closed due to {reason}.".format(reason=suggest_reason) + elif suggest_action == "暂不处理": + result = " is skipped due to {reason}.".format(reason=suggest_reason) + else: + result = " is handled and review is published." + return result +>>>>>>> e05f663 (update oe_review for easy handle) def review_repo(user_gitee, owner, repo, editor): """" @@ -207,8 +223,13 @@ def review_repo(user_gitee, owner, repo, editor): return else: for pr in PRs: +<<<<<<< HEAD result += '\n' + "!{number}: {title}".format(number=pr["number"], title=pr["title"]) review_pr(user_gitee, repo, pr['number'], owner, editor) +======= + h_res = review_pr(user_gitee, repo, pr['number'], owner, editor) + result += '\n' + "!{number}: {title}{res}".format(number=pr["number"], title=pr["title"], res=h_res) +>>>>>>> e05f663 (update oe_review for easy handle) print(result) def main(): -- Gitee From 9987dbae9b6325e20c7c4761e59645916100006e Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Fri, 25 Oct 2024 18:35:30 +0800 Subject: [PATCH 3/5] update oe_review for easy handle --- advisors/oe_review.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/advisors/oe_review.py b/advisors/oe_review.py index fcb3ddd5..10bd4802 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -136,13 +136,10 @@ def review_pr(user_gitee, repo_name, pull_id, group, editor): review_content = "" pull_request = user_gitee.get_pr(repo_name, pull_id, group) -<<<<<<< HEAD -======= suggest_action = "" result = "" ->>>>>>> e05f663 (update oe_review for easy handle) review_content += "# !{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) if not pull_request["mergeable"]: review_content += "# This PR is not mergeable!\n" @@ -198,20 +195,12 @@ def review_pr(user_gitee, repo_name, pull_id, group, editor): user_gitee.create_pr_comment(repo_name, pull_id, review_comment, group) if suggest_action == "/close": -<<<<<<< HEAD - print("!{id}: {title} is closed due to {reason}.".format(id=pull_id, title=pull_request["title"], reason=suggest_reason)) - elif suggest_action == "暂不处理": - print("!{id}: {title} is skipped due to {reason}.".format(id=pull_id, title=pull_request["title"], reason=suggest_reason)) - else: - print("push review list finish.") -======= result = " is closed due to {reason}.".format(reason=suggest_reason) elif suggest_action == "暂不处理": result = " is skipped due to {reason}.".format(reason=suggest_reason) else: result = " is handled and review is published." return result ->>>>>>> e05f663 (update oe_review for easy handle) def review_repo(user_gitee, owner, repo, editor): """" @@ -223,13 +212,8 @@ def review_repo(user_gitee, owner, repo, editor): return else: for pr in PRs: -<<<<<<< HEAD - result += '\n' + "!{number}: {title}".format(number=pr["number"], title=pr["title"]) - review_pr(user_gitee, repo, pr['number'], owner, editor) -======= h_res = review_pr(user_gitee, repo, pr['number'], owner, editor) result += '\n' + "!{number}: {title}{res}".format(number=pr["number"], title=pr["title"], res=h_res) ->>>>>>> e05f663 (update oe_review for easy handle) print(result) def main(): -- Gitee From c20511b738f262621de1dc7ca626a9f6e46db899 Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Sat, 26 Oct 2024 16:27:06 +0800 Subject: [PATCH 4/5] init threading support in heavy review batch --- advisors/oe_review.py | 259 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 243 insertions(+), 16 deletions(-) diff --git a/advisors/oe_review.py b/advisors/oe_review.py index 10bd4802..9d9b1604 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -52,6 +52,46 @@ patches from easy patches, core changes from leaf changes. Please evaluate the b a rating number in range 1-100. """ +# define data structure that contains queue and mutex lock for thread sharing +import threading +import time + +class ThreadSafeQueue: + def __init__(self): + self.queue = [] # Your data structure (list) can be replaced with any other like deque from collections + self.lock = threading.Lock() + self.condition = threading.Condition(self.lock) + + def put(self, item): + with self.lock: + self.queue.append(item) + self.condition.notify_all() # Notify all waiting threads that new item is added + + def get(self): + with self.lock: + while len(self.queue) == 0: + self.condition.wait() # Wait until there are items in the queue + item = self.queue.pop(0) + return item + + def qsize(self): + with self.lock: + return len(self.queue) + + +# 建三个队列，一个是待处理PR列表，一个是经过预处理的PR列表，一个是待提交PR列表 +# 批处理，首先关闭所有可以关闭的PR，直接合并sync且没有ci_failed的PR +# 然后对所有其他的PR再进行review +# define 3 queues to be shared across threads +# List of PRs to be reviewed, by review_repos() +PENDING_PRS = ThreadSafeQueue() +# review_pr() get pr from PENDING_PRS, if can be obviously handled, put comment into submitting_prs, otherwise, move to NEED_REVIEW_PRS +# that are being preprocessed for review +NEED_REVIEW_PRS = ThreadSafeQueue() +MANUAL_REVIEW_PRS = ThreadSafeQueue() +# PRs that are being submitted +SUBMITTING_PRS = ThreadSafeQueue() + def generate_review_from_ollama(pr_content, prompt, model="llama3.1:8b"): base_url = "http://localhost:11434/api" json_resp = [] @@ -127,7 +167,149 @@ def edit_content(text, editor): subprocess.call([editor, path]) text_new = open(path).read() return text_new + +def easy_classify(pull_request): + suggest_action = "" + suggest_reason = "" + sync_pr = False + + if not pull_request["mergeable"]: + suggest_action = "/close" + suggest_reason = "存在冲突" + + if pull_request["title"].startswith("[sync] PR-") and pull_request["user"]["login"]=="openeuler-sync-bot": + sync_pr = True + + for label in pull_request["labels"]: + if label['name'] == "ci_failed": + suggest_action = "/close" + suggest_reason = "CI 失败" + elif label['name'] == 'openeuler-cla/no': + suggest_action = "/check-cla" + suggest_reason = "CLA 未签署" + elif label['name'] == 'ci_processing': + suggest_action = "暂不处理" + suggest_reason = "等待 CI 处理结果" + elif label['name'] == 'kind/wait_for_update': + suggest_action = "暂不处理" + suggest_reason = "等待提交人更新" + elif label['name'] == 'wait_confirm': + suggest_action = "暂不处理" + suggest_reason = "等待相关开发者确认" + elif label['name'] == 'ci_successful': + if sync_pr == True: + suggest_action = "/lgtm\n/approve" + suggest_reason = "分支同步 PR，构建成功，默认合入。" + else: + pass + return suggest_action, suggest_reason + +def sort_pr(user_gitee): + while True: + item = PENDING_PRS.get() + if not item: + break + print(f"Got {item} from queue") + + pull_request = user_gitee.get_pr(item["repo"], item["number"], item["owner"]) + + suggest_action, suggest_reason = easy_classify(pull_request) + + if suggest_action == "": + need_review_pr = {} + need_review_pr['pull_request'] = pull_request + need_review_pr['pr_info'] = item + NEED_REVIEW_PRS.put(need_review_pr) + else: + review_comment_raw = suggest_action + "\n" + suggest_reason + submitting_pr = {} + submitting_pr['review_comment'] = review_comment_raw + submitting_pr['pr_info'] = item + submitting_pr['suggest_action'] = suggest_action + submitting_pr['suggest_reason'] = suggest_reason + SUBMITTING_PRS.put(submitting_pr) + + NEED_REVIEW_PRS.put(None) + +def ai_review(user_gitee): + while True: + item = NEED_REVIEW_PRS.get() + if not item: + break + pr_info = item["pr_info"] + pr_diff = user_gitee.get_diff(pr_info['repo'], pr_info['number'], pr_info['owner']) + if not pr_diff: + print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pr_info['number'], pr_info['owner'], pr_info['repo'])) + continue + review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) + review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT) + manual_review_pr = {} + manual_review_pr['pr_info'] = pr_info + manual_review_pr['pull_request'] = item['pull_request'] + manual_review_pr['pr_diff'] = pr_diff + manual_review_pr['review'] = review + manual_review_pr['review_rating'] = review_rating + MANUAL_REVIEW_PRS.put(manual_review_pr) + MANUAL_REVIEW_PRS.put(None) + +def manually_review(user_gitee, editor): + while True: + item = MANUAL_REVIEW_PRS.get() + if not item: + break + review_content = "" + pull_request = item['pull_request'] + pr_info = item["pr_info"] + review = item['review'] + review_rating = item['review_rating'] + pr_diff = item['pr_diff'] + + review_content += "!{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) + + review_content += "This PR has following labels:\n" + for label in pull_request["labels"]: + review_content += f"{label['name']} " + + review_comment_raw = edit_content(review_content + '\n\n' + review + '\n\n' + review_rating + '\n\n' + pr_diff, editor) + + submitting_pr = {} + submitting_pr['review_comment'] = review_comment_raw + submitting_pr['pr_info'] = item['pr_info'] + submitting_pr['pull_request'] = pull_request + SUBMITTING_PRS.put(submitting_pr) + + SUBMITTING_PRS.put(None) + +def submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action="", suggest_reason=""): + result = " is handled and review is published." + + user_gitee.create_pr_comment(pr_info['repo'], pr_info['number'], review_comment, pr_info['owner']) + + if suggest_action == "/close": + result = " is closed due to {reason}.".format(reason=suggest_reason) + elif suggest_action == "暂不处理": + result = " is skipped due to {reason}.".format(reason=suggest_reason) + elif suggest_action == "/lgtm\n/approve": + result = " is approved due to {reason}.".format(reason=suggest_reason) + else: + pass + print("!{number}: {title}{res}".format(number=pr_info["number"], title=pull_request["title"], res=result)) + +def submmit_review(user_gitee): + while True: + item = SUBMITTING_PRS.get() + print(item) + if not item: + break + review_comment = item['review_comment'] + pr_info = item['pr_info'] + pull_request = item['pull_request'] + suggest_action = item.get('suggest_action', "") + suggest_reason = item.get('suggest_reason', "") + + submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action, suggest_reason) + def review_pr(user_gitee, repo_name, pull_id, group, editor): """ @@ -140,19 +322,22 @@ def review_pr(user_gitee, repo_name, pull_id, group, editor): result = "" - review_content += "# !{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) + review_content += "!{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) if not pull_request["mergeable"]: - review_content += "# This PR is not mergeable!\n" + review_content += "This PR is not mergeable!\n" suggest_action = "/close" suggest_reason = "存在冲突" - review_content += "# This PR has following labels:\n" + if pull_request["title"].startswith("[sync] PR-") and pull_request["user"]["login"]=="openeuler-sync-bot": + sync_pr = True + + review_content += "This PR has following labels:\n" for label in pull_request["labels"]: if label['name'] == "ci_failed": suggest_action = "/close" suggest_reason = "CI 失败" elif label['name'] == 'openeuler-cla/no': - suggest_action = "/close" + suggest_action = "/check-cla" suggest_reason = "CLA 未签署" elif label['name'] == 'ci_processing': suggest_action = "暂不处理" @@ -163,12 +348,15 @@ def review_pr(user_gitee, repo_name, pull_id, group, editor): elif label['name'] == 'wait_confirm': suggest_action = "暂不处理" suggest_reason = "等待相关开发者确认" + elif label['name'] == 'ci_successful': + if sync_pr == True: + suggest_action = "/lgtm\n/approve" + suggest_reason = "分支同步 PR，构建成功，默认合入。" else: pass review_content += f"{label['name']} " review_content += "\n\n" - pr_diff = user_gitee.get_diff(repo_name, pull_id, group) if not pr_diff: print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo_name)) @@ -177,9 +365,15 @@ def review_pr(user_gitee, repo_name, pull_id, group, editor): if suggest_action == "/close": review = "/close" review_comment_raw = review + "\n" + suggest_reason + elif suggest_action == "/check-cla": + review = "/check-cla" + review_comment_raw = review + "\n" + suggest_reason elif suggest_action == "暂不处理": review = "暂不处理" review_comment_raw = review + "\n" + suggest_reason + elif suggest_action == "/lgtm\n/approve": + review = "/lgtm\n/approve" + review_comment_raw = review + "\n" + suggest_reason else: review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT) @@ -198,13 +392,15 @@ def review_pr(user_gitee, repo_name, pull_id, group, editor): result = " is closed due to {reason}.".format(reason=suggest_reason) elif suggest_action == "暂不处理": result = " is skipped due to {reason}.".format(reason=suggest_reason) + elif suggest_action == "/lgtm\n/approve": + result = " is approved due to {reason}.".format(reason=suggest_reason) else: result = " is handled and review is published." return result -def review_repo(user_gitee, owner, repo, editor): +def review_repo(user_gitee, owner, repo): """" - Review PRs in give repo, or doing nothing if no PR + Get PRs in give repo, or doing nothing if no PR """ result = f'{owner}/{repo}'.format(owner=owner, repo=repo) PRs = user_gitee.list_pr(repo, owner) @@ -212,9 +408,29 @@ def review_repo(user_gitee, owner, repo, editor): return else: for pr in PRs: - h_res = review_pr(user_gitee, repo, pr['number'], owner, editor) - result += '\n' + "!{number}: {title}{res}".format(number=pr["number"], title=pr["title"], res=h_res) - print(result) + pending_pr = {} + pending_pr['repo'] = repo + pending_pr['number'] = pr["number"] + pending_pr['owner'] = owner + #print(pending_pr) + PENDING_PRS.put(pending_pr) + +def generate_pending_prs(user_gitee, sig): + """ + Generate pending PRs + """ + src_oe_repos = user_gitee.get_repos_by_sig(sig) + for repo in src_oe_repos: + review_repo(user_gitee, 'src-openeuler', repo) + + oe_repos = user_gitee.get_openeuler_repos_by_sig(sig) + for repo in oe_repos: + review_repo(user_gitee, 'openeuler', repo) + + PENDING_PRS.put(None) + print("DONE PENDING GENERATE") + return 0 + def main(): """ @@ -231,13 +447,24 @@ def main(): sys.exit(1) if args.active_user: - src_oe_repos = user_gitee.get_repos_by_sig(args.sig) - for repo in src_oe_repos: - review_repo(user_gitee, 'src-openeuler', repo, args.editor) + generate_pending_prs_thread = threading.Thread(target=generate_pending_prs, args=(user_gitee, args.sig)) + sort_pr_thread = threading.Thread(target=sort_pr, args=(user_gitee,)) + ai_review_thread = threading.Thread(target=ai_review, args=(user_gitee,)) + manually_review_thread = threading.Thread(target=manually_review, args=(user_gitee, args.editor)) + submmit_review_thread = threading.Thread(target=submmit_review, args=(user_gitee,)) + + generate_pending_prs_thread.start() + sort_pr_thread.start() + ai_review_thread.start() + manually_review_thread.start() + submmit_review_thread.start() + + generate_pending_prs_thread.join() + sort_pr_thread.join() + ai_review_thread.join() + manually_review_thread.join() + submmit_review_thread.join() - oe_repos = user_gitee.get_openeuler_repos_by_sig(args.sig) - for repo in oe_repos: - review_repo(user_gitee, 'openeuler', repo, args.editor) # 获取当前用户所有担任 maintainer 的 sig，以及所有担任 committer 的 代码仓 # 此处应考虑建立本地缓存，保留PR的标签，最近更新，如果和本地缓存相比没有更新就忽略，否则更新 review # 然后根据不同的模型生成 review -- Gitee From 8256a29932eb4e717e6c2abb4cde97880d3ce301 Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Sun, 27 Oct 2024 20:03:45 +0800 Subject: [PATCH 5/5] fix and it runs --- advisors/oe_review.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/advisors/oe_review.py b/advisors/oe_review.py index 9d9b1604..468059ba 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -63,12 +63,12 @@ class ThreadSafeQueue: self.condition = threading.Condition(self.lock) def put(self, item): - with self.lock: + with self.condition: self.queue.append(item) self.condition.notify_all() # Notify all waiting threads that new item is added def get(self): - with self.lock: + with self.condition: while len(self.queue) == 0: self.condition.wait() # Wait until there are items in the queue item = self.queue.pop(0) @@ -172,7 +172,7 @@ def easy_classify(pull_request): suggest_action = "" suggest_reason = "" sync_pr = False - + if not pull_request["mergeable"]: suggest_action = "/close" suggest_reason = "存在冲突" @@ -209,7 +209,7 @@ def sort_pr(user_gitee): item = PENDING_PRS.get() if not item: break - print(f"Got {item} from queue") + #print(f"Got {item} from queue") pull_request = user_gitee.get_pr(item["repo"], item["number"], item["owner"]) @@ -224,16 +224,19 @@ def sort_pr(user_gitee): review_comment_raw = suggest_action + "\n" + suggest_reason submitting_pr = {} submitting_pr['review_comment'] = review_comment_raw + submitting_pr['pull_request'] = pull_request submitting_pr['pr_info'] = item submitting_pr['suggest_action'] = suggest_action submitting_pr['suggest_reason'] = suggest_reason SUBMITTING_PRS.put(submitting_pr) NEED_REVIEW_PRS.put(None) + print("sort pr exits") def ai_review(user_gitee): while True: item = NEED_REVIEW_PRS.get() + #print("ai review works") if not item: break pr_info = item["pr_info"] @@ -252,10 +255,12 @@ def ai_review(user_gitee): manual_review_pr['review_rating'] = review_rating MANUAL_REVIEW_PRS.put(manual_review_pr) MANUAL_REVIEW_PRS.put(None) + print("ai review exits") def manually_review(user_gitee, editor): while True: item = MANUAL_REVIEW_PRS.get() + #print("manually review works") if not item: break review_content = "" @@ -280,6 +285,7 @@ def manually_review(user_gitee, editor): SUBMITTING_PRS.put(submitting_pr) SUBMITTING_PRS.put(None) + print("manually review exits") def submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action="", suggest_reason=""): result = " is handled and review is published." @@ -299,7 +305,8 @@ def submit_review_impl(user_gitee, pr_info, pull_request, review_comment, sugges def submmit_review(user_gitee): while True: item = SUBMITTING_PRS.get() - print(item) + #print("submit review works") + #print(item) if not item: break review_comment = item['review_comment'] @@ -309,7 +316,7 @@ def submmit_review(user_gitee): suggest_reason = item.get('suggest_reason', "") submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action, suggest_reason) - + print("submit review exits") def review_pr(user_gitee, repo_name, pull_id, group, editor): """ @@ -403,7 +410,11 @@ def review_repo(user_gitee, owner, repo): Get PRs in give repo, or doing nothing if no PR """ result = f'{owner}/{repo}'.format(owner=owner, repo=repo) - PRs = user_gitee.list_pr(repo, owner) + try: + PRs = user_gitee.list_pr(repo, owner) + except urllib.URLErro as e: + print(e) + print(f'Failed to get PRs in {owner}/{repo}'.format(owner=owner, repo=repo)) if not PRs: return else: -- Gitee