diff --git a/0009-optimize-log-records-when-obtaining-issue-content.patch b/0009-optimize-log-records-when-obtaining-issue-content.patch new file mode 100644 index 0000000..2b6c0cc --- /dev/null +++ b/0009-optimize-log-records-when-obtaining-issue-content.patch @@ -0,0 +1,134 @@ +diff --git a/packageship/application/apps/lifecycle/function/gitee.py b/packageship/application/apps/lifecycle/function/gitee.py +index 4ac077f..8ca4ccf 100644 +--- a/packageship/application/apps/lifecycle/function/gitee.py ++++ b/packageship/application/apps/lifecycle/function/gitee.py +@@ -8,6 +8,7 @@ from json import JSONDecodeError + from retrying import retry + import requests + from requests.exceptions import HTTPError ++from requests.exceptions import RequestException + from sqlalchemy.exc import SQLAlchemyError + from packageship.libs.dbutils import DBHelper + from packageship.libs.configutils.readconfig import ReadConfig +@@ -42,6 +43,8 @@ class Gitee(): + "patch_files_path") + self.table_name = table_name + self.producer_consumer = ProducerConsumer() ++ self._issue_url = None ++ self.total_page = 0 + + def query_issues_info(self, issue_id=""): + """ +@@ -53,55 +56,58 @@ class Gitee(): + Raises: + + """ +- issue_url = self.api_url + \ +- "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id) ++ self._issue_url = self.api_url + \ ++ "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id) + try: +- response = requests.get( +- issue_url, params={"state": "all", "per_page": 100}) +- except Error as error: ++ response = self._request_issue(0) ++ except (HTTPError, RequestException) as error: + LOGGER.logger.error(error) + return None +- if response.status_code != 200: +- return None +- total_page = 1 if issue_id else int(response.headers['total_page']) ++ ++ self.total_page = 1 if issue_id else int( ++ response.headers['total_page']) + total_count = int(response.headers['total_count']) ++ + if total_count > 0: +- issue_list = self._query_per_page_issue_info(total_page, issue_url) ++ issue_list = self._query_per_page_issue_info() + if not issue_list: + LOGGER.logger.error( + "An error occurred while querying {}".format(self.repo)) + return None + self._save_issues(issue_list) + +- def _query_per_page_issue_info(self, total_page, issue_url): ++ @retry(stop_max_attempt_number=3, stop_max_delay=1000) ++ def _request_issue(self, page): ++ try: ++ response = requests.get(self._issue_url, ++ params={"state": "all", "per_page": 100, "page": page}) ++ except RequestException as error: ++ raise RequestException(error) ++ if response.status_code != 200: ++ _msg = "There is an exception with the remote service [%s]，" \ ++ "Please try again later.The HTTP error code is：%s" % (self._issue_url, str( ++ response.status_code)) ++ raise HTTPError(_msg) ++ return response ++ ++ def _query_per_page_issue_info(self): + """ + Description: View the issue details + Args: + total_page: total page +- issue_url: issue url + + Returns: + + """ + issue_content_list = [] +- for i in range(1, total_page + 1): +- +- @retry(stop_max_attempt_number=3, stop_max_delay=1000) +- def request_issue(page, issue_url): +- try: +- response = requests.get(issue_url, +- params={"state": "all", "per_page": 100, "page": page}) +- except HTTPError: +- raise HTTPError('Network request error') +- return response +- ++ for i in range(1, self.total_page + 1): + try: +- response = request_issue(i, issue_url) +- if response.status_code != 200: +- LOGGER.logger.warning(response.content.decode("utf-8")) +- continue ++ response = self._request_issue(i) + issue_content_list.extend( + self.parse_issues_content(response.json())) ++ except (HTTPError, RequestException) as error: ++ LOGGER.logger.error(error) ++ continue + except (JSONDecodeError, Error) as error: + LOGGER.logger.error(error) + return issue_content_list +@@ -114,12 +120,9 @@ class Gitee(): + try: + def _save(issue_module): + with DBHelper(db_name='lifecycle') as database: +- + exist_issues = database.session.query(PackagesIssue).filter( + PackagesIssue.issue_id == issue_module['issue_id']).first() + if exist_issues: +- +- # Save the issue + for key, val in issue_module.items(): + setattr(exist_issues, key, val) + else: +@@ -130,11 +133,11 @@ class Gitee(): + with DBHelper(db_name='lifecycle') as database: + database.add(package_module) + ++ # Save the issue + for issue_item in issue_list: +- self.producer_consumer.put( +- (copy.deepcopy(issue_item), _save)) ++ self.producer_consumer.put((copy.deepcopy(issue_item), _save)) + +- # The number of various issues in the update package ++ # The number of various issues in the update package + self.pkg_info.defect = self.defect + self.pkg_info.feature = self.feature + self.pkg_info.cve = self.cve diff --git a/pkgship.spec b/pkgship.spec index 3ab3333..63286a7 100644 --- a/pkgship.spec +++ b/pkgship.spec @@ -1,6 +1,6 @@ Name: pkgship Version: 1.1.0 -Release: 12 +Release: 13 Summary: Pkgship implements rpm package dependence ,maintainer, patch query and so no. License: Mulan 2.0 URL: https://gitee.com/openeuler/openEuler-Advisor @@ -29,11 +29,13 @@ Patch4: 0005-fix-the-error-when-source-package-has-no-sub-packages.patch Patch5: 0006-fix-memory_caused-service-crash-and-data-duplication-issue.patch # Fix the problem of function parameters -Patch6: 0007-fix-the-problem-of-function-parameters.patch +Patch6: 0007-fix-the-problem-of-function-parameters.patch # Fix the selfbuild error message -Patch7: 0008-fix-selfbuild-error-message.patch +Patch7: 0008-fix-selfbuild-error-message.patch +# Optimize-log-records-when-obtaining-issue-content +Patch8: 0009-optimize-log-records-when-obtaining-issue-content.patch BuildArch: noarch BuildRequires: python3-flask-restful python3-flask python3 python3-pyyaml python3-sqlalchemy @@ -82,6 +84,9 @@ rm -rf $log_path %attr(0755,root,root) %{_bindir}/pkgship %changelog +* Fri Sep 25 2020 Cheng Shaowei 1.1.0-13 +- Optimize-log-records-when-obtaining-issue-content + * Fri Sep 25 2020 Zhang Tao - 1.1.0-12 - In the selfbuild scenario, add the error message that the software package cannot be found