add log

0009-optimize-log-records-when-obtaining-issue-content.patch

@@ -0,0 +1,134 @@
+diff --git a/packageship/application/apps/lifecycle/function/gitee.py b/packageship/application/apps/lifecycle/function/gitee.py
+index 4ac077f..8ca4ccf 100644
+--- a/packageship/application/apps/lifecycle/function/gitee.py
++++ b/packageship/application/apps/lifecycle/function/gitee.py
+@@ -8,6 +8,7 @@ from json import JSONDecodeError
+ from retrying import retry
+ import requests
+ from requests.exceptions import HTTPError
++from requests.exceptions import RequestException
+ from sqlalchemy.exc import SQLAlchemyError
+ from packageship.libs.dbutils import DBHelper
+ from packageship.libs.configutils.readconfig import ReadConfig
+@@ -42,6 +43,8 @@ class Gitee():
+             "patch_files_path")
+         self.table_name = table_name
+         self.producer_consumer = ProducerConsumer()
++        self._issue_url = None
++        self.total_page = 0
+ 
+     def query_issues_info(self, issue_id=""):
+         """
+@@ -53,55 +56,58 @@ class Gitee():
+         Raises:
+ 
+         """
+-        issue_url = self.api_url + \
+-                    "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id)
++        self._issue_url = self.api_url + \
++                          "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id)
+         try:
+-            response = requests.get(
+-                issue_url, params={"state": "all", "per_page": 100})
+-        except Error as error:
++            response = self._request_issue(0)
++        except (HTTPError, RequestException) as error:
+             LOGGER.logger.error(error)
+             return None
+-        if response.status_code != 200:
+-            return None
+-        total_page = 1 if issue_id else int(response.headers['total_page'])
++
++        self.total_page = 1 if issue_id else int(
++            response.headers['total_page'])
+         total_count = int(response.headers['total_count'])
++
+         if total_count > 0:
+-            issue_list = self._query_per_page_issue_info(total_page, issue_url)
++            issue_list = self._query_per_page_issue_info()
+             if not issue_list:
+                 LOGGER.logger.error(
+                     "An error occurred while querying {}".format(self.repo))
+                 return None
+             self._save_issues(issue_list)
+ 
+-    def _query_per_page_issue_info(self, total_page, issue_url):
++    @retry(stop_max_attempt_number=3, stop_max_delay=1000)
++    def _request_issue(self, page):
++        try:
++            response = requests.get(self._issue_url,
++                                    params={"state": "all", "per_page": 100, "page": page})
++        except RequestException as error:
++            raise RequestException(error)
++        if response.status_code != 200:
++            _msg = "There is an exception with the remote service [%s]，" \
++                   "Please try again later.The HTTP error code is：%s" % (self._issue_url, str(
++                response.status_code))
++            raise HTTPError(_msg)
++        return response
++
++    def _query_per_page_issue_info(self):
+         """
+         Description: View the issue details
+         Args:
+             total_page: total page
+-            issue_url: issue url
+ 
+         Returns:
+ 
+         """
+         issue_content_list = []
+-        for i in range(1, total_page + 1):
+-
+-            @retry(stop_max_attempt_number=3, stop_max_delay=1000)
+-            def request_issue(page, issue_url):
+-                try:
+-                    response = requests.get(issue_url,
+-                                            params={"state": "all", "per_page": 100, "page": page})
+-                except HTTPError:
+-                    raise HTTPError('Network request error')
+-                return response
+-
++        for i in range(1, self.total_page + 1):
+             try:
+-                response = request_issue(i, issue_url)
+-                if response.status_code != 200:
+-                    LOGGER.logger.warning(response.content.decode("utf-8"))
+-                    continue
++                response = self._request_issue(i)
+                 issue_content_list.extend(
+                     self.parse_issues_content(response.json()))
++            except (HTTPError, RequestException) as error:
++                LOGGER.logger.error(error)
++                continue
+             except (JSONDecodeError, Error) as error:
+                 LOGGER.logger.error(error)
+         return issue_content_list
+@@ -114,12 +120,9 @@ class Gitee():
+         try:
+             def _save(issue_module):
+                 with DBHelper(db_name='lifecycle') as database:
+-
+                     exist_issues = database.session.query(PackagesIssue).filter(
+                         PackagesIssue.issue_id == issue_module['issue_id']).first()
+                     if exist_issues:
+-
+-                        # Save the issue
+                         for key, val in issue_module.items():
+                             setattr(exist_issues, key, val)
+                     else:
+@@ -130,11 +133,11 @@ class Gitee():
+                 with DBHelper(db_name='lifecycle') as database:
+                     database.add(package_module)
+ 
++            # Save the issue
+             for issue_item in issue_list:
+-                self.producer_consumer.put(
+-                    (copy.deepcopy(issue_item), _save))
++                self.producer_consumer.put((copy.deepcopy(issue_item), _save))
+ 
+-                # The number of various issues in the update package
++            # The number of various issues in the update package
+             self.pkg_info.defect = self.defect
+             self.pkg_info.feature = self.feature
+             self.pkg_info.cve = self.cve

pkgship.spec

@@ -1,6 +1,6 @@
 Name:           pkgship
 Version:        1.1.0
-Release:        12
+Release:        13
 Summary:        Pkgship implements rpm package dependence ,maintainer, patch query and so no.
 License:        Mulan 2.0
 URL:            https://gitee.com/openeuler/openEuler-Advisor
@@ -29,11 +29,13 @@ Patch4:         0005-fix-the-error-when-source-package-has-no-sub-packages.patch
 Patch5:         0006-fix-memory_caused-service-crash-and-data-duplication-issue.patch
 
 # Fix the problem of function parameters
-Patch6:		0007-fix-the-problem-of-function-parameters.patch
+Patch6:		    0007-fix-the-problem-of-function-parameters.patch
 
 # Fix the selfbuild error message
-Patch7:		0008-fix-selfbuild-error-message.patch
+Patch7:		    0008-fix-selfbuild-error-message.patch
 
+# Optimize-log-records-when-obtaining-issue-content
+Patch8:         0009-optimize-log-records-when-obtaining-issue-content.patch
 BuildArch:      noarch
 
 BuildRequires: python3-flask-restful python3-flask python3 python3-pyyaml python3-sqlalchemy
@@ -82,6 +84,9 @@ rm -rf $log_path
 %attr(0755,root,root) %{_bindir}/pkgship
 
 %changelog
+* Fri Sep 25 2020 Cheng Shaowei <chenshaowei3@huawei.com> 1.1.0-13
+- Optimize-log-records-when-obtaining-issue-content
+
 * Fri Sep 25 2020 Zhang Tao <zhangtao307@huawei.com> - 1.1.0-12
 - In the selfbuild scenario, add the error message that the software package cannot be found