135 lines
5.5 KiB
Diff
135 lines
5.5 KiB
Diff
diff --git a/packageship/application/apps/lifecycle/function/gitee.py b/packageship/application/apps/lifecycle/function/gitee.py
|
||
index 4ac077f..8ca4ccf 100644
|
||
--- a/packageship/application/apps/lifecycle/function/gitee.py
|
||
+++ b/packageship/application/apps/lifecycle/function/gitee.py
|
||
@@ -8,6 +8,7 @@ from json import JSONDecodeError
|
||
from retrying import retry
|
||
import requests
|
||
from requests.exceptions import HTTPError
|
||
+from requests.exceptions import RequestException
|
||
from sqlalchemy.exc import SQLAlchemyError
|
||
from packageship.libs.dbutils import DBHelper
|
||
from packageship.libs.configutils.readconfig import ReadConfig
|
||
@@ -42,6 +43,8 @@ class Gitee():
|
||
"patch_files_path")
|
||
self.table_name = table_name
|
||
self.producer_consumer = ProducerConsumer()
|
||
+ self._issue_url = None
|
||
+ self.total_page = 0
|
||
|
||
def query_issues_info(self, issue_id=""):
|
||
"""
|
||
@@ -53,55 +56,58 @@ class Gitee():
|
||
Raises:
|
||
|
||
"""
|
||
- issue_url = self.api_url + \
|
||
- "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id)
|
||
+ self._issue_url = self.api_url + \
|
||
+ "/{}/{}/issues/{}".format(self.owner, self.repo, issue_id)
|
||
try:
|
||
- response = requests.get(
|
||
- issue_url, params={"state": "all", "per_page": 100})
|
||
- except Error as error:
|
||
+ response = self._request_issue(0)
|
||
+ except (HTTPError, RequestException) as error:
|
||
LOGGER.logger.error(error)
|
||
return None
|
||
- if response.status_code != 200:
|
||
- return None
|
||
- total_page = 1 if issue_id else int(response.headers['total_page'])
|
||
+
|
||
+ self.total_page = 1 if issue_id else int(
|
||
+ response.headers['total_page'])
|
||
total_count = int(response.headers['total_count'])
|
||
+
|
||
if total_count > 0:
|
||
- issue_list = self._query_per_page_issue_info(total_page, issue_url)
|
||
+ issue_list = self._query_per_page_issue_info()
|
||
if not issue_list:
|
||
LOGGER.logger.error(
|
||
"An error occurred while querying {}".format(self.repo))
|
||
return None
|
||
self._save_issues(issue_list)
|
||
|
||
- def _query_per_page_issue_info(self, total_page, issue_url):
|
||
+ @retry(stop_max_attempt_number=3, stop_max_delay=1000)
|
||
+ def _request_issue(self, page):
|
||
+ try:
|
||
+ response = requests.get(self._issue_url,
|
||
+ params={"state": "all", "per_page": 100, "page": page})
|
||
+ except RequestException as error:
|
||
+ raise RequestException(error)
|
||
+ if response.status_code != 200:
|
||
+ _msg = "There is an exception with the remote service [%s]，" \
|
||
+ "Please try again later.The HTTP error code is：%s" % (self._issue_url, str(
|
||
+ response.status_code))
|
||
+ raise HTTPError(_msg)
|
||
+ return response
|
||
+
|
||
+ def _query_per_page_issue_info(self):
|
||
"""
|
||
Description: View the issue details
|
||
Args:
|
||
total_page: total page
|
||
- issue_url: issue url
|
||
|
||
Returns:
|
||
|
||
"""
|
||
issue_content_list = []
|
||
- for i in range(1, total_page + 1):
|
||
-
|
||
- @retry(stop_max_attempt_number=3, stop_max_delay=1000)
|
||
- def request_issue(page, issue_url):
|
||
- try:
|
||
- response = requests.get(issue_url,
|
||
- params={"state": "all", "per_page": 100, "page": page})
|
||
- except HTTPError:
|
||
- raise HTTPError('Network request error')
|
||
- return response
|
||
-
|
||
+ for i in range(1, self.total_page + 1):
|
||
try:
|
||
- response = request_issue(i, issue_url)
|
||
- if response.status_code != 200:
|
||
- LOGGER.logger.warning(response.content.decode("utf-8"))
|
||
- continue
|
||
+ response = self._request_issue(i)
|
||
issue_content_list.extend(
|
||
self.parse_issues_content(response.json()))
|
||
+ except (HTTPError, RequestException) as error:
|
||
+ LOGGER.logger.error(error)
|
||
+ continue
|
||
except (JSONDecodeError, Error) as error:
|
||
LOGGER.logger.error(error)
|
||
return issue_content_list
|
||
@@ -114,12 +120,9 @@ class Gitee():
|
||
try:
|
||
def _save(issue_module):
|
||
with DBHelper(db_name='lifecycle') as database:
|
||
-
|
||
exist_issues = database.session.query(PackagesIssue).filter(
|
||
PackagesIssue.issue_id == issue_module['issue_id']).first()
|
||
if exist_issues:
|
||
-
|
||
- # Save the issue
|
||
for key, val in issue_module.items():
|
||
setattr(exist_issues, key, val)
|
||
else:
|
||
@@ -130,11 +133,11 @@ class Gitee():
|
||
with DBHelper(db_name='lifecycle') as database:
|
||
database.add(package_module)
|
||
|
||
+ # Save the issue
|
||
for issue_item in issue_list:
|
||
- self.producer_consumer.put(
|
||
- (copy.deepcopy(issue_item), _save))
|
||
+ self.producer_consumer.put((copy.deepcopy(issue_item), _save))
|
||
|
||
- # The number of various issues in the update package
|
||
+ # The number of various issues in the update package
|
||
self.pkg_info.defect = self.defect
|
||
self.pkg_info.feature = self.feature
|
||
self.pkg_info.cve = self.cve
|