105 lines
3.7 KiB
Diff
105 lines
3.7 KiB
Diff
From 0152aabe5a422d18f7f63e53913191ebc4211a70 Mon Sep 17 00:00:00 2001
|
|
From: young <954906362@qq.com>
|
|
Date: Wed, 10 May 2023 16:39:45 +0800
|
|
Subject: [PATCH 1/2] Add network request exception capture
|
|
MIME-Version: 1.0
|
|
Content-Type: text/plain; charset=UTF-8
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
---
|
|
apollo/cron/download_sa_manager.py | 20 ++++++++++++--------
|
|
1 file changed, 12 insertions(+), 8 deletions(-)
|
|
|
|
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
|
|
index a46ad5a..e9f89b3 100644
|
|
--- a/apollo/cron/download_sa_manager.py
|
|
+++ b/apollo/cron/download_sa_manager.py
|
|
@@ -10,7 +10,7 @@
|
|
# PURPOSE.
|
|
# See the Mulan PSL v2 for more details.
|
|
# ******************************************************************************/
|
|
-from gevent import monkey; monkey.patch_all(thread=False)
|
|
+from gevent import monkey; monkey.patch_all()
|
|
import gevent
|
|
import datetime
|
|
import os
|
|
@@ -67,10 +67,11 @@ class TimedDownloadSATask(TimedTaskBase):
|
|
download_record, download_failed_advisory = proxy.get_advisory_download_record()
|
|
sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list(
|
|
download_record)
|
|
-
|
|
- jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
|
|
- for sa_name in sa_name_list]
|
|
- gevent.joinall(jobs)
|
|
+ # Limit the number of requests to 20 per time
|
|
+ for i in range(0, len(sa_name_list), 20):
|
|
+ jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
|
|
+ for sa_name in sa_name_list[i: i + 20]]
|
|
+ gevent.joinall(jobs)
|
|
|
|
TimedDownloadSATask.save_security_advisory_to_database(proxy)
|
|
|
|
@@ -134,10 +135,13 @@ class TimedDownloadSATask(TimedTaskBase):
|
|
response body or "", "" means request failed
|
|
"""
|
|
try:
|
|
- response = urllib.request.urlopen(url, timeout=10)
|
|
+ response = urllib.request.urlopen(url, timeout=30)
|
|
return response.read()
|
|
- except urllib.error.HTTPError:
|
|
- LOGGER.info("Exception %s")
|
|
+ except urllib.error.HTTPError as e:
|
|
+ LOGGER.info("Exception HTTPError %s" % e)
|
|
+ return None
|
|
+ except urllib.error.URLError as e:
|
|
+ LOGGER.info("Exception URLError %s" % e)
|
|
return ""
|
|
|
|
@staticmethod
|
|
--
|
|
Gitee
|
|
|
|
|
|
From 8979cee0900fa64e05a06c68b6f0032c859cb904 Mon Sep 17 00:00:00 2001
|
|
From: young <954906362@qq.com>
|
|
Date: Thu, 11 May 2023 11:37:10 +0800
|
|
Subject: [PATCH 2/2] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E5=BC=95=E5=85=A5monkey.?=
|
|
=?UTF-8?q?patch=5Fall=E7=9A=84=E4=BD=8D=E7=BD=AE=EF=BC=8C=E6=B6=88?=
|
|
=?UTF-8?q?=E9=99=A4=E8=AD=A6=E5=91=8A=E4=BF=A1=E6=81=AF?=
|
|
MIME-Version: 1.0
|
|
Content-Type: text/plain; charset=UTF-8
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
---
|
|
apollo/cron/download_sa_manager.py | 1 -
|
|
apollo/manage.py | 1 +
|
|
2 files changed, 1 insertion(+), 1 deletion(-)
|
|
|
|
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
|
|
index e9f89b3..9fcadef 100644
|
|
--- a/apollo/cron/download_sa_manager.py
|
|
+++ b/apollo/cron/download_sa_manager.py
|
|
@@ -10,7 +10,6 @@
|
|
# PURPOSE.
|
|
# See the Mulan PSL v2 for more details.
|
|
# ******************************************************************************/
|
|
-from gevent import monkey; monkey.patch_all()
|
|
import gevent
|
|
import datetime
|
|
import os
|
|
diff --git a/apollo/manage.py b/apollo/manage.py
|
|
index 5c13c70..7796fed 100644
|
|
--- a/apollo/manage.py
|
|
+++ b/apollo/manage.py
|
|
@@ -15,6 +15,7 @@ Time:
|
|
Author:
|
|
Description: Manager that start aops-manager
|
|
"""
|
|
+from gevent import monkey; monkey.patch_all(thread=False)
|
|
import redis
|
|
import sqlalchemy
|
|
from flask import Flask
|
|
--
|
|
Gitee
|
|
|