From 0152aabe5a422d18f7f63e53913191ebc4211a70 Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Wed, 10 May 2023 16:39:45 +0800
Subject: [PATCH 1/2] Add network request exception capture
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 apollo/cron/download_sa_manager.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index a46ad5a..e9f89b3 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -10,7 +10,7 @@
 # PURPOSE.
 # See the Mulan PSL v2 for more details.
 # ******************************************************************************/
-from gevent import monkey; monkey.patch_all(thread=False)
+from gevent import monkey; monkey.patch_all()
 import gevent
 import datetime
 import os
@@ -67,10 +67,11 @@ class TimedDownloadSATask(TimedTaskBase):
                 download_record, download_failed_advisory = proxy.get_advisory_download_record()
                 sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list(
                     download_record)
-
-                jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
-                        for sa_name in sa_name_list]
-                gevent.joinall(jobs)
+                # Limit the number of requests to 20 per time
+                for i in range(0, len(sa_name_list), 20):
+                    jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
+                            for sa_name in sa_name_list[i: i + 20]]
+                    gevent.joinall(jobs)
 
                 TimedDownloadSATask.save_security_advisory_to_database(proxy)
 
@@ -134,10 +135,13 @@ class TimedDownloadSATask(TimedTaskBase):
             response body or "", "" means request failed
         """
         try:
-            response = urllib.request.urlopen(url, timeout=10)
+            response = urllib.request.urlopen(url, timeout=30)
             return response.read()
-        except urllib.error.HTTPError:
-            LOGGER.info("Exception %s")
+        except urllib.error.HTTPError as e:
+            LOGGER.info("Exception HTTPError %s" % e)
+            return None
+        except urllib.error.URLError as e:
+            LOGGER.info("Exception URLError %s" % e)
             return ""
 
     @staticmethod
-- 
Gitee


From 8979cee0900fa64e05a06c68b6f0032c859cb904 Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Thu, 11 May 2023 11:37:10 +0800
Subject: [PATCH 2/2] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E5=BC=95=E5=85=A5monkey.?=
 =?UTF-8?q?patch=5Fall=E7=9A=84=E4=BD=8D=E7=BD=AE=EF=BC=8C=E6=B6=88?=
 =?UTF-8?q?=E9=99=A4=E8=AD=A6=E5=91=8A=E4=BF=A1=E6=81=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 apollo/cron/download_sa_manager.py | 1 -
 apollo/manage.py                   | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index e9f89b3..9fcadef 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -10,7 +10,6 @@
 # PURPOSE.
 # See the Mulan PSL v2 for more details.
 # ******************************************************************************/
-from gevent import monkey; monkey.patch_all()
 import gevent
 import datetime
 import os
diff --git a/apollo/manage.py b/apollo/manage.py
index 5c13c70..7796fed 100644
--- a/apollo/manage.py
+++ b/apollo/manage.py
@@ -15,6 +15,7 @@ Time:
 Author:
 Description: Manager that start aops-manager
 """
+from gevent import monkey; monkey.patch_all(thread=False)
 import redis
 import sqlalchemy
 from flask import Flask
-- 
Gitee