From d6cd91cf4221f0b4d1b6af3d23a2989e6eb1a4c4 Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Fri, 28 Apr 2023 10:36:33 +0800
Subject: [PATCH] download SA using a collaborative process
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 aops-apollo.spec                   |  2 +-
 apollo/cron/download_sa_manager.py | 49 ++++++++++++++++--------------
 setup.py                           |  1 +
 3 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/aops-apollo.spec b/aops-apollo.spec
index 0fb2db3..2d3a771 100644
--- a/aops-apollo.spec
+++ b/aops-apollo.spec
@@ -10,7 +10,7 @@ BuildRequires:  python3-setuptools
 Requires:   aops-vulcanus >= v1.2.0
 Requires:   python3-elasticsearch python3-flask-restful python3-marshmallow >= 3.13.0
 Requires:   python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0
-Requires:   python3-PyYAML python3-flask
+Requires:   python3-PyYAML python3-flask python3-gevent
 Requires:   python3-retrying python3-lxml
 Provides:   aops-apollo
 
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index 0bb6ef8..6ce97bb 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -18,6 +18,8 @@ import sqlalchemy
 import requests
 import retrying
 from retrying import retry
+from gevent import monkey; monkey.patch_all(ssl=False)
+import gevent
 
 from apollo.conf import configuration
 from apollo.conf.constant import ADVISORY_SAVED_PATH, TIMED_TASK_CONFIG_PATH
@@ -52,6 +54,11 @@ class TimedDownloadSATask(TimedTaskBase):
         """
         LOGGER.info("Begin to download advisory in %s.",
                     str(datetime.datetime.now()))
+
+        if os.path.exists(ADVISORY_SAVED_PATH):
+            shutil.rmtree(ADVISORY_SAVED_PATH)
+        os.makedirs(ADVISORY_SAVED_PATH)
+
         try:
             with CveProxy(configuration) as proxy:
                 ElasticsearchProxy.connect(proxy)
@@ -61,7 +68,10 @@ class TimedDownloadSATask(TimedTaskBase):
                 sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list(
                     download_record)
 
-                TimedDownloadSATask.download_security_advisory(sa_name_list)
+                jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
+                        for sa_name in sa_name_list]
+                gevent.joinall(jobs)
+
                 TimedDownloadSATask.save_security_advisory_to_database(proxy)
 
                 proxy.save_advisory_download_record(
@@ -132,35 +142,30 @@ class TimedDownloadSATask(TimedTaskBase):
             return ""
 
     @staticmethod
-    def download_security_advisory(sa_name_list: list):
+    def download_security_advisory(sa_name: str):
         """
         Get each url from the list, download and save it locally, and save it to the database if the download fails
 
         Args:
-            sa_name_list: Advisory url list, each element is the url of the security announcement
+            sa_name: sa`s name, e.g. cvrf-openEuler-SA-2021-1022.xml
         """
-        if os.path.exists(ADVISORY_SAVED_PATH):
-            shutil.rmtree(ADVISORY_SAVED_PATH)
-        os.makedirs(ADVISORY_SAVED_PATH)
-
-        for sa_name in sa_name_list:
-            advisory_year, advisory_serial_number = re.findall("\d+", sa_name)
-            sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}"
-            try:
-                response = TimedDownloadSATask.get_response(sa_url)
-                if response:
-                    with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w:
-                        w.write(response.content)
-                else:
-                    LOGGER.error(f"Download failed request timeout: {sa_name}")
-                    TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
-                                                               "advisory_serial_number": advisory_serial_number,
-                                                               "download_status": False})
-            except retrying.RetryError:
-                LOGGER.error(f"Download failed max retries: {sa_name}")
+        advisory_year, advisory_serial_number = re.findall("\d+", sa_name)
+        sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}"
+        try:
+            response = TimedDownloadSATask.get_response(sa_url)
+            if response:
+                with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w:
+                    w.write(response.content)
+            else:
+                LOGGER.error(f"Download failed request timeout: {sa_name}")
                 TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
                                                            "advisory_serial_number": advisory_serial_number,
                                                            "download_status": False})
+        except retrying.RetryError:
+            LOGGER.error(f"Download failed max retries: {sa_name}")
+            TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
+                                                       "advisory_serial_number": advisory_serial_number,
+                                                       "download_status": False})
 
     @staticmethod
     def get_advisory_url_list() -> list:
diff --git a/setup.py b/setup.py
index 891c23e..30ab44d 100644
--- a/setup.py
+++ b/setup.py
@@ -23,6 +23,7 @@ REQUIRES = [
     'PyYAML',
     'retrying',
     'lxml'
+    'gevent'
 ]
 
 setup(
-- 
Gitee