132 lines
5.5 KiB
Diff
132 lines
5.5 KiB
Diff
From d6cd91cf4221f0b4d1b6af3d23a2989e6eb1a4c4 Mon Sep 17 00:00:00 2001
|
|
From: young <954906362@qq.com>
|
|
Date: Fri, 28 Apr 2023 10:36:33 +0800
|
|
Subject: [PATCH] download SA using a collaborative process
|
|
MIME-Version: 1.0
|
|
Content-Type: text/plain; charset=UTF-8
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
---
|
|
aops-apollo.spec | 2 +-
|
|
apollo/cron/download_sa_manager.py | 49 ++++++++++++++++--------------
|
|
setup.py | 1 +
|
|
3 files changed, 29 insertions(+), 23 deletions(-)
|
|
|
|
diff --git a/aops-apollo.spec b/aops-apollo.spec
|
|
index 0fb2db3..2d3a771 100644
|
|
--- a/aops-apollo.spec
|
|
+++ b/aops-apollo.spec
|
|
@@ -10,7 +10,7 @@ BuildRequires: python3-setuptools
|
|
Requires: aops-vulcanus >= v1.2.0
|
|
Requires: python3-elasticsearch python3-flask-restful python3-marshmallow >= 3.13.0
|
|
Requires: python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0
|
|
-Requires: python3-PyYAML python3-flask
|
|
+Requires: python3-PyYAML python3-flask python3-gevent
|
|
Requires: python3-retrying python3-lxml
|
|
Provides: aops-apollo
|
|
|
|
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
|
|
index 0bb6ef8..6ce97bb 100644
|
|
--- a/apollo/cron/download_sa_manager.py
|
|
+++ b/apollo/cron/download_sa_manager.py
|
|
@@ -18,6 +18,8 @@ import sqlalchemy
|
|
import requests
|
|
import retrying
|
|
from retrying import retry
|
|
+from gevent import monkey; monkey.patch_all(ssl=False)
|
|
+import gevent
|
|
|
|
from apollo.conf import configuration
|
|
from apollo.conf.constant import ADVISORY_SAVED_PATH, TIMED_TASK_CONFIG_PATH
|
|
@@ -52,6 +54,11 @@ class TimedDownloadSATask(TimedTaskBase):
|
|
"""
|
|
LOGGER.info("Begin to download advisory in %s.",
|
|
str(datetime.datetime.now()))
|
|
+
|
|
+ if os.path.exists(ADVISORY_SAVED_PATH):
|
|
+ shutil.rmtree(ADVISORY_SAVED_PATH)
|
|
+ os.makedirs(ADVISORY_SAVED_PATH)
|
|
+
|
|
try:
|
|
with CveProxy(configuration) as proxy:
|
|
ElasticsearchProxy.connect(proxy)
|
|
@@ -61,7 +68,10 @@ class TimedDownloadSATask(TimedTaskBase):
|
|
sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list(
|
|
download_record)
|
|
|
|
- TimedDownloadSATask.download_security_advisory(sa_name_list)
|
|
+ jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
|
|
+ for sa_name in sa_name_list]
|
|
+ gevent.joinall(jobs)
|
|
+
|
|
TimedDownloadSATask.save_security_advisory_to_database(proxy)
|
|
|
|
proxy.save_advisory_download_record(
|
|
@@ -132,35 +142,30 @@ class TimedDownloadSATask(TimedTaskBase):
|
|
return ""
|
|
|
|
@staticmethod
|
|
- def download_security_advisory(sa_name_list: list):
|
|
+ def download_security_advisory(sa_name: str):
|
|
"""
|
|
Get each url from the list, download and save it locally, and save it to the database if the download fails
|
|
|
|
Args:
|
|
- sa_name_list: Advisory url list, each element is the url of the security announcement
|
|
+ sa_name: sa`s name, e.g. cvrf-openEuler-SA-2021-1022.xml
|
|
"""
|
|
- if os.path.exists(ADVISORY_SAVED_PATH):
|
|
- shutil.rmtree(ADVISORY_SAVED_PATH)
|
|
- os.makedirs(ADVISORY_SAVED_PATH)
|
|
-
|
|
- for sa_name in sa_name_list:
|
|
- advisory_year, advisory_serial_number = re.findall("\d+", sa_name)
|
|
- sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}"
|
|
- try:
|
|
- response = TimedDownloadSATask.get_response(sa_url)
|
|
- if response:
|
|
- with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w:
|
|
- w.write(response.content)
|
|
- else:
|
|
- LOGGER.error(f"Download failed request timeout: {sa_name}")
|
|
- TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
|
|
- "advisory_serial_number": advisory_serial_number,
|
|
- "download_status": False})
|
|
- except retrying.RetryError:
|
|
- LOGGER.error(f"Download failed max retries: {sa_name}")
|
|
+ advisory_year, advisory_serial_number = re.findall("\d+", sa_name)
|
|
+ sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}"
|
|
+ try:
|
|
+ response = TimedDownloadSATask.get_response(sa_url)
|
|
+ if response:
|
|
+ with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w:
|
|
+ w.write(response.content)
|
|
+ else:
|
|
+ LOGGER.error(f"Download failed request timeout: {sa_name}")
|
|
TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
|
|
"advisory_serial_number": advisory_serial_number,
|
|
"download_status": False})
|
|
+ except retrying.RetryError:
|
|
+ LOGGER.error(f"Download failed max retries: {sa_name}")
|
|
+ TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
|
|
+ "advisory_serial_number": advisory_serial_number,
|
|
+ "download_status": False})
|
|
|
|
@staticmethod
|
|
def get_advisory_url_list() -> list:
|
|
diff --git a/setup.py b/setup.py
|
|
index 891c23e..30ab44d 100644
|
|
--- a/setup.py
|
|
+++ b/setup.py
|
|
@@ -23,6 +23,7 @@ REQUIRES = [
|
|
'PyYAML',
|
|
'retrying',
|
|
'lxml'
|
|
+ 'gevent'
|
|
]
|
|
|
|
setup(
|
|
--
|
|
Gitee
|
|
|