packages/aops-apollo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update to v1.2.1

Browse Source
This commit is contained in:
rabbitali 2023-06-01 20:53:46 +08:00
parent 1c2a71232b
commit 8d49852e58
9 changed files with 184 additions and 561 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

208
0001-fix-args-not-effective-bug.patch
View File

@ -1,208 +0,0 @@
From c7b84c97fde88b30f39f01e3345e2649714c2b7e Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Thu, 27 Apr 2023 17:25:40 +0800
Subject: [PATCH] bug fix:continuously sending emails when can't read config info
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
apollo/cron/download_sa_manager.py | 43 ++++++-------------
apollo/cron/manager/__init__.py | 11 +++--
apollo/cron/timed_correct_manager.py | 2 +-
apollo/database/proxy/cve.py | 2 +-
.../task_handler/manager/scan_manager.py | 9 ++--
conf/apollo_crontab.ini | 3 +-
6 files changed, 28 insertions(+), 42 deletions(-)
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index 7b3cea2..0bb6ef8 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -17,7 +17,6 @@ import shutil
import sqlalchemy
import requests
import retrying
-from lxml import etree
from retrying import retry
from apollo.conf import configuration
@@ -37,16 +36,10 @@ class TimedDownloadSATask(TimedTaskBase):
Timed download sa tasks
"""
config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH)
- security_base_url = config_info.get(
- "download_sa", dict()).get("base_url", "")
- if not security_base_url:
- LOGGER.error("Please add base_url in configuration file")
- advisory_years = config_info.get("download_sa", dict()).get("sa_years", "")
- if advisory_years:
- advisory_years = re.findall(r"\d{4}", advisory_years)
- else:
- LOGGER.error("Please add sa_years in configuration file")
- raise KeyError
+ cvrf_url = config_info.get(
+ "download_sa", dict()).get("cvrf_url", "")
+ if not cvrf_url:
+ LOGGER.error("Please add cvrf_url in configuration file")
save_sa_record = []
@@ -152,7 +145,7 @@ class TimedDownloadSATask(TimedTaskBase):
for sa_name in sa_name_list:
advisory_year, advisory_serial_number = re.findall("\d+", sa_name)
- sa_url = f"{TimedDownloadSATask.security_base_url}/{advisory_year}/{sa_name}"
+ sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}"
try:
response = TimedDownloadSATask.get_response(sa_url)
if response:
@@ -170,22 +163,21 @@ class TimedDownloadSATask(TimedTaskBase):
"download_status": False})
@staticmethod
- def get_advisory_url_list(url: str) -> list:
+ def get_advisory_url_list() -> list:
"""
Send a request and parse the data on the page to get all the security bulletins url and store them in the list
- Args:
- url(str): the url of all security announcements
-
Returns:
list: security url list
"""
try:
- response = TimedDownloadSATask.get_response(url)
+ response = TimedDownloadSATask.get_response(TimedDownloadSATask.cvrf_url + "/index.txt")
if response:
- html = etree.HTML(response.text)
+ html = response.text
+ sa_list = html.replace("\r", "").split("\n")
security_files = [
- file for file in html.xpath("//*[@id='list']/tbody/tr/td[1]/a/@title")
+ # 2021/cvrf-openEuler-SA-2021-1022.xml, we don't need the first five characters
+ sa_name[5:] for sa_name in sa_list
]
return security_files
else:
@@ -206,18 +198,7 @@ class TimedDownloadSATask(TimedTaskBase):
Returns:
list: The name of the sa that needs to be downloaded
"""
- today = datetime.datetime.today()
- current_year = str(today.year)
- TimedDownloadSATask.advisory_years.append(current_year)
- all_sa_name_list = []
-
- for year in set(TimedDownloadSATask.advisory_years):
- try:
- sa_name_list = TimedDownloadSATask.get_advisory_url_list(
- TimedDownloadSATask.security_base_url + "/" + str(year) + "/")
- all_sa_name_list.extend(sa_name_list)
- except retrying.RetryError:
- LOGGER.error(f"Failed to get the {year} sa file")
+ all_sa_name_list = TimedDownloadSATask.get_advisory_url_list()
succeed_sa_name_list = []
for succeed_record in download_succeed_record:
diff --git a/apollo/cron/manager/__init__.py b/apollo/cron/manager/__init__.py
index 1ec9b72..1f3a855 100644
--- a/apollo/cron/manager/__init__.py
+++ b/apollo/cron/manager/__init__.py
@@ -91,9 +91,12 @@ class TimedTaskManager():
return
timed_task_parameters['id'] = task_id
- if "base_url" in timed_task_parameters and "sa_years" in timed_task_parameters:
- timed_task_parameters.pop("base_url")
- timed_task_parameters.pop("sa_years")
+ if "day_of_week" not in timed_task_parameters or "hour" not in timed_task_parameters:
+ LOGGER.error("Create scheduled task is missing required fields.")
+ return
+
+ if "cvrf_url" in timed_task_parameters:
+ timed_task_parameters.pop("cvrf_url")
if "service_timeout_threshold_min" in timed_task_parameters:
timed_task_parameters.pop("service_timeout_threshold_min")
@@ -102,7 +105,7 @@ class TimedTaskManager():
if "auto_start" in timed_task_parameters:
auto_start = timed_task_parameters['auto_start']
timed_task_parameters.pop("auto_start")
- if not auto_start:
+ if auto_start == "false":
LOGGER.info(f"{task_id}, This task is configured to not start.")
return
TimedTaskManager._APscheduler.add_job(**timed_task_parameters)
diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py
index 4f1459e..fc2f62a 100644
--- a/apollo/cron/timed_correct_manager.py
+++ b/apollo/cron/timed_correct_manager.py
@@ -33,7 +33,7 @@ class TimedCorrectTask(TimedTaskBase):
"""
config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH)
SERVICE_TIMEOUT_THRESHOLD_MIN = config_info.get(
- "correct_data").get("service_timeout_threshold_min")
+ "correct_data").get("service_timeout_threshold_min", 15)
@staticmethod
def task_enter():
diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py
index 1a77211..75ab71c 100644
--- a/apollo/database/proxy/cve.py
+++ b/apollo/database/proxy/cve.py
@@ -951,7 +951,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy):
"""
Save the record of download sa
Args:
- sa_record_rows(list): aach element is a record of the AdvisoryDownloadRecord table,e.g.
+ sa_record_rows(list): each element is a record of the AdvisoryDownloadRecord table,e.g.
[{"advisory_year": 2022,
"advisory_serial_number": 1230,
"download_status": 1}]
diff --git a/apollo/handler/task_handler/manager/scan_manager.py b/apollo/handler/task_handler/manager/scan_manager.py
index efd6356..48ee71b 100644
--- a/apollo/handler/task_handler/manager/scan_manager.py
+++ b/apollo/handler/task_handler/manager/scan_manager.py
@@ -195,16 +195,19 @@ class ScanManager(Manager):
tmp = {}
chart_data = []
file_content = "序号,CVE_ID,主机IP,主机名称,CVSS评分,评分级别\n"
- for num, row in enumerate(rows, 1):
+ excel_row_num = 1
+ for row in rows:
if row.host_ip in tmp:
tmp[row.host_ip]["count"] += 1
- file_content += f"{num},{row.cve_id},{row.host_ip}," \
+ file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \
f"{row.host_name},{row.cvss_score},{row.severity}\n"
+ excel_row_num += 1
else:
if row.cve_id is not None:
tmp[row.host_ip] = {"count": 1, "host_name": row.host_name}
- file_content += f"{num},{row.cve_id},{row.host_ip}," \
+ file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \
f"{row.host_name},{row.cvss_score},{row.severity}\n"
+ excel_row_num += 1
else:
tmp[row.host_ip] = {"count": 0, "host_name": row.host_name}
diff --git a/conf/apollo_crontab.ini b/conf/apollo_crontab.ini
index 3ac8eeb..7ab1bfa 100644
--- a/conf/apollo_crontab.ini
+++ b/conf/apollo_crontab.ini
@@ -13,8 +13,7 @@ id = download sa
day_of_week = 0-6
hour = 3
auto_start = true
-base_url = https://repo.openeuler.org/security/data/cvrf
-sa_years = 2021,2022,2023
+cvrf_url = https://repo.openeuler.org/security/data/cvrf
[correct_data]
id = correct data
--
Gitee

147
0001-fix-some-apis-which-has-filter-fault.patch Normal file
View File

@ -0,0 +1,147 @@
From ca1388c59c97d31dbbdae3c48e7033dbc2d11b47 Mon Sep 17 00:00:00 2001
From: rabbitali <shusheng.wen@outlook.com>
Date: Mon, 29 May 2023 17:05:17 +0800
Subject: [PATCH] fix issue where some fields of the interface cannot support filtering
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
apollo/database/proxy/cve.py | 12 +++++-------
apollo/database/proxy/host.py | 7 ++++---
apollo/database/proxy/task.py | 14 ++++----------
apollo/function/schema/task.py | 3 ++-
4 files changed, 15 insertions(+), 21 deletions(-)
diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py
index ed4c1d2..9dc96ae 100644
--- a/apollo/database/proxy/cve.py
+++ b/apollo/database/proxy/cve.py
@@ -867,17 +867,15 @@ class CveProxy(CveMysqlProxy, CveEsProxy):
exist_cve_query = self.session.query(CveHostAssociation.cve_id) \
.join(Host, Host.host_id == CveHostAssociation.host_id) \
.filter(Host.user == username, CveHostAssociation.affected == 1, CveHostAssociation.fixed == 0)
- # get first column value from tuple to list
- exist_cve_list = list(zip(*exist_cve_query))[0]
related_cve_query = self.session.query(CveAffectedPkgs.cve_id) \
- .filter(CveAffectedPkgs.package.in_(pkg_list)) \
- .filter(CveAffectedPkgs.cve_id.in_(exist_cve_list))
- related_cve = set(list(zip(*related_cve_query))[0])
+ .filter(CveAffectedPkgs.package.in_(pkg_list),CveAffectedPkgs.cve_id.in_(exist_cve_query.subquery())) \
+ .distinct()
- related_cve.remove(cve_id)
- return list(related_cve)
+ related_cve = [row[0] for row in related_cve_query.all() if row[0] != cve_id]
+ return related_cve
+
@staticmethod
def _cve_info_row2dict(row, description_dict, pkg_list):
"""
diff --git a/apollo/database/proxy/host.py b/apollo/database/proxy/host.py
index f3fe51e..a9431a9 100644
--- a/apollo/database/proxy/host.py
+++ b/apollo/database/proxy/host.py
@@ -514,7 +514,8 @@ class HostProxy(HostMysqlProxy, CveEsProxy):
Returns:
set
"""
- filters = {CveHostAssociation.fixed == filter_dict.get("fixed", False)}
+ fixed = filter_dict.get("fixed", False)
+ filters = {CveHostAssociation.fixed == fixed}
if not filter_dict:
return filters
@@ -525,9 +526,9 @@ class HostProxy(HostMysqlProxy, CveEsProxy):
if filter_dict.get("severity"):
filters.add(Cve.severity.in_(filter_dict["severity"]))
- if filter_dict.get("hotpatch") and filter_dict.get("fixed") is True:
+ if filter_dict.get("hotpatch") and fixed is True:
filters.add(CveHostAssociation.fixed_by_hp.in_(filter_dict["hotpatch"]))
- elif filter_dict.get("hotpatch") and filter_dict.get("fixed") is False:
+ elif filter_dict.get("hotpatch") and fixed is False:
filters.add(CveHostAssociation.support_hp.in_(filter_dict["hotpatch"]))
if "affected" in filter_dict:
diff --git a/apollo/database/proxy/task.py b/apollo/database/proxy/task.py
index f457043..e660f02 100644
--- a/apollo/database/proxy/task.py
+++ b/apollo/database/proxy/task.py
@@ -924,9 +924,7 @@ class TaskMysqlProxy(MysqlProxy):
filters = set()
if filter_dict.get("cve_id"):
- filters.add(Cve.cve_id.like("%" + filter_dict["cve_id"] + "%"))
- if filter_dict.get("reboot"):
- filters.add(Cve.reboot == filter_dict["reboot"])
+ filters.add(CveHostAssociation.cve_id.like("%" + filter_dict["cve_id"] + "%"))
return filters
def _query_cve_task(self, username, task_id, filters):
@@ -948,12 +946,11 @@ class TaskMysqlProxy(MysqlProxy):
}
"""
task_cve_query = self.session.query(TaskCveHostAssociation.cve_id,
- Cve.reboot,
CveAffectedPkgs.package,
TaskCveHostAssociation.host_id,
TaskCveHostAssociation.status) \
- .outerjoin(Cve, Cve.cve_id == TaskCveHostAssociation.cve_id) \
- .outerjoin(CveAffectedPkgs, CveAffectedPkgs.cve_id == Cve.cve_id) \
+ .outerjoin(CveHostAssociation, CveHostAssociation.cve_id == TaskCveHostAssociation.cve_id) \
+ .outerjoin(CveAffectedPkgs, CveAffectedPkgs.cve_id == CveHostAssociation.cve_id) \
.outerjoin(Task, Task.task_id == TaskCveHostAssociation.task_id) \
.filter(Task.task_id == task_id, Task.username == username) \
.filter(*filters)
@@ -969,7 +966,6 @@ class TaskMysqlProxy(MysqlProxy):
{
"cve_id": "CVE-2021-0001",
"package": "tensorflow",
- "reboot": True,
"host_id": "id1",
"status": "fixed"
}
@@ -979,7 +975,6 @@ class TaskMysqlProxy(MysqlProxy):
[{
"cve_id": "CVE-2021-0001",
"package": "tensorflow",
- "reboot": True,
"host_num": 3,
"status": "running"
}]
@@ -991,8 +986,7 @@ class TaskMysqlProxy(MysqlProxy):
for row in task_cve_query:
cve_id = row.cve_id
if cve_id not in cve_dict:
- cve_dict[cve_id] = {"package": {row.package}, "reboot": row.reboot,
- "host_set": {row.host_id}, "status_set": {row.status}}
+ cve_dict[cve_id] = {"package": {row.package}, "host_set": {row.host_id}, "status_set": {row.status}}
else:
cve_dict[cve_id]["package"].add(row.package)
cve_dict[cve_id]["host_set"].add(row.host_id)
diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py
index 1fa776c..472fd53 100644
--- a/apollo/function/schema/task.py
+++ b/apollo/function/schema/task.py
@@ -19,6 +19,7 @@ from marshmallow import Schema
from marshmallow import fields
from marshmallow import validate
+from apollo.conf.constant import TaskType
class TaskListFilterSchema(Schema):
"""
@@ -26,7 +27,7 @@ class TaskListFilterSchema(Schema):
"""
task_name = fields.String(required=False, validate=lambda s: len(s) > 0)
task_type = fields.List(fields.String(
- validate=validate.OneOf(["cve fix", "repo set"])), required=False)
+ validate=validate.OneOf([getattr(TaskType,p) for p in dir(TaskType) if p.isupper()])), required=False)
class GetTaskListSchema(Schema):
--

131
0002-download-SA-collaborative-process.patch
View File

@ -1,131 +0,0 @@
From d6cd91cf4221f0b4d1b6af3d23a2989e6eb1a4c4 Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Fri, 28 Apr 2023 10:36:33 +0800
Subject: [PATCH] download SA using a collaborative process
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
aops-apollo.spec | 2 +-
apollo/cron/download_sa_manager.py | 49 ++++++++++++++++--------------
setup.py | 1 +
3 files changed, 29 insertions(+), 23 deletions(-)
diff --git a/aops-apollo.spec b/aops-apollo.spec
index 0fb2db3..2d3a771 100644
--- a/aops-apollo.spec
+++ b/aops-apollo.spec
@@ -10,7 +10,7 @@ BuildRequires: python3-setuptools
Requires: aops-vulcanus >= v1.2.0
Requires: python3-elasticsearch python3-flask-restful python3-marshmallow >= 3.13.0
Requires: python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0
-Requires: python3-PyYAML python3-flask
+Requires: python3-PyYAML python3-flask python3-gevent
Requires: python3-retrying python3-lxml
Provides: aops-apollo
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index 0bb6ef8..6ce97bb 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -18,6 +18,8 @@ import sqlalchemy
import requests
import retrying
from retrying import retry
+from gevent import monkey; monkey.patch_all(ssl=False)
+import gevent
from apollo.conf import configuration
from apollo.conf.constant import ADVISORY_SAVED_PATH, TIMED_TASK_CONFIG_PATH
@@ -52,6 +54,11 @@ class TimedDownloadSATask(TimedTaskBase):
"""
LOGGER.info("Begin to download advisory in %s.",
str(datetime.datetime.now()))
+
+ if os.path.exists(ADVISORY_SAVED_PATH):
+ shutil.rmtree(ADVISORY_SAVED_PATH)
+ os.makedirs(ADVISORY_SAVED_PATH)
+
try:
with CveProxy(configuration) as proxy:
ElasticsearchProxy.connect(proxy)
@@ -61,7 +68,10 @@ class TimedDownloadSATask(TimedTaskBase):
sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list(
download_record)
- TimedDownloadSATask.download_security_advisory(sa_name_list)
+ jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
+ for sa_name in sa_name_list]
+ gevent.joinall(jobs)
+
TimedDownloadSATask.save_security_advisory_to_database(proxy)
proxy.save_advisory_download_record(
@@ -132,35 +142,30 @@ class TimedDownloadSATask(TimedTaskBase):
return ""
@staticmethod
- def download_security_advisory(sa_name_list: list):
+ def download_security_advisory(sa_name: str):
"""
Get each url from the list, download and save it locally, and save it to the database if the download fails
Args:
- sa_name_list: Advisory url list, each element is the url of the security announcement
+ sa_name: sa`s name, e.g. cvrf-openEuler-SA-2021-1022.xml
"""
- if os.path.exists(ADVISORY_SAVED_PATH):
- shutil.rmtree(ADVISORY_SAVED_PATH)
- os.makedirs(ADVISORY_SAVED_PATH)
-
- for sa_name in sa_name_list:
- advisory_year, advisory_serial_number = re.findall("\d+", sa_name)
- sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}"
- try:
- response = TimedDownloadSATask.get_response(sa_url)
- if response:
- with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w:
- w.write(response.content)
- else:
- LOGGER.error(f"Download failed request timeout: {sa_name}")
- TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
- "advisory_serial_number": advisory_serial_number,
- "download_status": False})
- except retrying.RetryError:
- LOGGER.error(f"Download failed max retries: {sa_name}")
+ advisory_year, advisory_serial_number = re.findall("\d+", sa_name)
+ sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}"
+ try:
+ response = TimedDownloadSATask.get_response(sa_url)
+ if response:
+ with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w:
+ w.write(response.content)
+ else:
+ LOGGER.error(f"Download failed request timeout: {sa_name}")
TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
"advisory_serial_number": advisory_serial_number,
"download_status": False})
+ except retrying.RetryError:
+ LOGGER.error(f"Download failed max retries: {sa_name}")
+ TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
+ "advisory_serial_number": advisory_serial_number,
+ "download_status": False})
@staticmethod
def get_advisory_url_list() -> list:
diff --git a/setup.py b/setup.py
index 891c23e..30ab44d 100644
--- a/setup.py
+++ b/setup.py
@@ -23,6 +23,7 @@ REQUIRES = [
'PyYAML',
'retrying',
'lxml'
+ 'gevent'
]
setup(
--
Gitee

111
0003-fix-send-two-emails-bug.patch
View File

@ -1,111 +0,0 @@
From 4848a4f5e9244fe7e7dfc7958982857cf3dfeb44 Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Tue, 9 May 2023 09:31:04 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=89=E4=B8=AAissue?=
=?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
aops-apollo.spec | 2 +-
apollo/cron/download_sa_manager.py | 22 ++++++++++------------
conf/apollo.ini | 4 ++--
3 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/aops-apollo.spec b/aops-apollo.spec
index 2d3a771..1e15294 100644
--- a/aops-apollo.spec
+++ b/aops-apollo.spec
@@ -20,7 +20,7 @@ Cve management service, monitor machine vulnerabilities and provide fix function
%package -n dnf-hotpatch-plugin
Summary: dnf hotpatch plugin
-Requires: python3-hawkey python3-dnf syscare
+Requires: python3-hawkey python3-dnf syscare >= 1.0.1
%description -n dnf-hotpatch-plugin
dnf hotpatch plugin, it's about hotpatch query and fix
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index 6ce97bb..a46ad5a 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -10,16 +10,17 @@
# PURPOSE.
# See the Mulan PSL v2 for more details.
# ******************************************************************************/
+from gevent import monkey; monkey.patch_all(thread=False)
+import gevent
import datetime
import os
import re
import shutil
import sqlalchemy
-import requests
+import urllib.request
+import urllib.error
import retrying
from retrying import retry
-from gevent import monkey; monkey.patch_all(ssl=False)
-import gevent
from apollo.conf import configuration
from apollo.conf.constant import ADVISORY_SAVED_PATH, TIMED_TASK_CONFIG_PATH
@@ -64,7 +65,6 @@ class TimedDownloadSATask(TimedTaskBase):
ElasticsearchProxy.connect(proxy)
download_record, download_failed_advisory = proxy.get_advisory_download_record()
-
sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list(
download_record)
@@ -134,11 +134,10 @@ class TimedDownloadSATask(TimedTaskBase):
response body or "", "" means request failed
"""
try:
- response = requests.get(url, timeout=10)
- if response.status_code != requests.codes["ok"]:
- return None
- return response
- except requests.exceptions.RequestException:
+ response = urllib.request.urlopen(url, timeout=10)
+ return response.read()
+ except urllib.error.HTTPError:
+ LOGGER.info("Exception %s")
return ""
@staticmethod
@@ -155,7 +154,7 @@ class TimedDownloadSATask(TimedTaskBase):
response = TimedDownloadSATask.get_response(sa_url)
if response:
with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w:
- w.write(response.content)
+ w.write(response)
else:
LOGGER.error(f"Download failed request timeout: {sa_name}")
TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year,
@@ -178,8 +177,7 @@ class TimedDownloadSATask(TimedTaskBase):
try:
response = TimedDownloadSATask.get_response(TimedDownloadSATask.cvrf_url + "/index.txt")
if response:
- html = response.text
- sa_list = html.replace("\r", "").split("\n")
+ sa_list = response.decode("utf-8").replace("\r", "").split("\n")
security_files = [
# 2021/cvrf-openEuler-SA-2021-1022.xml, we don't need the first five characters
sa_name[5:] for sa_name in sa_list
diff --git a/conf/apollo.ini b/conf/apollo.ini
index 412fff1..f215e63 100644
--- a/conf/apollo.ini
+++ b/conf/apollo.ini
@@ -40,5 +40,5 @@ wsgi-file=manage.py
daemonize=/var/log/aops/uwsgi/apollo.log
http-timeout=600
harakiri=600
-processes=2
-threads=4
+processes=1
+gevent=100
--
Gitee

104
0004-Add-network-request-exception-capture.patch
View File

@ -1,104 +0,0 @@
From 0152aabe5a422d18f7f63e53913191ebc4211a70 Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Wed, 10 May 2023 16:39:45 +0800
Subject: [PATCH 1/2] Add network request exception capture
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
apollo/cron/download_sa_manager.py | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index a46ad5a..e9f89b3 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -10,7 +10,7 @@
# PURPOSE.
# See the Mulan PSL v2 for more details.
# ******************************************************************************/
-from gevent import monkey; monkey.patch_all(thread=False)
+from gevent import monkey; monkey.patch_all()
import gevent
import datetime
import os
@@ -67,10 +67,11 @@ class TimedDownloadSATask(TimedTaskBase):
download_record, download_failed_advisory = proxy.get_advisory_download_record()
sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list(
download_record)
-
- jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
- for sa_name in sa_name_list]
- gevent.joinall(jobs)
+ # Limit the number of requests to 20 per time
+ for i in range(0, len(sa_name_list), 20):
+ jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name)
+ for sa_name in sa_name_list[i: i + 20]]
+ gevent.joinall(jobs)
TimedDownloadSATask.save_security_advisory_to_database(proxy)
@@ -134,10 +135,13 @@ class TimedDownloadSATask(TimedTaskBase):
response body or "", "" means request failed
"""
try:
- response = urllib.request.urlopen(url, timeout=10)
+ response = urllib.request.urlopen(url, timeout=30)
return response.read()
- except urllib.error.HTTPError:
- LOGGER.info("Exception %s")
+ except urllib.error.HTTPError as e:
+ LOGGER.info("Exception HTTPError %s" % e)
+ return None
+ except urllib.error.URLError as e:
+ LOGGER.info("Exception URLError %s" % e)
return ""
@staticmethod
--
Gitee
From 8979cee0900fa64e05a06c68b6f0032c859cb904 Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Thu, 11 May 2023 11:37:10 +0800
Subject: [PATCH 2/2] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E5=BC=95=E5=85=A5monkey.?=
=?UTF-8?q?patch=5Fall=E7=9A=84=E4=BD=8D=E7=BD=AE=EF=BC=8C=E6=B6=88?=
=?UTF-8?q?=E9=99=A4=E8=AD=A6=E5=91=8A=E4=BF=A1=E6=81=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
apollo/cron/download_sa_manager.py | 1 -
apollo/manage.py | 1 +
2 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index e9f89b3..9fcadef 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -10,7 +10,6 @@
# PURPOSE.
# See the Mulan PSL v2 for more details.
# ******************************************************************************/
-from gevent import monkey; monkey.patch_all()
import gevent
import datetime
import os
diff --git a/apollo/manage.py b/apollo/manage.py
index 5c13c70..7796fed 100644
--- a/apollo/manage.py
+++ b/apollo/manage.py
@@ -15,6 +15,7 @@ Time:
Author:
Description: Manager that start aops-manager
"""
+from gevent import monkey; monkey.patch_all(thread=False)
import redis
import sqlalchemy
from flask import Flask
--
Gitee

BIN
aops-apollo-v1.1.1.tar.gz
View File

Binary file not shown.

BIN
aops-apollo-v1.2.0.tar.gz
View File

Binary file not shown.

BIN
aops-apollo-v1.2.1.tar.gz Normal file
View File

Binary file not shown.

44
aops-apollo.spec
View File

@ -1,20 +1,17 @@
Name: aops-apollo
Version: v1.2.0
Release: 4
Version: v1.2.1
Release: 2
Summary: Cve management service, monitor machine vulnerabilities and provide fix functions.
License: MulanPSL2
URL: https://gitee.com/openeuler/%{name}
Source0: %{name}-%{version}.tar.gz
Patch0001: 0001-fix-args-not-effective-bug.patch
Patch0002: 0002-download-SA-collaborative-process.patch
Patch0003: 0003-fix-send-two-emails-bug.patch
Patch0004: 0004-Add-network-request-exception-capture.patch
Patch0001: 0001-fix-some-apis-which-has-filter-fault.patch
BuildRequires: python3-setuptools
Requires: aops-vulcanus >= v1.2.0
Requires: python3-elasticsearch python3-flask-restful python3-marshmallow >= 3.13.0
Requires: python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0
Requires: python3-PyYAML python3-flask
Requires: python3-PyYAML python3-flask python3-gevent
Requires: python3-retrying python3-lxml
Provides: aops-apollo
@ -29,6 +26,13 @@ Requires: python3-hawkey python3-dnf syscare >= 1.0.1
%description -n dnf-hotpatch-plugin
dnf hotpatch plugin, it's about hotpatch query and fix
%package -n aops-apollo-tool
Summary: Small tools for aops-apollo, e.g. updateinfo.xml generater
Requires: python3-rpm
%description -n aops-apollo-tool
smalltools for aops-apollo, e.g.updateinfo.xml generater
%prep
%autosetup -n %{name}-%{version} -p1
@ -36,13 +40,23 @@ dnf hotpatch plugin, it's about hotpatch query and fix
# build for aops-apollo
%py3_build
# build for aops-apollo-tool
pushd aops-apollo-tool
%py3_build
popd
# install for aops-apollo
%py3_install
# install for aops-apollo-tool
pushd aops-apollo-tool
%py3_install
popd
#install for aops-dnf-plugin
cp -r hotpatch %{buildroot}/%{python3_sitelib}/dnf-plugins/
%files
%doc README.*
%attr(0644,root,root) %{_sysconfdir}/aops/apollo.ini
@ -55,7 +69,23 @@ cp -r hotpatch %{buildroot}/%{python3_sitelib}/dnf-plugins/
%files -n dnf-hotpatch-plugin
%{python3_sitelib}/dnf-plugins/*
%files -n aops-apollo-tool
%attr(0644,root,root) %{_sysconfdir}/aops_apollo_tool/updateinfo_config.ini
%attr(0755,root,root) %{_bindir}/gen-updateinfo
%{python3_sitelib}/aops_apollo_tool*.egg-info/*
%{python3_sitelib}/aops_apollo_tool/*
%changelog
* Wed May 31 2023 wenxin<shusheng.wen@outlook.com> - v1.2.1-2
- fix issue that can not be filtered by CVE ID when query cve rollbak task info
- fix issue that can not be filtered by cve rollback when query task list
- fix issue that can not be filtered by hotpatch when query host cve info
- fix issue that fail to read cve information when all realted hosts have been fixed
* Tue May 23 2023 zhu-yuncheng<zhuyuncheng@huawei.com> - v1.2.1-1
- Better dnf hotpatch plugin for more syscare command
- Add updateinfo.xml generation tool
* Thu May 11 2023 ptyang<1475324955@qq.com> - v1.2.0-4
- Add network request exception capture