diff --git a/0001-fix-args-not-effective-bug.patch b/0001-fix-args-not-effective-bug.patch deleted file mode 100644 index e14ee0b..0000000 --- a/0001-fix-args-not-effective-bug.patch +++ /dev/null @@ -1,208 +0,0 @@ -From c7b84c97fde88b30f39f01e3345e2649714c2b7e Mon Sep 17 00:00:00 2001 -From: young <954906362@qq.com> -Date: Thu, 27 Apr 2023 17:25:40 +0800 -Subject: [PATCH] bug fix:continuously sending emails when can't read config info -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - apollo/cron/download_sa_manager.py | 43 ++++++------------- - apollo/cron/manager/__init__.py | 11 +++-- - apollo/cron/timed_correct_manager.py | 2 +- - apollo/database/proxy/cve.py | 2 +- - .../task_handler/manager/scan_manager.py | 9 ++-- - conf/apollo_crontab.ini | 3 +- - 6 files changed, 28 insertions(+), 42 deletions(-) - -diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py -index 7b3cea2..0bb6ef8 100644 ---- a/apollo/cron/download_sa_manager.py -+++ b/apollo/cron/download_sa_manager.py -@@ -17,7 +17,6 @@ import shutil - import sqlalchemy - import requests - import retrying --from lxml import etree - from retrying import retry - - from apollo.conf import configuration -@@ -37,16 +36,10 @@ class TimedDownloadSATask(TimedTaskBase): - Timed download sa tasks - """ - config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH) -- security_base_url = config_info.get( -- "download_sa", dict()).get("base_url", "") -- if not security_base_url: -- LOGGER.error("Please add base_url in configuration file") -- advisory_years = config_info.get("download_sa", dict()).get("sa_years", "") -- if advisory_years: -- advisory_years = re.findall(r"\d{4}", advisory_years) -- else: -- LOGGER.error("Please add sa_years in configuration file") -- raise KeyError -+ cvrf_url = config_info.get( -+ "download_sa", dict()).get("cvrf_url", "") -+ if not cvrf_url: -+ LOGGER.error("Please add cvrf_url in configuration file") - - save_sa_record = [] - -@@ -152,7 +145,7 @@ class TimedDownloadSATask(TimedTaskBase): - - for sa_name in sa_name_list: - advisory_year, advisory_serial_number = re.findall("\d+", sa_name) -- sa_url = f"{TimedDownloadSATask.security_base_url}/{advisory_year}/{sa_name}" -+ sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}" - try: - response = TimedDownloadSATask.get_response(sa_url) - if response: -@@ -170,22 +163,21 @@ class TimedDownloadSATask(TimedTaskBase): - "download_status": False}) - - @staticmethod -- def get_advisory_url_list(url: str) -> list: -+ def get_advisory_url_list() -> list: - """ - Send a request and parse the data on the page to get all the security bulletins url and store them in the list - -- Args: -- url(str): the url of all security announcements -- - Returns: - list: security url list - """ - try: -- response = TimedDownloadSATask.get_response(url) -+ response = TimedDownloadSATask.get_response(TimedDownloadSATask.cvrf_url + "/index.txt") - if response: -- html = etree.HTML(response.text) -+ html = response.text -+ sa_list = html.replace("\r", "").split("\n") - security_files = [ -- file for file in html.xpath("//*[@id='list']/tbody/tr/td[1]/a/@title") -+ # 2021/cvrf-openEuler-SA-2021-1022.xml, we don't need the first five characters -+ sa_name[5:] for sa_name in sa_list - ] - return security_files - else: -@@ -206,18 +198,7 @@ class TimedDownloadSATask(TimedTaskBase): - Returns: - list: The name of the sa that needs to be downloaded - """ -- today = datetime.datetime.today() -- current_year = str(today.year) -- TimedDownloadSATask.advisory_years.append(current_year) -- all_sa_name_list = [] -- -- for year in set(TimedDownloadSATask.advisory_years): -- try: -- sa_name_list = TimedDownloadSATask.get_advisory_url_list( -- TimedDownloadSATask.security_base_url + "/" + str(year) + "/") -- all_sa_name_list.extend(sa_name_list) -- except retrying.RetryError: -- LOGGER.error(f"Failed to get the {year} sa file") -+ all_sa_name_list = TimedDownloadSATask.get_advisory_url_list() - - succeed_sa_name_list = [] - for succeed_record in download_succeed_record: -diff --git a/apollo/cron/manager/__init__.py b/apollo/cron/manager/__init__.py -index 1ec9b72..1f3a855 100644 ---- a/apollo/cron/manager/__init__.py -+++ b/apollo/cron/manager/__init__.py -@@ -91,9 +91,12 @@ class TimedTaskManager(): - return - timed_task_parameters['id'] = task_id - -- if "base_url" in timed_task_parameters and "sa_years" in timed_task_parameters: -- timed_task_parameters.pop("base_url") -- timed_task_parameters.pop("sa_years") -+ if "day_of_week" not in timed_task_parameters or "hour" not in timed_task_parameters: -+ LOGGER.error("Create scheduled task is missing required fields.") -+ return -+ -+ if "cvrf_url" in timed_task_parameters: -+ timed_task_parameters.pop("cvrf_url") - if "service_timeout_threshold_min" in timed_task_parameters: - timed_task_parameters.pop("service_timeout_threshold_min") - -@@ -102,7 +105,7 @@ class TimedTaskManager(): - if "auto_start" in timed_task_parameters: - auto_start = timed_task_parameters['auto_start'] - timed_task_parameters.pop("auto_start") -- if not auto_start: -+ if auto_start == "false": - LOGGER.info(f"{task_id}, This task is configured to not start.") - return - TimedTaskManager._APscheduler.add_job(**timed_task_parameters) -diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py -index 4f1459e..fc2f62a 100644 ---- a/apollo/cron/timed_correct_manager.py -+++ b/apollo/cron/timed_correct_manager.py -@@ -33,7 +33,7 @@ class TimedCorrectTask(TimedTaskBase): - """ - config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH) - SERVICE_TIMEOUT_THRESHOLD_MIN = config_info.get( -- "correct_data").get("service_timeout_threshold_min") -+ "correct_data").get("service_timeout_threshold_min", 15) - - @staticmethod - def task_enter(): -diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py -index 1a77211..75ab71c 100644 ---- a/apollo/database/proxy/cve.py -+++ b/apollo/database/proxy/cve.py -@@ -951,7 +951,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy): - """ - Save the record of download sa - Args: -- sa_record_rows(list): aach element is a record of the AdvisoryDownloadRecord table,e.g. -+ sa_record_rows(list): each element is a record of the AdvisoryDownloadRecord table,e.g. - [{"advisory_year": 2022, - "advisory_serial_number": 1230, - "download_status": 1}] -diff --git a/apollo/handler/task_handler/manager/scan_manager.py b/apollo/handler/task_handler/manager/scan_manager.py -index efd6356..48ee71b 100644 ---- a/apollo/handler/task_handler/manager/scan_manager.py -+++ b/apollo/handler/task_handler/manager/scan_manager.py -@@ -195,16 +195,19 @@ class ScanManager(Manager): - tmp = {} - chart_data = [] - file_content = "序号,CVE_ID,主机IP,主机名称,CVSS评分,评分级别\n" -- for num, row in enumerate(rows, 1): -+ excel_row_num = 1 -+ for row in rows: - if row.host_ip in tmp: - tmp[row.host_ip]["count"] += 1 -- file_content += f"{num},{row.cve_id},{row.host_ip}," \ -+ file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \ - f"{row.host_name},{row.cvss_score},{row.severity}\n" -+ excel_row_num += 1 - else: - if row.cve_id is not None: - tmp[row.host_ip] = {"count": 1, "host_name": row.host_name} -- file_content += f"{num},{row.cve_id},{row.host_ip}," \ -+ file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \ - f"{row.host_name},{row.cvss_score},{row.severity}\n" -+ excel_row_num += 1 - else: - tmp[row.host_ip] = {"count": 0, "host_name": row.host_name} - -diff --git a/conf/apollo_crontab.ini b/conf/apollo_crontab.ini -index 3ac8eeb..7ab1bfa 100644 ---- a/conf/apollo_crontab.ini -+++ b/conf/apollo_crontab.ini -@@ -13,8 +13,7 @@ id = download sa - day_of_week = 0-6 - hour = 3 - auto_start = true --base_url = https://repo.openeuler.org/security/data/cvrf --sa_years = 2021,2022,2023 -+cvrf_url = https://repo.openeuler.org/security/data/cvrf - - [correct_data] - id = correct data --- -Gitee - - diff --git a/0001-fix-some-apis-which-has-filter-fault.patch b/0001-fix-some-apis-which-has-filter-fault.patch new file mode 100644 index 0000000..796fa51 --- /dev/null +++ b/0001-fix-some-apis-which-has-filter-fault.patch @@ -0,0 +1,147 @@ +From ca1388c59c97d31dbbdae3c48e7033dbc2d11b47 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Mon, 29 May 2023 17:05:17 +0800 +Subject: [PATCH] fix issue where some fields of the interface cannot support filtering +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + apollo/database/proxy/cve.py | 12 +++++------- + apollo/database/proxy/host.py | 7 ++++--- + apollo/database/proxy/task.py | 14 ++++---------- + apollo/function/schema/task.py | 3 ++- + 4 files changed, 15 insertions(+), 21 deletions(-) + +diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py +index ed4c1d2..9dc96ae 100644 +--- a/apollo/database/proxy/cve.py ++++ b/apollo/database/proxy/cve.py +@@ -867,17 +867,15 @@ class CveProxy(CveMysqlProxy, CveEsProxy): + exist_cve_query = self.session.query(CveHostAssociation.cve_id) \ + .join(Host, Host.host_id == CveHostAssociation.host_id) \ + .filter(Host.user == username, CveHostAssociation.affected == 1, CveHostAssociation.fixed == 0) +- # get first column value from tuple to list +- exist_cve_list = list(zip(*exist_cve_query))[0] + + related_cve_query = self.session.query(CveAffectedPkgs.cve_id) \ +- .filter(CveAffectedPkgs.package.in_(pkg_list)) \ +- .filter(CveAffectedPkgs.cve_id.in_(exist_cve_list)) +- related_cve = set(list(zip(*related_cve_query))[0]) ++ .filter(CveAffectedPkgs.package.in_(pkg_list),CveAffectedPkgs.cve_id.in_(exist_cve_query.subquery())) \ ++ .distinct() + +- related_cve.remove(cve_id) +- return list(related_cve) ++ related_cve = [row[0] for row in related_cve_query.all() if row[0] != cve_id] + ++ return related_cve ++ + @staticmethod + def _cve_info_row2dict(row, description_dict, pkg_list): + """ +diff --git a/apollo/database/proxy/host.py b/apollo/database/proxy/host.py +index f3fe51e..a9431a9 100644 +--- a/apollo/database/proxy/host.py ++++ b/apollo/database/proxy/host.py +@@ -514,7 +514,8 @@ class HostProxy(HostMysqlProxy, CveEsProxy): + Returns: + set + """ +- filters = {CveHostAssociation.fixed == filter_dict.get("fixed", False)} ++ fixed = filter_dict.get("fixed", False) ++ filters = {CveHostAssociation.fixed == fixed} + + if not filter_dict: + return filters +@@ -525,9 +526,9 @@ class HostProxy(HostMysqlProxy, CveEsProxy): + if filter_dict.get("severity"): + filters.add(Cve.severity.in_(filter_dict["severity"])) + +- if filter_dict.get("hotpatch") and filter_dict.get("fixed") is True: ++ if filter_dict.get("hotpatch") and fixed is True: + filters.add(CveHostAssociation.fixed_by_hp.in_(filter_dict["hotpatch"])) +- elif filter_dict.get("hotpatch") and filter_dict.get("fixed") is False: ++ elif filter_dict.get("hotpatch") and fixed is False: + filters.add(CveHostAssociation.support_hp.in_(filter_dict["hotpatch"])) + + if "affected" in filter_dict: +diff --git a/apollo/database/proxy/task.py b/apollo/database/proxy/task.py +index f457043..e660f02 100644 +--- a/apollo/database/proxy/task.py ++++ b/apollo/database/proxy/task.py +@@ -924,9 +924,7 @@ class TaskMysqlProxy(MysqlProxy): + filters = set() + + if filter_dict.get("cve_id"): +- filters.add(Cve.cve_id.like("%" + filter_dict["cve_id"] + "%")) +- if filter_dict.get("reboot"): +- filters.add(Cve.reboot == filter_dict["reboot"]) ++ filters.add(CveHostAssociation.cve_id.like("%" + filter_dict["cve_id"] + "%")) + return filters + + def _query_cve_task(self, username, task_id, filters): +@@ -948,12 +946,11 @@ class TaskMysqlProxy(MysqlProxy): + } + """ + task_cve_query = self.session.query(TaskCveHostAssociation.cve_id, +- Cve.reboot, + CveAffectedPkgs.package, + TaskCveHostAssociation.host_id, + TaskCveHostAssociation.status) \ +- .outerjoin(Cve, Cve.cve_id == TaskCveHostAssociation.cve_id) \ +- .outerjoin(CveAffectedPkgs, CveAffectedPkgs.cve_id == Cve.cve_id) \ ++ .outerjoin(CveHostAssociation, CveHostAssociation.cve_id == TaskCveHostAssociation.cve_id) \ ++ .outerjoin(CveAffectedPkgs, CveAffectedPkgs.cve_id == CveHostAssociation.cve_id) \ + .outerjoin(Task, Task.task_id == TaskCveHostAssociation.task_id) \ + .filter(Task.task_id == task_id, Task.username == username) \ + .filter(*filters) +@@ -969,7 +966,6 @@ class TaskMysqlProxy(MysqlProxy): + { + "cve_id": "CVE-2021-0001", + "package": "tensorflow", +- "reboot": True, + "host_id": "id1", + "status": "fixed" + } +@@ -979,7 +975,6 @@ class TaskMysqlProxy(MysqlProxy): + [{ + "cve_id": "CVE-2021-0001", + "package": "tensorflow", +- "reboot": True, + "host_num": 3, + "status": "running" + }] +@@ -991,8 +986,7 @@ class TaskMysqlProxy(MysqlProxy): + for row in task_cve_query: + cve_id = row.cve_id + if cve_id not in cve_dict: +- cve_dict[cve_id] = {"package": {row.package}, "reboot": row.reboot, +- "host_set": {row.host_id}, "status_set": {row.status}} ++ cve_dict[cve_id] = {"package": {row.package}, "host_set": {row.host_id}, "status_set": {row.status}} + else: + cve_dict[cve_id]["package"].add(row.package) + cve_dict[cve_id]["host_set"].add(row.host_id) +diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py +index 1fa776c..472fd53 100644 +--- a/apollo/function/schema/task.py ++++ b/apollo/function/schema/task.py +@@ -19,6 +19,7 @@ from marshmallow import Schema + from marshmallow import fields + from marshmallow import validate + ++from apollo.conf.constant import TaskType + + class TaskListFilterSchema(Schema): + """ +@@ -26,7 +27,7 @@ class TaskListFilterSchema(Schema): + """ + task_name = fields.String(required=False, validate=lambda s: len(s) > 0) + task_type = fields.List(fields.String( +- validate=validate.OneOf(["cve fix", "repo set"])), required=False) ++ validate=validate.OneOf([getattr(TaskType,p) for p in dir(TaskType) if p.isupper()])), required=False) + + + class GetTaskListSchema(Schema): +-- + diff --git a/0002-download-SA-collaborative-process.patch b/0002-download-SA-collaborative-process.patch deleted file mode 100644 index e486d23..0000000 --- a/0002-download-SA-collaborative-process.patch +++ /dev/null @@ -1,131 +0,0 @@ -From d6cd91cf4221f0b4d1b6af3d23a2989e6eb1a4c4 Mon Sep 17 00:00:00 2001 -From: young <954906362@qq.com> -Date: Fri, 28 Apr 2023 10:36:33 +0800 -Subject: [PATCH] download SA using a collaborative process -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - aops-apollo.spec | 2 +- - apollo/cron/download_sa_manager.py | 49 ++++++++++++++++-------------- - setup.py | 1 + - 3 files changed, 29 insertions(+), 23 deletions(-) - -diff --git a/aops-apollo.spec b/aops-apollo.spec -index 0fb2db3..2d3a771 100644 ---- a/aops-apollo.spec -+++ b/aops-apollo.spec -@@ -10,7 +10,7 @@ BuildRequires: python3-setuptools - Requires: aops-vulcanus >= v1.2.0 - Requires: python3-elasticsearch python3-flask-restful python3-marshmallow >= 3.13.0 - Requires: python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0 --Requires: python3-PyYAML python3-flask -+Requires: python3-PyYAML python3-flask python3-gevent - Requires: python3-retrying python3-lxml - Provides: aops-apollo - -diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py -index 0bb6ef8..6ce97bb 100644 ---- a/apollo/cron/download_sa_manager.py -+++ b/apollo/cron/download_sa_manager.py -@@ -18,6 +18,8 @@ import sqlalchemy - import requests - import retrying - from retrying import retry -+from gevent import monkey; monkey.patch_all(ssl=False) -+import gevent - - from apollo.conf import configuration - from apollo.conf.constant import ADVISORY_SAVED_PATH, TIMED_TASK_CONFIG_PATH -@@ -52,6 +54,11 @@ class TimedDownloadSATask(TimedTaskBase): - """ - LOGGER.info("Begin to download advisory in %s.", - str(datetime.datetime.now())) -+ -+ if os.path.exists(ADVISORY_SAVED_PATH): -+ shutil.rmtree(ADVISORY_SAVED_PATH) -+ os.makedirs(ADVISORY_SAVED_PATH) -+ - try: - with CveProxy(configuration) as proxy: - ElasticsearchProxy.connect(proxy) -@@ -61,7 +68,10 @@ class TimedDownloadSATask(TimedTaskBase): - sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list( - download_record) - -- TimedDownloadSATask.download_security_advisory(sa_name_list) -+ jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name) -+ for sa_name in sa_name_list] -+ gevent.joinall(jobs) -+ - TimedDownloadSATask.save_security_advisory_to_database(proxy) - - proxy.save_advisory_download_record( -@@ -132,35 +142,30 @@ class TimedDownloadSATask(TimedTaskBase): - return "" - - @staticmethod -- def download_security_advisory(sa_name_list: list): -+ def download_security_advisory(sa_name: str): - """ - Get each url from the list, download and save it locally, and save it to the database if the download fails - - Args: -- sa_name_list: Advisory url list, each element is the url of the security announcement -+ sa_name: sa`s name, e.g. cvrf-openEuler-SA-2021-1022.xml - """ -- if os.path.exists(ADVISORY_SAVED_PATH): -- shutil.rmtree(ADVISORY_SAVED_PATH) -- os.makedirs(ADVISORY_SAVED_PATH) -- -- for sa_name in sa_name_list: -- advisory_year, advisory_serial_number = re.findall("\d+", sa_name) -- sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}" -- try: -- response = TimedDownloadSATask.get_response(sa_url) -- if response: -- with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w: -- w.write(response.content) -- else: -- LOGGER.error(f"Download failed request timeout: {sa_name}") -- TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year, -- "advisory_serial_number": advisory_serial_number, -- "download_status": False}) -- except retrying.RetryError: -- LOGGER.error(f"Download failed max retries: {sa_name}") -+ advisory_year, advisory_serial_number = re.findall("\d+", sa_name) -+ sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}" -+ try: -+ response = TimedDownloadSATask.get_response(sa_url) -+ if response: -+ with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w: -+ w.write(response.content) -+ else: -+ LOGGER.error(f"Download failed request timeout: {sa_name}") - TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year, - "advisory_serial_number": advisory_serial_number, - "download_status": False}) -+ except retrying.RetryError: -+ LOGGER.error(f"Download failed max retries: {sa_name}") -+ TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year, -+ "advisory_serial_number": advisory_serial_number, -+ "download_status": False}) - - @staticmethod - def get_advisory_url_list() -> list: -diff --git a/setup.py b/setup.py -index 891c23e..30ab44d 100644 ---- a/setup.py -+++ b/setup.py -@@ -23,6 +23,7 @@ REQUIRES = [ - 'PyYAML', - 'retrying', - 'lxml' -+ 'gevent' - ] - - setup( --- -Gitee - diff --git a/0003-fix-send-two-emails-bug.patch b/0003-fix-send-two-emails-bug.patch deleted file mode 100644 index 117fec6..0000000 --- a/0003-fix-send-two-emails-bug.patch +++ /dev/null @@ -1,111 +0,0 @@ -From 4848a4f5e9244fe7e7dfc7958982857cf3dfeb44 Mon Sep 17 00:00:00 2001 -From: young <954906362@qq.com> -Date: Tue, 9 May 2023 09:31:04 +0800 -Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=89=E4=B8=AAissue?= - =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - aops-apollo.spec | 2 +- - apollo/cron/download_sa_manager.py | 22 ++++++++++------------ - conf/apollo.ini | 4 ++-- - 3 files changed, 13 insertions(+), 15 deletions(-) - -diff --git a/aops-apollo.spec b/aops-apollo.spec -index 2d3a771..1e15294 100644 ---- a/aops-apollo.spec -+++ b/aops-apollo.spec -@@ -20,7 +20,7 @@ Cve management service, monitor machine vulnerabilities and provide fix function - - %package -n dnf-hotpatch-plugin - Summary: dnf hotpatch plugin --Requires: python3-hawkey python3-dnf syscare -+Requires: python3-hawkey python3-dnf syscare >= 1.0.1 - - %description -n dnf-hotpatch-plugin - dnf hotpatch plugin, it's about hotpatch query and fix -diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py -index 6ce97bb..a46ad5a 100644 ---- a/apollo/cron/download_sa_manager.py -+++ b/apollo/cron/download_sa_manager.py -@@ -10,16 +10,17 @@ - # PURPOSE. - # See the Mulan PSL v2 for more details. - # ******************************************************************************/ -+from gevent import monkey; monkey.patch_all(thread=False) -+import gevent - import datetime - import os - import re - import shutil - import sqlalchemy --import requests -+import urllib.request -+import urllib.error - import retrying - from retrying import retry --from gevent import monkey; monkey.patch_all(ssl=False) --import gevent - - from apollo.conf import configuration - from apollo.conf.constant import ADVISORY_SAVED_PATH, TIMED_TASK_CONFIG_PATH -@@ -64,7 +65,6 @@ class TimedDownloadSATask(TimedTaskBase): - ElasticsearchProxy.connect(proxy) - - download_record, download_failed_advisory = proxy.get_advisory_download_record() -- - sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list( - download_record) - -@@ -134,11 +134,10 @@ class TimedDownloadSATask(TimedTaskBase): - response body or "", "" means request failed - """ - try: -- response = requests.get(url, timeout=10) -- if response.status_code != requests.codes["ok"]: -- return None -- return response -- except requests.exceptions.RequestException: -+ response = urllib.request.urlopen(url, timeout=10) -+ return response.read() -+ except urllib.error.HTTPError: -+ LOGGER.info("Exception %s") - return "" - - @staticmethod -@@ -155,7 +154,7 @@ class TimedDownloadSATask(TimedTaskBase): - response = TimedDownloadSATask.get_response(sa_url) - if response: - with open(os.path.join(ADVISORY_SAVED_PATH, sa_name), "wb")as w: -- w.write(response.content) -+ w.write(response) - else: - LOGGER.error(f"Download failed request timeout: {sa_name}") - TimedDownloadSATask.save_sa_record.append({"advisory_year": advisory_year, -@@ -178,8 +177,7 @@ class TimedDownloadSATask(TimedTaskBase): - try: - response = TimedDownloadSATask.get_response(TimedDownloadSATask.cvrf_url + "/index.txt") - if response: -- html = response.text -- sa_list = html.replace("\r", "").split("\n") -+ sa_list = response.decode("utf-8").replace("\r", "").split("\n") - security_files = [ - # 2021/cvrf-openEuler-SA-2021-1022.xml, we don't need the first five characters - sa_name[5:] for sa_name in sa_list -diff --git a/conf/apollo.ini b/conf/apollo.ini -index 412fff1..f215e63 100644 ---- a/conf/apollo.ini -+++ b/conf/apollo.ini -@@ -40,5 +40,5 @@ wsgi-file=manage.py - daemonize=/var/log/aops/uwsgi/apollo.log - http-timeout=600 - harakiri=600 --processes=2 --threads=4 -+processes=1 -+gevent=100 --- -Gitee - diff --git a/0004-Add-network-request-exception-capture.patch b/0004-Add-network-request-exception-capture.patch deleted file mode 100644 index d29316e..0000000 --- a/0004-Add-network-request-exception-capture.patch +++ /dev/null @@ -1,104 +0,0 @@ -From 0152aabe5a422d18f7f63e53913191ebc4211a70 Mon Sep 17 00:00:00 2001 -From: young <954906362@qq.com> -Date: Wed, 10 May 2023 16:39:45 +0800 -Subject: [PATCH 1/2] Add network request exception capture -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - apollo/cron/download_sa_manager.py | 20 ++++++++++++-------- - 1 file changed, 12 insertions(+), 8 deletions(-) - -diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py -index a46ad5a..e9f89b3 100644 ---- a/apollo/cron/download_sa_manager.py -+++ b/apollo/cron/download_sa_manager.py -@@ -10,7 +10,7 @@ - # PURPOSE. - # See the Mulan PSL v2 for more details. - # ******************************************************************************/ --from gevent import monkey; monkey.patch_all(thread=False) -+from gevent import monkey; monkey.patch_all() - import gevent - import datetime - import os -@@ -67,10 +67,11 @@ class TimedDownloadSATask(TimedTaskBase): - download_record, download_failed_advisory = proxy.get_advisory_download_record() - sa_name_list = TimedDownloadSATask.get_incremental_sa_name_list( - download_record) -- -- jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name) -- for sa_name in sa_name_list] -- gevent.joinall(jobs) -+ # Limit the number of requests to 20 per time -+ for i in range(0, len(sa_name_list), 20): -+ jobs = [gevent.spawn(TimedDownloadSATask.download_security_advisory, sa_name) -+ for sa_name in sa_name_list[i: i + 20]] -+ gevent.joinall(jobs) - - TimedDownloadSATask.save_security_advisory_to_database(proxy) - -@@ -134,10 +135,13 @@ class TimedDownloadSATask(TimedTaskBase): - response body or "", "" means request failed - """ - try: -- response = urllib.request.urlopen(url, timeout=10) -+ response = urllib.request.urlopen(url, timeout=30) - return response.read() -- except urllib.error.HTTPError: -- LOGGER.info("Exception %s") -+ except urllib.error.HTTPError as e: -+ LOGGER.info("Exception HTTPError %s" % e) -+ return None -+ except urllib.error.URLError as e: -+ LOGGER.info("Exception URLError %s" % e) - return "" - - @staticmethod --- -Gitee - - -From 8979cee0900fa64e05a06c68b6f0032c859cb904 Mon Sep 17 00:00:00 2001 -From: young <954906362@qq.com> -Date: Thu, 11 May 2023 11:37:10 +0800 -Subject: [PATCH 2/2] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E5=BC=95=E5=85=A5monkey.?= - =?UTF-8?q?patch=5Fall=E7=9A=84=E4=BD=8D=E7=BD=AE=EF=BC=8C=E6=B6=88?= - =?UTF-8?q?=E9=99=A4=E8=AD=A6=E5=91=8A=E4=BF=A1=E6=81=AF?= -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - apollo/cron/download_sa_manager.py | 1 - - apollo/manage.py | 1 + - 2 files changed, 1 insertion(+), 1 deletion(-) - -diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py -index e9f89b3..9fcadef 100644 ---- a/apollo/cron/download_sa_manager.py -+++ b/apollo/cron/download_sa_manager.py -@@ -10,7 +10,6 @@ - # PURPOSE. - # See the Mulan PSL v2 for more details. - # ******************************************************************************/ --from gevent import monkey; monkey.patch_all() - import gevent - import datetime - import os -diff --git a/apollo/manage.py b/apollo/manage.py -index 5c13c70..7796fed 100644 ---- a/apollo/manage.py -+++ b/apollo/manage.py -@@ -15,6 +15,7 @@ Time: - Author: - Description: Manager that start aops-manager - """ -+from gevent import monkey; monkey.patch_all(thread=False) - import redis - import sqlalchemy - from flask import Flask --- -Gitee - diff --git a/aops-apollo-v1.1.1.tar.gz b/aops-apollo-v1.1.1.tar.gz deleted file mode 100644 index c12942e..0000000 Binary files a/aops-apollo-v1.1.1.tar.gz and /dev/null differ diff --git a/aops-apollo-v1.2.0.tar.gz b/aops-apollo-v1.2.0.tar.gz deleted file mode 100644 index 12ae105..0000000 Binary files a/aops-apollo-v1.2.0.tar.gz and /dev/null differ diff --git a/aops-apollo-v1.2.1.tar.gz b/aops-apollo-v1.2.1.tar.gz new file mode 100644 index 0000000..f1a1529 Binary files /dev/null and b/aops-apollo-v1.2.1.tar.gz differ diff --git a/aops-apollo.spec b/aops-apollo.spec index dab63b1..25eb901 100644 --- a/aops-apollo.spec +++ b/aops-apollo.spec @@ -1,20 +1,17 @@ Name: aops-apollo -Version: v1.2.0 -Release: 4 +Version: v1.2.1 +Release: 2 Summary: Cve management service, monitor machine vulnerabilities and provide fix functions. License: MulanPSL2 URL: https://gitee.com/openeuler/%{name} Source0: %{name}-%{version}.tar.gz -Patch0001: 0001-fix-args-not-effective-bug.patch -Patch0002: 0002-download-SA-collaborative-process.patch -Patch0003: 0003-fix-send-two-emails-bug.patch -Patch0004: 0004-Add-network-request-exception-capture.patch +Patch0001: 0001-fix-some-apis-which-has-filter-fault.patch BuildRequires: python3-setuptools Requires: aops-vulcanus >= v1.2.0 Requires: python3-elasticsearch python3-flask-restful python3-marshmallow >= 3.13.0 Requires: python3-sqlalchemy python3-PyMySQL python3-Flask-APScheduler >= 1.11.0 -Requires: python3-PyYAML python3-flask +Requires: python3-PyYAML python3-flask python3-gevent Requires: python3-retrying python3-lxml Provides: aops-apollo @@ -29,6 +26,13 @@ Requires: python3-hawkey python3-dnf syscare >= 1.0.1 %description -n dnf-hotpatch-plugin dnf hotpatch plugin, it's about hotpatch query and fix +%package -n aops-apollo-tool +Summary: Small tools for aops-apollo, e.g. updateinfo.xml generater +Requires: python3-rpm + +%description -n aops-apollo-tool +smalltools for aops-apollo, e.g.updateinfo.xml generater + %prep %autosetup -n %{name}-%{version} -p1 @@ -36,13 +40,23 @@ dnf hotpatch plugin, it's about hotpatch query and fix # build for aops-apollo %py3_build +# build for aops-apollo-tool +pushd aops-apollo-tool +%py3_build +popd # install for aops-apollo %py3_install +# install for aops-apollo-tool +pushd aops-apollo-tool +%py3_install +popd + #install for aops-dnf-plugin cp -r hotpatch %{buildroot}/%{python3_sitelib}/dnf-plugins/ + %files %doc README.* %attr(0644,root,root) %{_sysconfdir}/aops/apollo.ini @@ -55,7 +69,23 @@ cp -r hotpatch %{buildroot}/%{python3_sitelib}/dnf-plugins/ %files -n dnf-hotpatch-plugin %{python3_sitelib}/dnf-plugins/* +%files -n aops-apollo-tool +%attr(0644,root,root) %{_sysconfdir}/aops_apollo_tool/updateinfo_config.ini +%attr(0755,root,root) %{_bindir}/gen-updateinfo +%{python3_sitelib}/aops_apollo_tool*.egg-info/* +%{python3_sitelib}/aops_apollo_tool/* + %changelog +* Wed May 31 2023 wenxin - v1.2.1-2 +- fix issue that can not be filtered by CVE ID when query cve rollbak task info +- fix issue that can not be filtered by cve rollback when query task list +- fix issue that can not be filtered by hotpatch when query host cve info +- fix issue that fail to read cve information when all realted hosts have been fixed + +* Tue May 23 2023 zhu-yuncheng - v1.2.1-1 +- Better dnf hotpatch plugin for more syscare command +- Add updateinfo.xml generation tool + * Thu May 11 2023 ptyang<1475324955@qq.com> - v1.2.0-4 - Add network request exception capture