From c7b84c97fde88b30f39f01e3345e2649714c2b7e Mon Sep 17 00:00:00 2001 From: young <954906362@qq.com> Date: Thu, 27 Apr 2023 17:25:40 +0800 Subject: [PATCH] bug fix:continuously sending emails when can't read config info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apollo/cron/download_sa_manager.py | 43 ++++++------------- apollo/cron/manager/__init__.py | 11 +++-- apollo/cron/timed_correct_manager.py | 2 +- apollo/database/proxy/cve.py | 2 +- .../task_handler/manager/scan_manager.py | 9 ++-- conf/apollo_crontab.ini | 3 +- 6 files changed, 28 insertions(+), 42 deletions(-) diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py index 7b3cea2..0bb6ef8 100644 --- a/apollo/cron/download_sa_manager.py +++ b/apollo/cron/download_sa_manager.py @@ -17,7 +17,6 @@ import shutil import sqlalchemy import requests import retrying -from lxml import etree from retrying import retry from apollo.conf import configuration @@ -37,16 +36,10 @@ class TimedDownloadSATask(TimedTaskBase): Timed download sa tasks """ config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH) - security_base_url = config_info.get( - "download_sa", dict()).get("base_url", "") - if not security_base_url: - LOGGER.error("Please add base_url in configuration file") - advisory_years = config_info.get("download_sa", dict()).get("sa_years", "") - if advisory_years: - advisory_years = re.findall(r"\d{4}", advisory_years) - else: - LOGGER.error("Please add sa_years in configuration file") - raise KeyError + cvrf_url = config_info.get( + "download_sa", dict()).get("cvrf_url", "") + if not cvrf_url: + LOGGER.error("Please add cvrf_url in configuration file") save_sa_record = [] @@ -152,7 +145,7 @@ class TimedDownloadSATask(TimedTaskBase): for sa_name in sa_name_list: advisory_year, advisory_serial_number = re.findall("\d+", sa_name) - sa_url = f"{TimedDownloadSATask.security_base_url}/{advisory_year}/{sa_name}" + sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}" try: response = TimedDownloadSATask.get_response(sa_url) if response: @@ -170,22 +163,21 @@ class TimedDownloadSATask(TimedTaskBase): "download_status": False}) @staticmethod - def get_advisory_url_list(url: str) -> list: + def get_advisory_url_list() -> list: """ Send a request and parse the data on the page to get all the security bulletins url and store them in the list - Args: - url(str): the url of all security announcements - Returns: list: security url list """ try: - response = TimedDownloadSATask.get_response(url) + response = TimedDownloadSATask.get_response(TimedDownloadSATask.cvrf_url + "/index.txt") if response: - html = etree.HTML(response.text) + html = response.text + sa_list = html.replace("\r", "").split("\n") security_files = [ - file for file in html.xpath("//*[@id='list']/tbody/tr/td[1]/a/@title") + # 2021/cvrf-openEuler-SA-2021-1022.xml, we don't need the first five characters + sa_name[5:] for sa_name in sa_list ] return security_files else: @@ -206,18 +198,7 @@ class TimedDownloadSATask(TimedTaskBase): Returns: list: The name of the sa that needs to be downloaded """ - today = datetime.datetime.today() - current_year = str(today.year) - TimedDownloadSATask.advisory_years.append(current_year) - all_sa_name_list = [] - - for year in set(TimedDownloadSATask.advisory_years): - try: - sa_name_list = TimedDownloadSATask.get_advisory_url_list( - TimedDownloadSATask.security_base_url + "/" + str(year) + "/") - all_sa_name_list.extend(sa_name_list) - except retrying.RetryError: - LOGGER.error(f"Failed to get the {year} sa file") + all_sa_name_list = TimedDownloadSATask.get_advisory_url_list() succeed_sa_name_list = [] for succeed_record in download_succeed_record: diff --git a/apollo/cron/manager/__init__.py b/apollo/cron/manager/__init__.py index 1ec9b72..1f3a855 100644 --- a/apollo/cron/manager/__init__.py +++ b/apollo/cron/manager/__init__.py @@ -91,9 +91,12 @@ class TimedTaskManager(): return timed_task_parameters['id'] = task_id - if "base_url" in timed_task_parameters and "sa_years" in timed_task_parameters: - timed_task_parameters.pop("base_url") - timed_task_parameters.pop("sa_years") + if "day_of_week" not in timed_task_parameters or "hour" not in timed_task_parameters: + LOGGER.error("Create scheduled task is missing required fields.") + return + + if "cvrf_url" in timed_task_parameters: + timed_task_parameters.pop("cvrf_url") if "service_timeout_threshold_min" in timed_task_parameters: timed_task_parameters.pop("service_timeout_threshold_min") @@ -102,7 +105,7 @@ class TimedTaskManager(): if "auto_start" in timed_task_parameters: auto_start = timed_task_parameters['auto_start'] timed_task_parameters.pop("auto_start") - if not auto_start: + if auto_start == "false": LOGGER.info(f"{task_id}, This task is configured to not start.") return TimedTaskManager._APscheduler.add_job(**timed_task_parameters) diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py index 4f1459e..fc2f62a 100644 --- a/apollo/cron/timed_correct_manager.py +++ b/apollo/cron/timed_correct_manager.py @@ -33,7 +33,7 @@ class TimedCorrectTask(TimedTaskBase): """ config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH) SERVICE_TIMEOUT_THRESHOLD_MIN = config_info.get( - "correct_data").get("service_timeout_threshold_min") + "correct_data").get("service_timeout_threshold_min", 15) @staticmethod def task_enter(): diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py index 1a77211..75ab71c 100644 --- a/apollo/database/proxy/cve.py +++ b/apollo/database/proxy/cve.py @@ -951,7 +951,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy): """ Save the record of download sa Args: - sa_record_rows(list): aach element is a record of the AdvisoryDownloadRecord table,e.g. + sa_record_rows(list): each element is a record of the AdvisoryDownloadRecord table,e.g. [{"advisory_year": 2022, "advisory_serial_number": 1230, "download_status": 1}] diff --git a/apollo/handler/task_handler/manager/scan_manager.py b/apollo/handler/task_handler/manager/scan_manager.py index efd6356..48ee71b 100644 --- a/apollo/handler/task_handler/manager/scan_manager.py +++ b/apollo/handler/task_handler/manager/scan_manager.py @@ -195,16 +195,19 @@ class ScanManager(Manager): tmp = {} chart_data = [] file_content = "序号,CVE_ID,主机IP,主机名称,CVSS评分,评分级别\n" - for num, row in enumerate(rows, 1): + excel_row_num = 1 + for row in rows: if row.host_ip in tmp: tmp[row.host_ip]["count"] += 1 - file_content += f"{num},{row.cve_id},{row.host_ip}," \ + file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \ f"{row.host_name},{row.cvss_score},{row.severity}\n" + excel_row_num += 1 else: if row.cve_id is not None: tmp[row.host_ip] = {"count": 1, "host_name": row.host_name} - file_content += f"{num},{row.cve_id},{row.host_ip}," \ + file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \ f"{row.host_name},{row.cvss_score},{row.severity}\n" + excel_row_num += 1 else: tmp[row.host_ip] = {"count": 0, "host_name": row.host_name} diff --git a/conf/apollo_crontab.ini b/conf/apollo_crontab.ini index 3ac8eeb..7ab1bfa 100644 --- a/conf/apollo_crontab.ini +++ b/conf/apollo_crontab.ini @@ -13,8 +13,7 @@ id = download sa day_of_week = 0-6 hour = 3 auto_start = true -base_url = https://repo.openeuler.org/security/data/cvrf -sa_years = 2021,2022,2023 +cvrf_url = https://repo.openeuler.org/security/data/cvrf [correct_data] id = correct data -- Gitee