!145 [sync] PR-143: 修复定时矫正任务,适配新任务类型;修复CVE关联主机信息查询错误;调整host ip字段校验方法
From: @openeuler-sync-bot Reviewed-by: @Lostwayzxc Signed-off-by: @Lostwayzxc
This commit is contained in:
openeuler-ci-bot
Gitee
commit
b87d5c2783
27
0002-fix-the-query-error-of-cve-associated-host.patch
Normal file
27
0002-fix-the-query-error-of-cve-associated-host.patch
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
From f8cab408f39bb8f6b793021cdfbf93338fb6ed0a Mon Sep 17 00:00:00 2001
|
||||||
|
From: gongzt <gong_zhengtang@163.com>
|
||||||
|
Date: Wed, 20 Dec 2023 15:13:07 +0800
|
||||||
|
Subject: [PATCH] Fix the query error of cve associated host
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Type: text/plain; charset=UTF-8
|
||||||
|
Content-Transfer-Encoding: 8bit
|
||||||
|
|
||||||
|
---
|
||||||
|
apollo/database/proxy/cve.py | 1 +
|
||||||
|
1 file changed, 1 insertion(+)
|
||||||
|
|
||||||
|
diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py
|
||||||
|
index 6210156..b5a61b9 100644
|
||||||
|
--- a/apollo/database/proxy/cve.py
|
||||||
|
+++ b/apollo/database/proxy/cve.py
|
||||||
|
@@ -1532,6 +1532,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy):
|
||||||
|
def _get_processed_cve_packages_host(self, data):
|
||||||
|
result = {"total_count": 0, "total_page": 0, "result": []}
|
||||||
|
filters = {
|
||||||
|
+ CveHostAssociation.host_user == data["username"],
|
||||||
|
CveHostAssociation.cve_id == data["cve_id"],
|
||||||
|
CveHostAssociation.installed_rpm == data["installed_rpm"],
|
||||||
|
CveHostAssociation.fixed == data["fixed"],
|
||||||
|
--
|
||||||
|
2.33.0
|
||||||
|
|
||||||
85
0003-update-verification-method-for-host-ip-fieldl.patch
Normal file
85
0003-update-verification-method-for-host-ip-fieldl.patch
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
From 47a4c1d6488f07aa55621454fefedb559fc1bbf8 Mon Sep 17 00:00:00 2001
|
||||||
|
From: rabbitali <wenxin32@foxmail.com>
|
||||||
|
Date: Wed, 20 Dec 2023 16:26:24 +0800
|
||||||
|
Subject: [PATCH] update verification method for host ip fieldl;fix repo
|
||||||
|
field filter error
|
||||||
|
|
||||||
|
---
|
||||||
|
apollo/database/proxy/cve.py | 8 ++++++--
|
||||||
|
apollo/function/schema/cve.py | 2 +-
|
||||||
|
apollo/function/schema/task.py | 6 +++---
|
||||||
|
3 files changed, 10 insertions(+), 6 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py
|
||||||
|
index 6210156..5d29544 100644
|
||||||
|
--- a/apollo/database/proxy/cve.py
|
||||||
|
+++ b/apollo/database/proxy/cve.py
|
||||||
|
@@ -20,7 +20,7 @@ import copy
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
from elasticsearch import ElasticsearchException
|
||||||
|
-from sqlalchemy import func, tuple_, case
|
||||||
|
+from sqlalchemy import func, tuple_, case, or_
|
||||||
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
|
from vulcanus.database.helper import sort_and_page, judge_return_code
|
||||||
|
from vulcanus.database.proxy import MysqlProxy, ElasticsearchProxy
|
||||||
|
@@ -200,7 +200,11 @@ class CveMysqlProxy(MysqlProxy):
|
||||||
|
if filter_dict.get("host_group"):
|
||||||
|
filters.add(Host.host_group_name.in_(filter_dict["host_group"]))
|
||||||
|
if filter_dict.get("repo"):
|
||||||
|
- filters.add(Host.repo_name.in_(filter_dict["repo"]))
|
||||||
|
+ if all(filter_dict.get("repo")):
|
||||||
|
+ filters.add(Host.repo_name.in_(filter_dict["repo"]))
|
||||||
|
+ else:
|
||||||
|
+ repo_names = list(filter(None, filter_dict["repo"]))
|
||||||
|
+ filters.add(or_(Host.repo_name.in_(repo_names), Host.repo_name == None))
|
||||||
|
return filters
|
||||||
|
|
||||||
|
def _query_cve_hosts(self, username: str, cve_id: str, filters: set):
|
||||||
|
diff --git a/apollo/function/schema/cve.py b/apollo/function/schema/cve.py
|
||||||
|
index 178672e..56d76ed 100644
|
||||||
|
--- a/apollo/function/schema/cve.py
|
||||||
|
+++ b/apollo/function/schema/cve.py
|
||||||
|
@@ -56,7 +56,7 @@ class CveHostFilterSchema(Schema):
|
||||||
|
|
||||||
|
host_name = fields.String(required=False, validate=lambda s: len(s) != 0)
|
||||||
|
host_group = fields.List(fields.String(validate=lambda s: len(s) != 0), required=False)
|
||||||
|
- repo = fields.List(fields.String(validate=lambda s: len(s) != 0), required=False)
|
||||||
|
+ repo = fields.List(fields.String(validate=lambda s: len(s) != 0, allow_none=True), required=False)
|
||||||
|
fixed = fields.Boolean(required=True, validate=validate.OneOf([True, False]))
|
||||||
|
|
||||||
|
|
||||||
|
diff --git a/apollo/function/schema/task.py b/apollo/function/schema/task.py
|
||||||
|
index de86194..e136182 100644
|
||||||
|
--- a/apollo/function/schema/task.py
|
||||||
|
+++ b/apollo/function/schema/task.py
|
||||||
|
@@ -18,7 +18,7 @@ Description: For task related restful interfaces schema
|
||||||
|
from marshmallow import Schema
|
||||||
|
from marshmallow import fields
|
||||||
|
from marshmallow import validate
|
||||||
|
-from vulcanus.restful.serialize.validate import PaginationSchema
|
||||||
|
+from vulcanus.restful.serialize.validate import PaginationSchema, ValidateRules
|
||||||
|
|
||||||
|
from apollo.conf.constant import TaskType, TaskStatus
|
||||||
|
|
||||||
|
@@ -248,7 +248,7 @@ class CveFixResultCallbackSchema(Schema):
|
||||||
|
class CallbackSchma(Schema):
|
||||||
|
task_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 32)
|
||||||
|
host_id = fields.Integer(required=True, validate=lambda s: s > 0)
|
||||||
|
- host_ip = fields.IP(required=True)
|
||||||
|
+ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check)
|
||||||
|
host_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50)
|
||||||
|
status = fields.String(required=True, validate=lambda s: len(s) != 0)
|
||||||
|
execution_time = fields.Integer(required=True)
|
||||||
|
@@ -270,7 +270,7 @@ class CheckItemsSchema(Schema):
|
||||||
|
class RepoSetCallbackSchema(Schema):
|
||||||
|
task_id = fields.String(required=True, validate=lambda s: 0 < len(s) <= 32)
|
||||||
|
host_id = fields.Integer(required=True, validate=lambda s: s > 0)
|
||||||
|
- host_ip = fields.IP(required=True)
|
||||||
|
+ host_ip = fields.String(required=True, validate=ValidateRules.ipv4_address_check)
|
||||||
|
host_name = fields.String(required=True, validate=lambda s: 0 < len(s) <= 50)
|
||||||
|
status = fields.String(required=True, validate=lambda s: len(s) != 0)
|
||||||
|
execution_time = fields.Integer(required=True)
|
||||||
|
--
|
||||||
|
2.33.0
|
||||||
|
|
||||||
168
0004-fix-TimedCorrectTask.patch
Normal file
168
0004-fix-TimedCorrectTask.patch
Normal file
@ -0,0 +1,168 @@
|
|||||||
|
From 8e19d92b6a484ddcf7ca7bf666ce21baa56ab326 Mon Sep 17 00:00:00 2001
|
||||||
|
From: rearcher <123781007@qq.com>
|
||||||
|
Date: Wed, 20 Dec 2023 17:21:22 +0800
|
||||||
|
Subject: [PATCH] fix TimedCorrectTask
|
||||||
|
|
||||||
|
---
|
||||||
|
apollo/cron/timed_correct_manager.py | 12 ++++--
|
||||||
|
apollo/database/proxy/task/base.py | 50 ++++++++++++++++++-----
|
||||||
|
apollo/database/proxy/task/timed_proxy.py | 8 ++++
|
||||||
|
apollo/tests/database/test_task.py | 2 +-
|
||||||
|
4 files changed, 57 insertions(+), 15 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py
|
||||||
|
index db0b4c1..ae3a1e3 100644
|
||||||
|
--- a/apollo/cron/timed_correct_manager.py
|
||||||
|
+++ b/apollo/cron/timed_correct_manager.py
|
||||||
|
@@ -41,11 +41,15 @@ class TimedCorrectTask(TimedTask):
|
||||||
|
"""
|
||||||
|
Start the correct after the specified time of day.
|
||||||
|
"""
|
||||||
|
- LOGGER.info("Begin to correct the whole host in %s.", str(datetime.datetime.now()))
|
||||||
|
+ LOGGER.info(
|
||||||
|
+ "Begin to correct the status of timeout tasks and scan timeout host in %s.",
|
||||||
|
+ str(datetime.datetime.now()))
|
||||||
|
abnormal_task_ids, abnormal_host_ids = self.get_abnormal_task()
|
||||||
|
- self._update_host_status(abnormal_host_ids)
|
||||||
|
- with TimedProxy() as proxy:
|
||||||
|
- proxy.timed_correct_error_task_status(abnormal_task_ids)
|
||||||
|
+ if len(abnormal_host_ids) != 0:
|
||||||
|
+ self._update_host_status(abnormal_host_ids)
|
||||||
|
+ if len(abnormal_task_ids) != 0:
|
||||||
|
+ with TimedProxy() as proxy:
|
||||||
|
+ proxy.timed_correct_error_task_status(abnormal_task_ids)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _abnormal_task(tasks):
|
||||||
|
diff --git a/apollo/database/proxy/task/base.py b/apollo/database/proxy/task/base.py
|
||||||
|
index 840c140..a5ddede 100644
|
||||||
|
--- a/apollo/database/proxy/task/base.py
|
||||||
|
+++ b/apollo/database/proxy/task/base.py
|
||||||
|
@@ -861,17 +861,17 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy):
|
||||||
|
|
||||||
|
raise EsOperationError("Delete task from elasticsearch failed due to internal error.")
|
||||||
|
|
||||||
|
- def get_running_task_form_task_cve_host(self) -> list:
|
||||||
|
+ def get_running_task_form_hotpatch_remove_task(self) -> list:
|
||||||
|
"""
|
||||||
|
- Get all CVE repair tasks with running status under Username
|
||||||
|
+ Get all hotpatch remove tasks with running status under Username
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list: task id list
|
||||||
|
"""
|
||||||
|
- task_cve_query = (
|
||||||
|
- self.session.query(HotpatchRemoveTask).filter(HotpatchRemoveTask.status == TaskStatus.RUNNING).all()
|
||||||
|
+ hotpatch_remove_query = (
|
||||||
|
+ self.session.query(HotpatchRemoveTask.task_id).filter(HotpatchRemoveTask.status == TaskStatus.RUNNING).all()
|
||||||
|
)
|
||||||
|
- task_id_list = [task.task_id for task in task_cve_query]
|
||||||
|
+ task_id_list = [task.task_id for task in hotpatch_remove_query]
|
||||||
|
return task_id_list
|
||||||
|
|
||||||
|
def get_running_task_form_task_host_repo(self) -> list:
|
||||||
|
@@ -882,13 +882,39 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy):
|
||||||
|
list: task id list
|
||||||
|
"""
|
||||||
|
host_repo_query = (
|
||||||
|
- self.session.query(TaskHostRepoAssociation)
|
||||||
|
+ self.session.query(TaskHostRepoAssociation.task_id)
|
||||||
|
.filter(TaskHostRepoAssociation.status == TaskStatus.RUNNING)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
task_id_list = [task.task_id for task in host_repo_query]
|
||||||
|
return task_id_list
|
||||||
|
|
||||||
|
+ def get_running_task_form_cve_fix_task(self) -> list:
|
||||||
|
+ """
|
||||||
|
+ Get all CVE fix tasks with running status
|
||||||
|
+
|
||||||
|
+ Returns:
|
||||||
|
+ list: task id list
|
||||||
|
+ """
|
||||||
|
+ cve_fix_query = (
|
||||||
|
+ self.session.query(CveFixTask.task_id).filter(CveFixTask.status == TaskStatus.RUNNING).all()
|
||||||
|
+ )
|
||||||
|
+ task_id_list = [task.task_id for task in cve_fix_query]
|
||||||
|
+ return task_id_list
|
||||||
|
+
|
||||||
|
+ def get_running_task_form_cve_rollback_task(self) -> list:
|
||||||
|
+ """
|
||||||
|
+ Get all CVE rollback tasks with running status
|
||||||
|
+
|
||||||
|
+ Returns:
|
||||||
|
+ list: task id list
|
||||||
|
+ """
|
||||||
|
+ cve_rollback_query = (
|
||||||
|
+ self.session.query(CveRollbackTask.task_id).filter(CveRollbackTask.status == TaskStatus.RUNNING).all()
|
||||||
|
+ )
|
||||||
|
+ task_id_list = [task.task_id for task in cve_rollback_query]
|
||||||
|
+ return task_id_list
|
||||||
|
+
|
||||||
|
def get_scanning_status_and_time_from_host(self) -> list:
|
||||||
|
"""
|
||||||
|
Get all host id and time with scanning status from the host table
|
||||||
|
@@ -907,13 +933,17 @@ class TaskProxy(TaskMysqlProxy, TaskEsProxy):
|
||||||
|
Returns:
|
||||||
|
list: Each element is a task information, including the task ID, task type, creation time
|
||||||
|
"""
|
||||||
|
- task_cve_id_list = self.get_running_task_form_task_cve_host()
|
||||||
|
- task_repo_id_list = self.get_running_task_form_task_host_repo()
|
||||||
|
host_info_list = self.get_scanning_status_and_time_from_host()
|
||||||
|
- task_id_list = task_cve_id_list + task_repo_id_list
|
||||||
|
+
|
||||||
|
+ task_cve_id_list = self.get_running_task_form_hotpatch_remove_task()
|
||||||
|
+ task_repo_id_list = self.get_running_task_form_task_host_repo()
|
||||||
|
+ task_cve_fix_list = self.get_running_task_form_cve_fix_task()
|
||||||
|
+ task_cve_rollback_list = self.get_running_task_form_cve_rollback_task()
|
||||||
|
+
|
||||||
|
+ task_id_list = task_cve_id_list + task_repo_id_list + task_cve_fix_list + task_cve_rollback_list
|
||||||
|
|
||||||
|
task_query = self.session.query(Task).filter(Task.task_id.in_(task_id_list)).all()
|
||||||
|
- running_task_list = [(task.task_id, task.create_time) for task in task_query]
|
||||||
|
+ running_task_list = [(task.task_id, task.latest_execute_time) for task in task_query]
|
||||||
|
return running_task_list, host_info_list
|
||||||
|
|
||||||
|
def validate_cves(self, cve_id: list) -> bool:
|
||||||
|
diff --git a/apollo/database/proxy/task/timed_proxy.py b/apollo/database/proxy/task/timed_proxy.py
|
||||||
|
index 436c3bd..fd396d1 100644
|
||||||
|
--- a/apollo/database/proxy/task/timed_proxy.py
|
||||||
|
+++ b/apollo/database/proxy/task/timed_proxy.py
|
||||||
|
@@ -22,6 +22,8 @@ from apollo.conf.constant import TaskStatus
|
||||||
|
from apollo.database.table import (
|
||||||
|
HotpatchRemoveTask,
|
||||||
|
TaskHostRepoAssociation,
|
||||||
|
+ CveFixTask,
|
||||||
|
+ CveRollbackTask,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@@ -42,6 +44,12 @@ class TimedProxy(MysqlProxy):
|
||||||
|
self.session.query(TaskHostRepoAssociation).filter(TaskHostRepoAssociation.task_id.in_(task_ids)).update(
|
||||||
|
{TaskHostRepoAssociation.status: TaskStatus.UNKNOWN}, synchronize_session=False
|
||||||
|
)
|
||||||
|
+ self.session.query(CveFixTask).filter(CveFixTask.task_id.in_(task_ids)).update(
|
||||||
|
+ {CveFixTask.status: TaskStatus.UNKNOWN}, synchronize_session=False
|
||||||
|
+ )
|
||||||
|
+ self.session.query(CveRollbackTask).filter(CveRollbackTask.task_id.in_(task_ids)).update(
|
||||||
|
+ {CveRollbackTask.status: TaskStatus.UNKNOWN}, synchronize_session=False
|
||||||
|
+ )
|
||||||
|
self.session.commit()
|
||||||
|
except SQLAlchemyError as error:
|
||||||
|
self.session.rollback()
|
||||||
|
diff --git a/apollo/tests/database/test_task.py b/apollo/tests/database/test_task.py
|
||||||
|
index 35d923b..ceb84ab 100644
|
||||||
|
--- a/apollo/tests/database/test_task.py
|
||||||
|
+++ b/apollo/tests/database/test_task.py
|
||||||
|
@@ -401,7 +401,7 @@ class TestTaskMysqlFirst(DatabaseTestCase):
|
||||||
|
|
||||||
|
def test_get_running_task_form_task_cve_host(self):
|
||||||
|
self.assertEqual(
|
||||||
|
- self.task_database.get_running_task_form_task_cve_host(),
|
||||||
|
+ self.task_database.get_running_task_form_hotpatch_remove_task(),
|
||||||
|
["1111111111poiuytrewqasdfghjklmnb"],
|
||||||
|
)
|
||||||
|
|
||||||
|
--
|
||||||
|
2.33.0
|
||||||
|
|
||||||
45
0005-add-reboot-field-to-query-host-info-api.patch
Normal file
45
0005-add-reboot-field-to-query-host-info-api.patch
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
From 5c803953928ad16d56795dcf7158bb9f3d340e1d Mon Sep 17 00:00:00 2001
|
||||||
|
From: gongzt <gong_zhengtang@163.com>
|
||||||
|
Date: Thu, 21 Dec 2023 17:33:23 +0800
|
||||||
|
Subject: [PATCH] add reboot field to query host info api
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Type: text/plain; charset=UTF-8
|
||||||
|
Content-Transfer-Encoding: 8bit
|
||||||
|
|
||||||
|
---
|
||||||
|
apollo/database/proxy/host.py | 5 ++++-
|
||||||
|
1 file changed, 4 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
|
diff --git a/apollo/database/proxy/host.py b/apollo/database/proxy/host.py
|
||||||
|
index ca709a8..6015a04 100644
|
||||||
|
--- a/apollo/database/proxy/host.py
|
||||||
|
+++ b/apollo/database/proxy/host.py
|
||||||
|
@@ -341,7 +341,8 @@ class HostMysqlProxy(MysqlProxy):
|
||||||
|
"repo": "20.03-update",
|
||||||
|
"affected_cve_num": 12,
|
||||||
|
"unaffected_cve_num": 1,
|
||||||
|
- "last_scan": 1111111111
|
||||||
|
+ "last_scan": 1111111111,
|
||||||
|
+ "reboot": true/false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
@@ -412,6 +413,7 @@ class HostMysqlProxy(MysqlProxy):
|
||||||
|
Host.host_group_name,
|
||||||
|
Host.repo_name,
|
||||||
|
Host.last_scan,
|
||||||
|
+ Host.reboot,
|
||||||
|
func.COUNT(func.IF(subquery.c.fixed == True, 1, None)).label("fixed_cve_num"),
|
||||||
|
func.COUNT(func.IF(and_(subquery.c.fixed == False, subquery.c.affected == True), 1, None)).label(
|
||||||
|
"affected_cve_num"
|
||||||
|
@@ -437,6 +439,7 @@ class HostMysqlProxy(MysqlProxy):
|
||||||
|
"unaffected_cve_num": row.unaffected_cve_num,
|
||||||
|
"last_scan": row.last_scan,
|
||||||
|
"fixed_cve_num": row.fixed_cve_num,
|
||||||
|
+ "reboot": row.reboot,
|
||||||
|
}
|
||||||
|
return host_info
|
||||||
|
|
||||||
|
--
|
||||||
|
Gitee
|
||||||
|
|
||||||
@ -1,11 +1,15 @@
|
|||||||
Name: aops-apollo
|
Name: aops-apollo
|
||||||
Version: v1.4.1
|
Version: v1.4.1
|
||||||
Release: 2
|
Release: 3
|
||||||
Summary: Cve management service, monitor machine vulnerabilities and provide fix functions.
|
Summary: Cve management service, monitor machine vulnerabilities and provide fix functions.
|
||||||
License: MulanPSL2
|
License: MulanPSL2
|
||||||
URL: https://gitee.com/openeuler/%{name}
|
URL: https://gitee.com/openeuler/%{name}
|
||||||
Source0: %{name}-%{version}.tar.gz
|
Source0: %{name}-%{version}.tar.gz
|
||||||
Patch0001: 0001-fix-cve-list-sort.patch
|
Patch0001: 0001-fix-cve-list-sort.patch
|
||||||
|
Patch0002: 0002-fix-the-query-error-of-cve-associated-host.patch
|
||||||
|
Patch0003: 0003-update-verification-method-for-host-ip-fieldl.patch
|
||||||
|
Patch0004: 0004-fix-TimedCorrectTask.patch
|
||||||
|
Patch0005: 0005-add-reboot-field-to-query-host-info-api.patch
|
||||||
|
|
||||||
BuildRequires: python3-setuptools
|
BuildRequires: python3-setuptools
|
||||||
Requires: aops-vulcanus >= v1.3.0
|
Requires: aops-vulcanus >= v1.3.0
|
||||||
@ -68,6 +72,11 @@ popd
|
|||||||
%{python3_sitelib}/aops_apollo_tool/*
|
%{python3_sitelib}/aops_apollo_tool/*
|
||||||
|
|
||||||
%changelog
|
%changelog
|
||||||
|
* Fri Dec 22 2023 wenxin<wenxin32@foxmail.com> - v1.4.1-3
|
||||||
|
- fix the query error of cve associated host
|
||||||
|
- update verification method for host ip fieldl;fix repo field filter error
|
||||||
|
- update TimedCorrectTask method
|
||||||
|
|
||||||
* Mon Dec 18 2023 luxuexian<luxuexian@huawei.com> - v1.4.1-2
|
* Mon Dec 18 2023 luxuexian<luxuexian@huawei.com> - v1.4.1-2
|
||||||
- fix cve_list sort order
|
- fix cve_list sort order
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user