packages/A-Tune-Collector
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feature: enable application configs

Browse Source 
(cherry picked from commit c9558b4276c94f7fc75fccf1d167f14b53ee1342)
This commit is contained in:
gaoruoshu@huawei.com 2023-08-01 09:43:39 +08:00 committed by openeuler-sync-bot
parent 2e3765a33c
commit 0bb8fedb32
12 changed files with 1624 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
README-add-data-collecting-output-description.patch Normal file
View File

@ -0,0 +1,47 @@
From c8d0e4c3030808fe0a575e595d8bd9a51d30124a Mon Sep 17 00:00:00 2001
From: HuBin95 <hubin73@huawei.com>
Date: Mon, 7 Mar 2022 02:10:24 +0000
Subject: [PATCH 02/11] README: add data collecting output description
---
README.en.md | 4 +++-
README.md | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/README.en.md b/README.en.md
index 2f11270..8a54fa4 100644
--- a/README.en.md
+++ b/README.en.md
@@ -52,9 +52,11 @@ Table 1 **collect_data.json** file
| sample_num | Sample number to be collected. | Integer | > 0 |
| interval | Interval for collecting data, in seconds. | Integer | > 0 |
| output_dir | Path for storing collected data. | Character string | - |
-| workload_type | Application load type of the collection environment. The default value is **default**. | Character string | - |
+| workload_type | Application load type of the collection environment, used as output file name. The default value is **default**. | Character string | - |
| collection_items | Table 2 lists the system parameters to be collected. | List | - |
+When data collecting is finished, the data will be saved as: `${output_dir}/${workload_type}-${finish_timestamp}.csv`
+
Table 2 Description of the **collection_items** configuration
| Parameter | Description | Type | Value Range |
diff --git a/README.md b/README.md
index 36d5954..c121c8d 100644
--- a/README.md
+++ b/README.md
@@ -53,9 +53,11 @@ python3 collect_data.py [OPTIONS]
| sample_num | 待采集的次数 | 整型 | >0 |
| interval | 待采集的间隔时间，单位为秒 | 整型 | >0 |
| output_dir | 采集完后数据存储的文件路径 | 字符串 | - |
-| workload_type | 采集环境的应用负载类型，默认为default | 字符串 | - |
+| workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - |
| collection_items | 需要采集的系统参数项，参见表2 | 列表 | - |
+最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv`
+
表2 collection_items项配置说明
| **配置名称** | **配置说明** | **参数类型** | **取值范围** |
--
2.27.0

39
Ubuntu-20.04.3-LTS.patch Normal file
View File

@ -0,0 +1,39 @@
From 2fd1148dcec21a8a99fbb59240ba2e2abfe7c31c Mon Sep 17 00:00:00 2001
From: Masterwater-y <yin12162000@outlook.com>
Date: Fri, 28 Jan 2022 18:07:52 +0800
Subject: [PATCH 01/11] adapt Ubuntu 20.04.3 LTS
---
atune_collector/plugin/monitor/common.py | 2 +-
atune_collector/plugin/monitor/memory/topo.py | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py
index a646b32..12a07f2 100755
--- a/atune_collector/plugin/monitor/common.py
+++ b/atune_collector/plugin/monitor/common.py
@@ -230,7 +230,7 @@ class Monitor(object):
def walk_class_type(father, class_type, desc, datas):
"""get key field"""
if "class" in father and father["class"] == class_type:
- if "description" in father and (desc is None or father["description"] == desc):
+ if "description" in father and (desc is None or father["description"].lower() == desc.lower()):
datas.append(father)
return
if "children" in father:
diff --git a/atune_collector/plugin/monitor/memory/topo.py b/atune_collector/plugin/monitor/memory/topo.py
index 7e828f3..2c0c39e 100755
--- a/atune_collector/plugin/monitor/memory/topo.py
+++ b/atune_collector/plugin/monitor/memory/topo.py
@@ -95,6 +95,8 @@ class MemTopo(Monitor):
cmd=self.__cmd).split(), stderr=no_print)
info = o_json.decode()
json_content = json.loads(info)
+ if (isinstance(json_content, list)):
+ json_content = {'children': json_content}
dict_datas = get_class_type(json_content, "memory", "System Memory")
if fmt == "json":
--
2.27.0

427
add-new-dims.patch Normal file
View File

@ -0,0 +1,427 @@
From 2d6269e360a654f11370f31ea4faf6ee4012d73f Mon Sep 17 00:00:00 2001
From: hamster <hujing@isrc.iscas.ac.cn>
Date: Mon, 14 Nov 2022 03:24:49 +0000
Subject: [PATCH 04/11] add new dimensions
From: @hujing2
Reviewed-by: @gaoruoshu
Signed-off-by: @gaoruoshu
---
README.md | 16 +++
atune_collector/collect_data.json | 14 ++
atune_collector/collect_data.py | 97 ++++++++------
atune_collector/plugin/monitor/__init__.py | 1 +
atune_collector/plugin/monitor/common.py | 1 +
.../plugin/monitor/process/__init__.py | 20 +++
.../plugin/monitor/process/sched.py | 124 ++++++++++++++++++
7 files changed, 233 insertions(+), 40 deletions(-)
create mode 100644 atune_collector/plugin/monitor/process/__init__.py
create mode 100644 atune_collector/plugin/monitor/process/sched.py
diff --git a/README.md b/README.md
index c121c8d..242f345 100644
--- a/README.md
+++ b/README.md
@@ -50,12 +50,14 @@ python3 collect_data.py [OPTIONS]
| ---------------- | ------------------------------------- | ------------ | ------------ |
| network | 待采集的指定网卡 | 字符串 | - |
| block | 待采集的指定磁盘 | 字符串 | - |
+| application | 需要采集的应用进程 | 字符串 | - |
| sample_num | 待采集的次数 | 整型 | >0 |
| interval | 待采集的间隔时间，单位为秒 | 整型 | >0 |
| output_dir | 采集完后数据存储的文件路径 | 字符串 | - |
| workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - |
| collection_items | 需要采集的系统参数项，参见表2 | 列表 | - |
+
最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv`
表2 collection_items项配置说明
@@ -76,6 +78,7 @@ collect_data.json文件配置示例：
{
"network": "eth0",
"block": "sda",
+ "application": "mysqld",
"sample_num": 20,
"interval": 5,
"output_dir": "/var/atuned/collect_data",
@@ -203,6 +206,19 @@ collect_data.json文件配置示例：
"metrics": [
"fd-util"
]
+ },
+ {
+ "name": "process",
+ "module": "PROCESS",
+ "purpose": "SCHED",
+ "metrics": [
+ "exec_start",
+ "vruntime",
+ "sum_exec_runtime",
+ "switches",
+ "voluntary_switches",
+ "involuntary_switches"
+ ]
}
]
}
diff --git a/atune_collector/collect_data.json b/atune_collector/collect_data.json
index db96501..af286bd 100755
--- a/atune_collector/collect_data.json
+++ b/atune_collector/collect_data.json
@@ -1,6 +1,7 @@
{
"network": "eth0",
"block": "sda",
+ "application": "firewalld,dockerd",
"sample_num": 20,
"interval": 5,
"output_dir": "/var/atuned/collect_data",
@@ -140,6 +141,19 @@
"metrics": [
"fd-util"
]
+ },
+ {
+ "name": "process",
+ "module": "PROCESS",
+ "purpose": "SCHED",
+ "metrics": [
+ "exec_start",
+ "vruntime",
+ "sum_exec_runtime",
+ "switches",
+ "voluntary_switches",
+ "involuntary_switches"
+ ]
}
]
}
\ No newline at end of file
diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py
index 1764304..3593db6 100755
--- a/atune_collector/collect_data.py
+++ b/atune_collector/collect_data.py
@@ -18,6 +18,7 @@ import argparse
import json
import os
import time
+import csv
from plugin.plugin import MPI
@@ -30,66 +31,80 @@ class Collector:
self.field_name = []
self.support_multi_block = ['storage']
self.support_multi_nic = ['network', 'network-err']
+ self.support_multi_app = ['process']
def parse_json(self):
"""parse json data"""
monitors = []
for item in self.data["collection_items"]:
- parameters = ["--interval=%s;" % self.data["interval"]]
- for metric in item["metrics"]:
- nics = self.data["network"].split(',')
- blocks = self.data["block"].split(',')
- if item["name"] in self.support_multi_nic and len(nics) > 1:
- for net in nics:
+ if item["name"] in self.support_multi_app and ('application' not in self.data or
+ self.data["application"] == ""):
+ continue
+ if item["name"] in self.support_multi_app:
+ applications = self.data["application"].split(',')
+ parameters = ["--interval=%s --app=%s;" %(self.data["interval"], self.data["application"])]
+ for application in applications:
+ for metric in item["metrics"]:
self.field_name.append(
- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net))
- elif item["name"] in self.support_multi_block and len(blocks) > 1:
- for block in blocks:
- self.field_name.append(
- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block))
- else:
- self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric))
- parameters.append("--fields=%s" % metric)
- if "threshold" in item:
- parameters.append("--threshold=%s" % item["threshold"])
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, application))
+ parameters.append("--fields=%s" % metric)
+ else:
+ parameters = ["--interval=%s;" % self.data["interval"]]
+ for metric in item["metrics"]:
+ nics = self.data["network"].split(',')
+ blocks = self.data["block"].split(',')
+
+ if item["name"] in self.support_multi_nic and len(nics) > 1:
+ for net in nics:
+ self.field_name.append(
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net))
+ elif item["name"] in self.support_multi_block and len(blocks) > 1:
+ for block in blocks:
+ self.field_name.append(
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block))
+ else:
+ self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric))
+ parameters.append("--fields=%s" % metric)
+ if "threshold" in item:
+ parameters.append("--threshold=%s" % item["threshold"])
+
parameters.append("--nic=%s" % self.data["network"])
parameters.append("--device=%s" % self.data["block"])
monitors.append([item["module"], item["purpose"], " ".join(parameters)])
return monitors
- def save_csv(self, field_data):
- """save data to csv file"""
+ def collect_data(self):
+ """collect data"""
+ collect_num = self.data["sample_num"]
+ if int(collect_num) < 1:
+ os.abort("sample_num must be greater than 0")
+
+ mpi = MPI()
+ monitors = self.parse_json()
path = self.data["output_dir"]
if not os.path.exists(path):
os.makedirs(path, 0o750)
file_name = "{}-{}.csv".format(self.data.get("workload_type", "default"),
int(round(time.time() * 1000)))
- import csv
+
+ print("start to collect data, csv path is %s" % os.path.join(path, file_name))
+ print(" ".join(self.field_name))
with open(os.path.join(path, file_name), "w") as csvfile:
writer = csv.writer(csvfile)
self.field_name.insert(0, "TimeStamp")
writer.writerow(self.field_name)
- writer.writerows(field_data)
+ csvfile.flush()
+ for _ in range(collect_num):
+ raw_data = mpi.get_monitors_data(monitors)
+ float_data = [float(num) for num in raw_data]
+ str_data = [str(round(value, 3)) for value in float_data]
+ print(" ".join(str_data))
+ float_data.insert(0, time.strftime("%H:%M:%S"))
+ # field_data.append(float_data)
+ writer.writerow(float_data)
+ csvfile.flush()
print("finish to collect data, csv path is %s" % os.path.join(path, file_name))
- def collect_data(self):
- """collect data"""
- collect_num = self.data["sample_num"]
- if int(collect_num) < 1:
- os.abort("sample_num must be greater than 0")
- field_data = []
- mpi = MPI()
- monitors = self.parse_json()
- print(" ".join(self.field_name))
- for _ in range(collect_num):
- raw_data = mpi.get_monitors_data(monitors)
- float_data = [float(num) for num in raw_data]
- str_data = [str(round(value, 3)) for value in float_data]
- print(" ".join(str_data))
- float_data.insert(0, time.strftime("%H:%M:%S"))
- field_data.append(float_data)
- return field_data
-
if __name__ == "__main__":
default_json_path = "/etc/atune_collector/collect_data.json"
@@ -100,5 +115,7 @@ if __name__ == "__main__":
with open(ARGS.config, 'r') as file:
json_data = json.load(file)
collector = Collector(json_data)
- dataset = collector.collect_data()
- collector.save_csv(dataset)
+ try:
+ collector.collect_data()
+ except KeyboardInterrupt:
+ print("user stop collect data")
\ No newline at end of file
diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py
index 6291401..9292071 100755
--- a/atune_collector/plugin/monitor/__init__.py
+++ b/atune_collector/plugin/monitor/__init__.py
@@ -19,6 +19,7 @@ __all__ = [
"memory",
"network",
"performance",
+ "process",
"processor",
"storage",
"common",
diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py
index 12a07f2..d0aa60c 100755
--- a/atune_collector/plugin/monitor/common.py
+++ b/atune_collector/plugin/monitor/common.py
@@ -134,6 +134,7 @@ class Monitor(object):
"--interval=" to specify period of time
"--cpu=" to select which cpu
"--event=" to select which event
+ "--app" to select which applications
:returns value: Success, collected info string
:raises Exceptions: Fail, with info
"""
diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py
new file mode 100644
index 0000000..4c4ceb3
--- /dev/null
+++ b/atune_collector/plugin/monitor/process/__init__.py
@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2022-10-14
+
+"""
+Init file.
+"""
+
+__all__ = ["sched"]
+
+from . import *
\ No newline at end of file
diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py
new file mode 100644
index 0000000..5289d84
--- /dev/null
+++ b/atune_collector/plugin/monitor/process/sched.py
@@ -0,0 +1,124 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2022-10-14
+
+"""
+The sub class of the monitor, used to collect the process sched info
+"""
+import inspect
+import logging
+import subprocess
+import getopt
+import re
+from ..common import Monitor
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ProcSched(Monitor):
+ """To collect the process sched info"""
+ _module = "PROCESS"
+ _purpose = "SCHED"
+ _option = "/proc/{}/sched"
+
+ def __init__(self, user=None):
+ Monitor.__init__(self, user)
+ self.__cmd = "cat"
+ self.__interval = 1
+ self.__applications = []
+ self.__pids = []
+
+ def _get(self, para=None):
+ output = ""
+ pids = []
+ if para is not None:
+ opts, _ = getopt.getopt(para.split(), None, ['interval=', 'app='])
+ for opt, val in opts:
+ if opt in '--interval':
+ if val.isdigit():
+ self.__interval = int(val)
+ else:
+ err = ValueError(
+ "Invalid parameter: {opt}={val}".format(
+ opt=opt, val=val))
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+ continue
+ elif opt in '--app':
+ if val is not None:
+ self.__applications = val.split(',')
+ else:
+ err = ValueError(
+ "{opt} parameter is none".format(
+ opt=opt))
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+
+ for app in self.__applications:
+ pid = subprocess.getoutput(
+ "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()[0]
+ pids.append(pid)
+ self.__pids = pids
+
+ for pid in self.__pids:
+ out = subprocess.check_output(
+ "{cmd} {opt}".format(
+ cmd=self.__cmd,
+ opt=self._option.format(pid)).split())
+ output = output + "" + out.decode()
+ return output
+
+ def decode(self, info, para):
+ """
+ decode the result of the operation
+ :param info: content that needs to be decoded
+ :param para: command line argument
+ :returns ret: operation result
+ """
+
+ if para is None:
+ return info
+
+ start = 0
+ keys = []
+ ret = ""
+
+ opts, _ = getopt.getopt(para.split(), None, ['nic=', 'fields=', 'device='])
+ for opt, val in opts:
+ if opt in '--fields':
+ keys.append(val)
+ continue
+
+ pattern = re.compile(
+ r"(\w+)\ {1,}\:\ {1,}(\d+.\d+)",
+ re.I | re.UNICODE | re.MULTILINE)
+ search_obj = pattern.findall(info)
+ search_list = []
+ for obj in search_obj:
+ if obj[0][:3] == "nr_":
+ search_list.append(obj[0][3:])
+ else:
+ search_list.append(obj[0])
+ search_list.append(obj[1])
+ if len(search_obj) == 0:
+ err = LookupError("Fail to find data")
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
+ inspect.stack()[0][3], str(err))
+ raise err
+
+ while start <= len(self.__applications) * len(keys):
+ for key in keys:
+ ret = ret + " " + search_list[search_list.index(key, start)+1]
+ start = search_list.index(key, start) + 1
+ return ret
\ No newline at end of file
--
2.27.0

17
atune-collector.spec
View File

@ -2,13 +2,23 @@
Name: atune-collector
Version: 1.1.0
Release: 2
Release: 3
Summary: A-Tune-Collector is used to collect various system resources.
License: Mulan PSL v2
URL: https://gitee.com/openeuler/A-Tune-Collector
Source: https://gitee.com/openeuler/A-Tune-Collector/repository/archive/v%{version}.tar.gz
Patch0: 0001-fix-bug-list-index-out-of-range-in-virtualbox.patch
Patch1: Ubuntu-20.04.3-LTS.patch
Patch2: README-add-data-collecting-output-description.patch
Patch3: fix-bug-list-index-out-of-range-in-virtualbox.patch
Patch4: add-new-dims.patch
Patch5: fix-CWE-23.patch
Patch6: memory-bandwidth-dimm-slot-atuned.patch
Patch7: feature-set-mysql.patch
Patch8: feature-set-redis-value.patch
Patch9: feature-set-nginx.patch
Patch10: bugfix-create-file-only-if-setting-params.patch
Patch11: change-import-behavior.patch
BuildRequires: python3-setuptools
Requires: python3-dict2xml
@ -33,6 +43,9 @@ The A-Tune-Collector is used to collect various system resources and can also be
%attr(0600,root,root) %{_sysconfdir}/atune_collector/*
%changelog
* Tue Aug 01 2023 gaoruoshu <gaoruoshu@huawei.com> - 1.1.0-3
- feature: enable application config
* Thu Mar 09 2023 zhangjian <zhang_jian7@hoperun.com> - 1.1.0-2
- fix bug: list index out of range in virtualbox

113
bugfix-create-file-only-if-setting-params.patch Normal file
View File

@ -0,0 +1,113 @@
From 1c513970f8f5e81268e998cc33ab8a93802bb722 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Fri, 14 Jul 2023 18:47:25 +0800
Subject: [PATCH 10/11] bugfix: create file only if setting params
---
atune_collector/plugin/configurator/mysql/mysql.py | 9 ++++++---
atune_collector/plugin/configurator/nginx/nginx.py | 7 ++++---
atune_collector/plugin/configurator/redis/redis.py | 7 +++++--
3 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py
index cfc533d..e472a1c 100644
--- a/atune_collector/plugin/configurator/mysql/mysql.py
+++ b/atune_collector/plugin/configurator/mysql/mysql.py
@@ -31,8 +31,10 @@ class Mysql(Configurator):
self.__cmd = ""
self.__file_path = "/etc/my.cnf"
self.__mysqld_ind = -1
+
+ def _init_file(self):
if not os.path.isfile(self.__file_path):
- with open(self.__file_path, 'w'):
+ with open(self.__file_path, 'w', 0o644):
pass
os.chmod(self.__file_path, 0o644)
self.__check_mysqld()
@@ -61,7 +63,8 @@ class Mysql(Configurator):
return False, ind
def _set(self, key, value):
- with open(self.__file_path, 'r') as f:
+ self._init_file()
+ with open(self.__file_path, 'r', 0o400) as f:
lines = f.readlines()
key_exist, ind = self.__check_file_exists(lines, key)
@@ -71,7 +74,7 @@ class Mysql(Configurator):
else:
lines[ind] = new_line
- with open(self.__file_path, 'w') as f:
+ with open(self.__file_path, 'w', 0o644) as f:
f.writelines(lines)
return 0
diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py
index 7ab619d..9d38dd0 100644
--- a/atune_collector/plugin/configurator/nginx/nginx.py
+++ b/atune_collector/plugin/configurator/nginx/nginx.py
@@ -42,18 +42,18 @@ class Nginx(Configurator):
os.mkdir(self.__file_dir)
os.chmod(self.__file_dir, 0o755)
if not os.path.isfile(self.__file_path):
- with open(self.__file_path, 'w'):
+ with open(self.__file_path, 'w', 0o644):
pass
os.chmod(self.__file_path, 0o644)
self._set_index()
def _get_lines(self):
- with open(self.__file_path, 'r') as f:
+ with open(self.__file_path, 'r', 0o444) as f:
lines = f.readlines()
return lines
def _set_lines(self, lines):
- with open(self.__file_path, 'w') as f:
+ with open(self.__file_path, 'w', 0o644) as f:
f.writelines(lines)
def _set_index(self):
@@ -209,3 +209,4 @@ def rewrite_value(value):
raise SetConfigError("Failed to get cpu number")
return output.stdout.decode().count("\n")
+
diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py
index c56f38e..aa14bcd 100644
--- a/atune_collector/plugin/configurator/redis/redis.py
+++ b/atune_collector/plugin/configurator/redis/redis.py
@@ -34,15 +34,18 @@ class Redis(Configurator):
self.__re = r"^#\?\s*{key}\s* "
self.__file_dir = "/etc/redis/"
self.__file_path = self.__file_dir + "redis.conf"
+
+ def _init_file(self):
if not os.path.isdir(self.__file_dir):
os.mkdir(self.__file_dir)
os.chmod(self.__file_dir, 0o755)
if not os.path.isfile(self.__file_path):
- with open(self.__file_path, 'w', 0o200):
+ with open(self.__file_path, 'w', 0o600):
pass
os.chmod(self.__file_path, 0o644)
def _set(self, key, value):
+ self._init_file()
re_cmd = self.__re.format(key=key)
grep_cmd = [r"grep", re_cmd, self.__file_path]
out, err = self.execute_cmd(grep_cmd)
@@ -51,7 +54,7 @@ class Redis(Configurator):
num_lines = out.count("\n")
new_line = self.__lines.format(key=key, value=value)
if num_lines == 0:
- with open(self.__file_path, 'a', 0o600) as f:
+ with open(self.__file_path, 'a', 0o644) as f:
f.write(new_line + '\n')
elif num_lines == 1:
sed_cmd = [r"sed", "-i", r"s/{}.*$/{}/g".format(re_cmd, new_line), self.__file_path]
--
2.27.0

355
change-import-behavior.patch Normal file
View File

@ -0,0 +1,355 @@
From c02c86fbd6927a3d1968d5a1e3030058bbb943a3 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Sat, 15 Jul 2023 15:50:06 +0800
Subject: [PATCH 11/11] change import behavior
---
.../plugin/configurator/__init__.py | 47 +++++++++++--------
.../plugin/configurator/affinity/__init__.py | 2 +-
.../plugin/configurator/bios/__init__.py | 2 +-
.../configurator/bootloader/__init__.py | 2 +-
atune_collector/plugin/configurator/common.py | 2 +-
.../configurator/file_config/__init__.py | 2 +-
.../configurator/kernel_config/__init__.py | 2 +-
.../plugin/configurator/mysql/__init__.py | 3 +-
.../plugin/configurator/nginx/__init__.py | 3 +-
.../plugin/configurator/redis/__init__.py | 3 +-
.../plugin/configurator/script/__init__.py | 2 +-
.../plugin/configurator/sysctl/__init__.py | 2 +-
.../plugin/configurator/sysfs/__init__.py | 2 +-
.../plugin/configurator/systemctl/__init__.py | 2 +-
.../plugin/configurator/ulimit/__init__.py | 2 +-
atune_collector/plugin/monitor/__init__.py | 11 ++++-
.../plugin/monitor/memory/__init__.py | 2 +-
.../plugin/monitor/network/__init__.py | 2 +-
.../plugin/monitor/performance/__init__.py | 2 +-
.../plugin/monitor/process/__init__.py | 2 +-
.../plugin/monitor/processor/__init__.py | 2 +-
.../plugin/monitor/storage/__init__.py | 2 +-
.../plugin/monitor/system/__init__.py | 2 +-
atune_collector/plugin/plugin.py | 4 +-
24 files changed, 60 insertions(+), 47 deletions(-)
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index 31520f1..fd7ea63 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -17,24 +17,33 @@ Init file.
import sys
import os
-sys.path.append(os.path.dirname(__file__))
+from .affinity import *
+from .bios import *
+from .bootloader import *
+from .file_config import *
+from .kernel_config import *
+from .script import *
+from .sysctl import *
+from .sysfs import *
+from .systemctl import *
+from .ulimit import *
+from .mysql import *
+from .redis import *
+from .nginx import *
-__all__ = [
- "exceptions",
- "affinity",
- "bios",
- "bootloader",
- "file_config",
- "kernel_config",
- "script",
- "sysctl",
- "sysfs",
- "systemctl",
- "ulimit",
- "mysql",
- "redis",
- "nginx",
- "common"]
-
-from . import *
+__all__ = ["exceptions",
+ "affinity",
+ "bios",
+ "bootloader",
+ "file_config",
+ "kernel_config",
+ "script",
+ "sysctl",
+ "sysfs",
+ "systemctl",
+ "ulimit",
+ "mysql",
+ "redis",
+ "nginx",
+ "common"]
diff --git a/atune_collector/plugin/configurator/affinity/__init__.py b/atune_collector/plugin/configurator/affinity/__init__.py
index e29377f..1610cc3 100755
--- a/atune_collector/plugin/configurator/affinity/__init__.py
+++ b/atune_collector/plugin/configurator/affinity/__init__.py
@@ -19,6 +19,6 @@ import os
__all__ = ["irq", "task", "processid"]
-from . import *
+from . import irq, task, processid
sys.path.insert(0, os.path.dirname(__file__))
diff --git a/atune_collector/plugin/configurator/bios/__init__.py b/atune_collector/plugin/configurator/bios/__init__.py
index 1e193cf..1f9df24 100755
--- a/atune_collector/plugin/configurator/bios/__init__.py
+++ b/atune_collector/plugin/configurator/bios/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["bios"]
-from . import *
+from . import bios
diff --git a/atune_collector/plugin/configurator/bootloader/__init__.py b/atune_collector/plugin/configurator/bootloader/__init__.py
index 2412c54..d6cf156 100755
--- a/atune_collector/plugin/configurator/bootloader/__init__.py
+++ b/atune_collector/plugin/configurator/bootloader/__init__.py
@@ -20,6 +20,6 @@ import os
__all__ = ["grub2", "cmdline"]
-from . import *
+from . import grub2, cmdline
sys.path.insert(0, os.path.dirname(__file__))
diff --git a/atune_collector/plugin/configurator/common.py b/atune_collector/plugin/configurator/common.py
index 0fbc2f3..4ab459a 100755
--- a/atune_collector/plugin/configurator/common.py
+++ b/atune_collector/plugin/configurator/common.py
@@ -20,7 +20,7 @@ import logging
import json
from functools import wraps
-from exceptions import SetConfigError
+from .exceptions import SetConfigError
LOGGER = logging.getLogger(__name__)
diff --git a/atune_collector/plugin/configurator/file_config/__init__.py b/atune_collector/plugin/configurator/file_config/__init__.py
index 48aeb30..023de44 100755
--- a/atune_collector/plugin/configurator/file_config/__init__.py
+++ b/atune_collector/plugin/configurator/file_config/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["fconfig"]
-from . import *
+from . import fconfig
diff --git a/atune_collector/plugin/configurator/kernel_config/__init__.py b/atune_collector/plugin/configurator/kernel_config/__init__.py
index 5905327..58dd2b0 100755
--- a/atune_collector/plugin/configurator/kernel_config/__init__.py
+++ b/atune_collector/plugin/configurator/kernel_config/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["kconfig"]
-from . import *
+from . import kconfig
diff --git a/atune_collector/plugin/configurator/mysql/__init__.py b/atune_collector/plugin/configurator/mysql/__init__.py
index 52bb8c3..bfdb807 100644
--- a/atune_collector/plugin/configurator/mysql/__init__.py
+++ b/atune_collector/plugin/configurator/mysql/__init__.py
@@ -17,5 +17,4 @@ Init file.
__all__ = ["mysql"]
-from . import *
-
+from . import mysql
diff --git a/atune_collector/plugin/configurator/nginx/__init__.py b/atune_collector/plugin/configurator/nginx/__init__.py
index e28a310..07c82b0 100644
--- a/atune_collector/plugin/configurator/nginx/__init__.py
+++ b/atune_collector/plugin/configurator/nginx/__init__.py
@@ -17,5 +17,4 @@ Init file.
__all__ = ["nginx"]
-from . import *
-
+from . import nginx
diff --git a/atune_collector/plugin/configurator/redis/__init__.py b/atune_collector/plugin/configurator/redis/__init__.py
index d386f1f..7d21825 100644
--- a/atune_collector/plugin/configurator/redis/__init__.py
+++ b/atune_collector/plugin/configurator/redis/__init__.py
@@ -17,5 +17,4 @@ Init file.
__all__ = ["redis"]
-from . import *
-
+from . import redis
diff --git a/atune_collector/plugin/configurator/script/__init__.py b/atune_collector/plugin/configurator/script/__init__.py
index f743a20..ec6aa17 100755
--- a/atune_collector/plugin/configurator/script/__init__.py
+++ b/atune_collector/plugin/configurator/script/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["script"]
-from . import *
+from . import script
diff --git a/atune_collector/plugin/configurator/sysctl/__init__.py b/atune_collector/plugin/configurator/sysctl/__init__.py
index d6b3f61..3d28df7 100755
--- a/atune_collector/plugin/configurator/sysctl/__init__.py
+++ b/atune_collector/plugin/configurator/sysctl/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["sysctl"]
-from . import *
+from . import sysctl
diff --git a/atune_collector/plugin/configurator/sysfs/__init__.py b/atune_collector/plugin/configurator/sysfs/__init__.py
index d220f40..9d5afb6 100755
--- a/atune_collector/plugin/configurator/sysfs/__init__.py
+++ b/atune_collector/plugin/configurator/sysfs/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["sysfs"]
-from . import *
+from . import sysfs
diff --git a/atune_collector/plugin/configurator/systemctl/__init__.py b/atune_collector/plugin/configurator/systemctl/__init__.py
index 94762b6..c51a07f 100755
--- a/atune_collector/plugin/configurator/systemctl/__init__.py
+++ b/atune_collector/plugin/configurator/systemctl/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["systemctl"]
-from . import *
+from . import systemctl
diff --git a/atune_collector/plugin/configurator/ulimit/__init__.py b/atune_collector/plugin/configurator/ulimit/__init__.py
index 2582b7c..c28a601 100755
--- a/atune_collector/plugin/configurator/ulimit/__init__.py
+++ b/atune_collector/plugin/configurator/ulimit/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["ulimit"]
-from . import *
+from . import ulimit
diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py
index 9292071..948f869 100755
--- a/atune_collector/plugin/monitor/__init__.py
+++ b/atune_collector/plugin/monitor/__init__.py
@@ -15,6 +15,15 @@
Init file.
"""
+from .memory import *
+from .network import *
+from .performance import *
+from .process import *
+from .processor import *
+from .storage import *
+from .system import *
+from . import common
+
__all__ = [
"memory",
"network",
@@ -24,5 +33,3 @@ __all__ = [
"storage",
"common",
"system"]
-
-from . import *
diff --git a/atune_collector/plugin/monitor/memory/__init__.py b/atune_collector/plugin/monitor/memory/__init__.py
index 0a5a356..5002d6e 100755
--- a/atune_collector/plugin/monitor/memory/__init__.py
+++ b/atune_collector/plugin/monitor/memory/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["numainfo", "topo", "bandwidth", "vmstat", "utilstat", "meminfo"]
-from . import *
+from . import numainfo, topo, bandwidth, vmstat, utilstat, meminfo
diff --git a/atune_collector/plugin/monitor/network/__init__.py b/atune_collector/plugin/monitor/network/__init__.py
index 5d40ab8..388611a 100755
--- a/atune_collector/plugin/monitor/network/__init__.py
+++ b/atune_collector/plugin/monitor/network/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["info", "netstat", "netestat", "topo"]
-from . import *
+from . import info, netstat, netestat, topo
diff --git a/atune_collector/plugin/monitor/performance/__init__.py b/atune_collector/plugin/monitor/performance/__init__.py
index 707f911..93fc6c4 100755
--- a/atune_collector/plugin/monitor/performance/__init__.py
+++ b/atune_collector/plugin/monitor/performance/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["stat", "top"]
-from . import *
+from . import stat, top
diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py
index 4c4ceb3..5a34c10 100644
--- a/atune_collector/plugin/monitor/process/__init__.py
+++ b/atune_collector/plugin/monitor/process/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["sched"]
-from . import *
\ No newline at end of file
+from . import sched
diff --git a/atune_collector/plugin/monitor/processor/__init__.py b/atune_collector/plugin/monitor/processor/__init__.py
index fc88e1b..f46d8fd 100755
--- a/atune_collector/plugin/monitor/processor/__init__.py
+++ b/atune_collector/plugin/monitor/processor/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["info", "stat", "topo"]
-from . import *
+from . import info, stat, topo
diff --git a/atune_collector/plugin/monitor/storage/__init__.py b/atune_collector/plugin/monitor/storage/__init__.py
index 670294d..9e5e152 100755
--- a/atune_collector/plugin/monitor/storage/__init__.py
+++ b/atune_collector/plugin/monitor/storage/__init__.py
@@ -17,4 +17,4 @@ Init file.
__all__ = ["iostat", "topo"]
-from . import *
+from . import iostat, topo
diff --git a/atune_collector/plugin/monitor/system/__init__.py b/atune_collector/plugin/monitor/system/__init__.py
index 5c6a00c..f66c1ef 100755
--- a/atune_collector/plugin/monitor/system/__init__.py
+++ b/atune_collector/plugin/monitor/system/__init__.py
@@ -17,4 +17,4 @@ The import content of the package.
__all__ = ["bios", "ldavg", "tasks", "filed", "interrupts"]
-from . import *
+from . import bios, ldavg, tasks, filed, interrupts
diff --git a/atune_collector/plugin/plugin.py b/atune_collector/plugin/plugin.py
index b83d3aa..3f84f2f 100755
--- a/atune_collector/plugin/plugin.py
+++ b/atune_collector/plugin/plugin.py
@@ -19,9 +19,9 @@ import logging
import threading
import time
-from configurator.common import Configurator
+from .configurator.common import Configurator
-from monitor.common import Monitor
+from .monitor.common import Monitor
LOGGER = logging.getLogger(__name__)
--
2.27.0

145
feature-set-mysql.patch Normal file
View File

@ -0,0 +1,145 @@
From 16ef5dc208cd79b9f9d4c7c9d66ce46dc9d013a1 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Tue, 11 Jul 2023 20:53:28 +0800
Subject: [PATCH 07/11] feature: set mysql
---
.../plugin/configurator/__init__.py | 1 +
.../plugin/configurator/mysql/__init__.py | 21 +++++
.../plugin/configurator/mysql/mysql.py | 84 +++++++++++++++++++
3 files changed, 106 insertions(+)
create mode 100644 atune_collector/plugin/configurator/mysql/__init__.py
create mode 100644 atune_collector/plugin/configurator/mysql/mysql.py
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index 29c674a..677d208 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -31,6 +31,7 @@ __all__ = [
"sysfs",
"systemctl",
"ulimit",
+ "mysql",
"common"]
from . import *
diff --git a/atune_collector/plugin/configurator/mysql/__init__.py b/atune_collector/plugin/configurator/mysql/__init__.py
new file mode 100644
index 0000000..52bb8c3
--- /dev/null
+++ b/atune_collector/plugin/configurator/mysql/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-11
+
+"""
+Init file.
+"""
+
+__all__ = ["mysql"]
+
+from . import *
+
diff --git a/atune_collector/plugin/configurator/mysql/mysql.py b/atune_collector/plugin/configurator/mysql/mysql.py
new file mode 100644
index 0000000..cfc533d
--- /dev/null
+++ b/atune_collector/plugin/configurator/mysql/mysql.py
@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-11
+
+"""
+The sub class of the Configurator, used to change the /etc/my.cnf config.
+"""
+
+
+import logging
+import os
+from ..common import Configurator
+
+LOGGER = logging.getLogger(__name__)
+
+class Mysql(Configurator):
+ _module = "MYSQL"
+ _submod = "MYSQL"
+
+ def __init__(self, user=None):
+ Configurator.__init__(self, user)
+ self.__cmd = ""
+ self.__file_path = "/etc/my.cnf"
+ self.__mysqld_ind = -1
+ if not os.path.isfile(self.__file_path):
+ with open(self.__file_path, 'w'):
+ pass
+ os.chmod(self.__file_path, 0o644)
+ self.__check_mysqld()
+
+ def __check_mysqld(self):
+ with open(self.__file_path, 'r') as f:
+ lines = f.readlines()
+
+ for line in lines:
+ self.__mysqld_ind = self.__mysqld_ind + 1
+ if line[:-1] == '[mysqld]':
+ self.__mysqld_ind = self.__mysqld_ind + 1
+ return
+
+ lines.insert(self.__mysqld_ind, "[mysqld]\n")
+ self.__mysqld_ind = self.__mysqld_ind + 1
+ with open(self.__file_path, 'w') as f:
+ f.writelines(lines)
+
+ def __check_file_exists(self, lines, start_with):
+ ind = -1
+ for line in lines:
+ ind = ind + 1
+ if line.split('=')[0].rstrip() == start_with:
+ return True, ind
+ return False, ind
+
+ def _set(self, key, value):
+ with open(self.__file_path, 'r') as f:
+ lines = f.readlines()
+
+ key_exist, ind = self.__check_file_exists(lines, key)
+ new_line = key + " = " + value + "\n"
+ if not key_exist:
+ lines.insert(self.__mysqld_ind, new_line)
+ else:
+ lines[ind] = new_line
+
+ with open(self.__file_path, 'w') as f:
+ f.writelines(lines)
+ return 0
+
+ def _get(self, key, _):
+ pass
+
+ @staticmethod
+ def check(config1, config2):
+ return True
+
--
2.27.0

272
feature-set-nginx.patch Normal file
View File

@ -0,0 +1,272 @@
From b723f4cc584e6df195fbd8b8d0213a3b53aeceea Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Fri, 14 Jul 2023 16:50:53 +0800
Subject: [PATCH 09/11] feature: set nginx
---
.../plugin/configurator/__init__.py | 1 +
.../plugin/configurator/nginx/__init__.py | 21 ++
.../plugin/configurator/nginx/nginx.py | 211 ++++++++++++++++++
3 files changed, 233 insertions(+)
create mode 100644 atune_collector/plugin/configurator/nginx/__init__.py
create mode 100644 atune_collector/plugin/configurator/nginx/nginx.py
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index 9597692..31520f1 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -33,6 +33,7 @@ __all__ = [
"ulimit",
"mysql",
"redis",
+ "nginx",
"common"]
from . import *
diff --git a/atune_collector/plugin/configurator/nginx/__init__.py b/atune_collector/plugin/configurator/nginx/__init__.py
new file mode 100644
index 0000000..e28a310
--- /dev/null
+++ b/atune_collector/plugin/configurator/nginx/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-14
+
+"""
+Init file.
+"""
+
+__all__ = ["nginx"]
+
+from . import *
+
diff --git a/atune_collector/plugin/configurator/nginx/nginx.py b/atune_collector/plugin/configurator/nginx/nginx.py
new file mode 100644
index 0000000..7ab619d
--- /dev/null
+++ b/atune_collector/plugin/configurator/nginx/nginx.py
@@ -0,0 +1,211 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-14
+
+"""
+The sub class of the Configurator, used to extend script for /etc/nginx/nginx.conf for CPI.
+"""
+
+import logging
+import os
+import subprocess
+
+from ..exceptions import SetConfigError
+from ..common import Configurator
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Nginx(Configurator):
+ _module = "NGINX"
+ _submod = "NGINX"
+
+ def __init__(self, user=None):
+ Configurator.__init__(self, user)
+ self.__file_dir = "/etc/nginx/"
+ self.__file_path = self.__file_dir + "nginx.conf"
+ self.__line = "{key} {value};\n"
+ self.__key_match = r"^([a-zA-Z]+(_[a-zA-Z]+)*.?[a-zA-Z]+)*$"
+ self.__ind = {"main": [0, 0]}
+
+ def _init_file(self):
+ if not os.path.isdir(self.__file_dir):
+ os.mkdir(self.__file_dir)
+ os.chmod(self.__file_dir, 0o755)
+ if not os.path.isfile(self.__file_path):
+ with open(self.__file_path, 'w'):
+ pass
+ os.chmod(self.__file_path, 0o644)
+ self._set_index()
+
+ def _get_lines(self):
+ with open(self.__file_path, 'r') as f:
+ lines = f.readlines()
+ return lines
+
+ def _set_lines(self, lines):
+ with open(self.__file_path, 'w') as f:
+ f.writelines(lines)
+
+ def _set_index(self):
+ lines = self._get_lines()
+ filo = []
+ ind = []
+ filo.append("start")
+ ind.append(0)
+ is_https_server = False
+ for i, line in enumerate(lines):
+ if i == len(lines) - 1:
+ self.__ind["main"] = [ind.pop(), i]
+ del filo, ind
+ if line.strip().startswith("}"):
+ close = filo.pop()
+ close_ind = ind.pop()
+ if close.startswith("events"):
+ self.__ind["events"] = [close_ind, i]
+ if close.startswith("http"):
+ self.__ind["http"] = [close_ind, i]
+ if close.startswith("server"):
+ if is_https_server:
+ self.__ind["https_server"] = [close_ind, i]
+ else:
+ self.__ind["http_server"] = [close_ind, i]
+ is_https_server = False
+ if "{" in line.strip().split("#")[0]:
+ filo.append(line.strip())
+ ind.append(i)
+ if line.strip().startswith("ssl_"):
+ is_https_server = True
+
+ def _check_key(self, key):
+ if len(key) == 0:
+ return False, 0
+ count = 1
+ if not ('a' <= key[0] <= 'z' or 'A' <= key[0] <= 'Z'):
+ return False, count
+ for i in range(1, len(key)):
+ if key[i] == '_' or key[i] == '.':
+ if not ('a' <= key[i - 1] <= 'z' or 'A' <= key[i - 1] <= 'Z'):
+ return False, count
+ if i == len(key) - 1:
+ return False, count
+ if key[i] == '.':
+ count = count + 1
+ elif 'a' <= key[i] <= 'z' or 'A' <= key[i] <= 'Z':
+ continue
+ else:
+ return False, count
+ return True, count
+
+ def _update_index(self, after_ind):
+ for key in self.__ind:
+ for i, val in enumerate(self.__ind[key]):
+ if val > after_ind:
+ self.__ind[key][i] = val + 1
+
+ def _append_lines(self, section, key, value):
+ lines = self._get_lines()
+ filo = []
+ for i in range(self.__ind[section][0], self.__ind[section][1]):
+ curr_line = lines[i].strip()
+ if "{" in curr_line.split("#")[0]:
+ filo.append(lines[i].strip())
+ elif curr_line.startswith("}"):
+ filo.pop()
+ elif section == 'main' and len(filo) > 0:
+ continue
+ elif len(filo) > 1:
+ continue
+ elif curr_line.startswith(key + " ") or curr_line.startswith(key + "\t"):
+ new_line = lines[i].split(key)[0]
+ new_line = new_line + self.__line.format(key=key, value=value)
+ lines[i] = new_line
+ return lines
+ new_line = ""
+ if section != "main" and not "server" in section:
+ new_line = lines[self.__ind[section][0]].split(section)[0] + " "
+ elif "server" in section:
+ new_line = lines[self.__ind[section][0]].split("server")[0] + " "
+ new_line = new_line + self.__line.format(key=key, value=value)
+ lines.insert(self.__ind[section][0] + 1, new_line)
+ self._update_index(self.__ind[section][0])
+ return lines
+
+ def _set_main_section(self, key, value):
+ return self._append_lines("main", key, value)
+
+ def _set_event_section(self, key, value):
+ if "events" not in self.__ind:
+ raise SetConfigError("No events section")
+ return self._append_lines("events", key, value)
+
+ def _set_http_section(self, key, value):
+ if "http" not in self.__ind:
+ raise SetConfigError("No http section")
+ return self._append_lines("http", key, value)
+
+ def _set_http_server(self, key, value):
+ if "http_server" not in self.__ind:
+ raise SetConfigError("No http server in http section")
+ return self._append_lines("http_server", key, value)
+
+ def _set_https_server(self, key, value):
+ if "https_server" not in self.__ind:
+ raise SetConfigError("No https server in http section")
+ return self._append_lines("https_server", key, value)
+
+ def _set(self, key, value):
+ self._init_file()
+ valid, count = self._check_key(key)
+ if not valid or count < 1 or count > 3:
+ raise SetConfigError("Invalid value {}".format(key))
+ if value == "!CPU_CORE!":
+ value = rewrite_value(value)
+ lines = []
+ try:
+ if count == 1:
+ lines = self._set_main_section(key, value)
+ elif count == 2 and key.split('.')[0] == "events":
+ lines = self._set_event_section(key.split('.')[1], value)
+ elif key.split('.')[0] == "http":
+ if count == 2:
+ lines = self._set_http_section(key.split('.')[1], value)
+ elif count == 3:
+ if key.split('.')[1] == "http":
+ lines = self._set_http_server(key.split('.')[2], value)
+ elif key.split('.')[1] == "https":
+ lines = self._set_https_server(key.split('.')[2], value)
+ else:
+ raise SetConfigError("Invalid value {}".format(key))
+ else:
+ raise SetConfigError("Invalid value {}".format(key))
+ else:
+ raise SetConfigError("Invalid value {}".format(key))
+ except SetConfigError as err:
+ raise err
+ self._set_lines(lines)
+ return 0
+
+ def _get(self, key, _):
+ pass
+
+ @staticmethod
+ def check(config1, config2):
+ return True
+
+def rewrite_value(value):
+ command = ["grep", "processor", "/proc/cpuinfo"]
+ output = subprocess.run(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if output.returncode != 0:
+ raise SetConfigError("Failed to get cpu number")
+ return output.stdout.decode().count("\n")
+
--
2.27.0

139
feature-set-redis-value.patch Normal file
View File

@ -0,0 +1,139 @@
From 70a1f8bc884b6a1b2772fd7199e103fa5351ca3d Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Thu, 13 Jul 2023 14:40:57 +0800
Subject: [PATCH 08/11] feature: set redis value
---
.../plugin/configurator/__init__.py | 2 +
.../plugin/configurator/redis/__init__.py | 21 +++++
.../plugin/configurator/redis/redis.py | 77 +++++++++++++++++++
3 files changed, 100 insertions(+)
create mode 100644 atune_collector/plugin/configurator/redis/__init__.py
create mode 100644 atune_collector/plugin/configurator/redis/redis.py
diff --git a/atune_collector/plugin/configurator/__init__.py b/atune_collector/plugin/configurator/__init__.py
index 677d208..9597692 100755
--- a/atune_collector/plugin/configurator/__init__.py
+++ b/atune_collector/plugin/configurator/__init__.py
@@ -32,6 +32,8 @@ __all__ = [
"systemctl",
"ulimit",
"mysql",
+ "redis",
"common"]
from . import *
+
diff --git a/atune_collector/plugin/configurator/redis/__init__.py b/atune_collector/plugin/configurator/redis/__init__.py
new file mode 100644
index 0000000..d386f1f
--- /dev/null
+++ b/atune_collector/plugin/configurator/redis/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-13
+
+"""
+Init file.
+"""
+
+__all__ = ["redis"]
+
+from . import *
+
diff --git a/atune_collector/plugin/configurator/redis/redis.py b/atune_collector/plugin/configurator/redis/redis.py
new file mode 100644
index 0000000..c56f38e
--- /dev/null
+++ b/atune_collector/plugin/configurator/redis/redis.py
@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2023-07-13
+
+"""
+The sub class of the Configurator, used to change the redis config.
+"""
+
+import logging
+import os
+import subprocess
+from ..exceptions import GetConfigError, SetConfigError
+from ..common import Configurator
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Redis(Configurator):
+ _module = "REDIS"
+ _submod = "REDIS"
+
+ def __init__(self, user=None):
+ Configurator.__init__(self, user)
+ self.__lines = "{key} {value}"
+ self.__re = r"^#\?\s*{key}\s* "
+ self.__file_dir = "/etc/redis/"
+ self.__file_path = self.__file_dir + "redis.conf"
+ if not os.path.isdir(self.__file_dir):
+ os.mkdir(self.__file_dir)
+ os.chmod(self.__file_dir, 0o755)
+ if not os.path.isfile(self.__file_path):
+ with open(self.__file_path, 'w', 0o200):
+ pass
+ os.chmod(self.__file_path, 0o644)
+
+ def _set(self, key, value):
+ re_cmd = self.__re.format(key=key)
+ grep_cmd = [r"grep", re_cmd, self.__file_path]
+ out, err = self.execute_cmd(grep_cmd)
+ if len(err) != 0:
+ raise SetConfigError("Failed to set {}: {}".format(key, err))
+ num_lines = out.count("\n")
+ new_line = self.__lines.format(key=key, value=value)
+ if num_lines == 0:
+ with open(self.__file_path, 'a', 0o600) as f:
+ f.write(new_line + '\n')
+ elif num_lines == 1:
+ sed_cmd = [r"sed", "-i", r"s/{}.*$/{}/g".format(re_cmd, new_line), self.__file_path]
+ _, err_rep = self.execute_cmd(sed_cmd)
+ if len(err_rep) > 0:
+ raise SetConfigError("Failed to set {}: {}".format(key, err_rep))
+ else:
+ raise SetConfigError("Failed to set {}: more than 1 key has same name".format(key))
+ return 0
+
+ def _get(self, key, _):
+ pass
+
+ @staticmethod
+ def check(config1, config2):
+ return True
+
+ @staticmethod
+ def execute_cmd(cmd):
+ output = subprocess.run(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ return output.stdout.decode(), output.stderr.decode()[:-1]
+
+
--
2.27.0

33
fix-CWE-23.patch Normal file
View File

@ -0,0 +1,33 @@
From 121a1bcbd68ef9b18ec0c0cdcc8ca0748fe08bdd Mon Sep 17 00:00:00 2001
From: unknown <u202012145@hust.edu.cn>
Date: Wed, 26 Apr 2023 21:44:16 +0800
Subject: [PATCH 05/11] fix CWE-23
collect_data.py直接使用命令行参数作为文件路径，可以被攻击者输入../访问上级目录，从而获取系统的敏感信息，或者写入任意文件（使用保存路径遍历文件名的恶意zip存档）。为了修复这一漏洞，我们采用了werzeug库中的secure_filename函数，这一函数会过滤掉文件路径中的所有危险字符，防范这一攻击方式。
---
atune_collector/collect_data.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py
index 3593db6..167141e 100755
--- a/atune_collector/collect_data.py
+++ b/atune_collector/collect_data.py
@@ -21,6 +21,7 @@ import time
import csv
from plugin.plugin import MPI
+from werkzeug.utils import secure_filename
class Collector:
@@ -112,6 +113,7 @@ if __name__ == "__main__":
ARG_PARSER.add_argument('-c', '--config', metavar='json',
default=default_json_path, help='input json path')
ARGS = ARG_PARSER.parse_args()
+ filename=secure_filename(ARGS.config)
with open(ARGS.config, 'r') as file:
json_data = json.load(file)
collector = Collector(json_data)
--
2.27.0

4
0001-fix-bug-list-index-out-of-range-in-virtualbox.patch → fix-bug-list-index-out-of-range-in-virtualbox.patch
View File

@ -1,7 +1,7 @@
From 4cb49adb623f36590f254b7e11c8aa5b0777bd00 Mon Sep 17 00:00:00 2001
From: gaoruoshu <gaoruoshu@huawei.com>
Date: Tue, 26 Apr 2022 02:01:27 +0000
Subject: [PATCH] fix bug: list index out of range in virtualbox
Subject: [PATCH 03/11] fix bug: list index out of range in virtualbox
---
atune_collector/plugin/monitor/memory/bandwidth.py | 2 +-
@ -21,5 +21,5 @@ index 593a4a8..f44925f 100755
for dimm in info["memorys"][0]["children"]:
if dimm.get("size") is None:
--
2.33.0
2.27.0

37
memory-bandwidth-dimm-slot-atuned.patch Normal file
View File

@ -0,0 +1,37 @@
From 7c21a2da2c931089e4ec2b86ae35a510f2f7793d Mon Sep 17 00:00:00 2001
From: SWWBF <1152719547@qq.com>
Date: Mon, 10 Jul 2023 05:52:54 +0800
Subject: [PATCH 06/11] some kvm has no slot which may cause panic of
atune
---
atune_collector/plugin/monitor/memory/bandwidth.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/atune_collector/plugin/monitor/memory/bandwidth.py b/atune_collector/plugin/monitor/memory/bandwidth.py
index f44925f..6d59ab6 100755
--- a/atune_collector/plugin/monitor/memory/bandwidth.py
+++ b/atune_collector/plugin/monitor/memory/bandwidth.py
@@ -158,12 +158,13 @@ class MemBandwidth(Monitor):
for dimm in info["memorys"][0]["children"]:
if dimm.get("size") is None:
continue
- locator = memtopo.table_get_locator(dimm["slot"])
- if locator is None:
- continue
- if dimms[locator[0]][locator[1]] == 0:
- dimms[locator[0]][locator[1]] = dimm["width"] * \
- memtopo.table_get_freq(dimm["description"]) / 8
+ if "slot" in dimm:
+ locator = memtopo.table_get_locator(dimm["slot"])
+ if locator is None:
+ continue
+ if dimms[locator[0]][locator[1]] == 0:
+ dimms[locator[0]][locator[1]] = dimm["width"] * \
+ memtopo.table_get_freq(dimm["description"]) / 8
ret = 0
for channel in dimms[socket]:
ret += channel
--
2.27.0