428 lines
16 KiB
Diff
428 lines
16 KiB
Diff
From 2d6269e360a654f11370f31ea4faf6ee4012d73f Mon Sep 17 00:00:00 2001
|
||
From: hamster <hujing@isrc.iscas.ac.cn>
|
||
Date: Mon, 14 Nov 2022 03:24:49 +0000
|
||
Subject: [PATCH 04/11] add new dimensions
|
||
|
||
From: @hujing2
|
||
Reviewed-by: @gaoruoshu
|
||
Signed-off-by: @gaoruoshu
|
||
---
|
||
README.md | 16 +++
|
||
atune_collector/collect_data.json | 14 ++
|
||
atune_collector/collect_data.py | 97 ++++++++------
|
||
atune_collector/plugin/monitor/__init__.py | 1 +
|
||
atune_collector/plugin/monitor/common.py | 1 +
|
||
.../plugin/monitor/process/__init__.py | 20 +++
|
||
.../plugin/monitor/process/sched.py | 124 ++++++++++++++++++
|
||
7 files changed, 233 insertions(+), 40 deletions(-)
|
||
create mode 100644 atune_collector/plugin/monitor/process/__init__.py
|
||
create mode 100644 atune_collector/plugin/monitor/process/sched.py
|
||
|
||
diff --git a/README.md b/README.md
|
||
index c121c8d..242f345 100644
|
||
--- a/README.md
|
||
+++ b/README.md
|
||
@@ -50,12 +50,14 @@ python3 collect_data.py [OPTIONS]
|
||
| ---------------- | ------------------------------------- | ------------ | ------------ |
|
||
| network | 待采集的指定网卡 | 字符串 | - |
|
||
| block | 待采集的指定磁盘 | 字符串 | - |
|
||
+| application | 需要采集的应用进程 | 字符串 | - |
|
||
| sample_num | 待采集的次数 | 整型 | >0 |
|
||
| interval | 待采集的间隔时间，单位为秒 | 整型 | >0 |
|
||
| output_dir | 采集完后数据存储的文件路径 | 字符串 | - |
|
||
| workload_type | 采集环境的应用负载类型，用作输出文件名，默认为default | 字符串 | - |
|
||
| collection_items | 需要采集的系统参数项，参见表2 | 列表 | - |
|
||
|
||
+
|
||
最终采集完后，数据将保存为: `${output_dir}/${workload_type}-${finish_timestamp}.csv`
|
||
|
||
表2 collection_items项配置说明
|
||
@@ -76,6 +78,7 @@ collect_data.json文件配置示例：
|
||
{
|
||
"network": "eth0",
|
||
"block": "sda",
|
||
+ "application": "mysqld",
|
||
"sample_num": 20,
|
||
"interval": 5,
|
||
"output_dir": "/var/atuned/collect_data",
|
||
@@ -203,6 +206,19 @@ collect_data.json文件配置示例：
|
||
"metrics": [
|
||
"fd-util"
|
||
]
|
||
+ },
|
||
+ {
|
||
+ "name": "process",
|
||
+ "module": "PROCESS",
|
||
+ "purpose": "SCHED",
|
||
+ "metrics": [
|
||
+ "exec_start",
|
||
+ "vruntime",
|
||
+ "sum_exec_runtime",
|
||
+ "switches",
|
||
+ "voluntary_switches",
|
||
+ "involuntary_switches"
|
||
+ ]
|
||
}
|
||
]
|
||
}
|
||
diff --git a/atune_collector/collect_data.json b/atune_collector/collect_data.json
|
||
index db96501..af286bd 100755
|
||
--- a/atune_collector/collect_data.json
|
||
+++ b/atune_collector/collect_data.json
|
||
@@ -1,6 +1,7 @@
|
||
{
|
||
"network": "eth0",
|
||
"block": "sda",
|
||
+ "application": "firewalld,dockerd",
|
||
"sample_num": 20,
|
||
"interval": 5,
|
||
"output_dir": "/var/atuned/collect_data",
|
||
@@ -140,6 +141,19 @@
|
||
"metrics": [
|
||
"fd-util"
|
||
]
|
||
+ },
|
||
+ {
|
||
+ "name": "process",
|
||
+ "module": "PROCESS",
|
||
+ "purpose": "SCHED",
|
||
+ "metrics": [
|
||
+ "exec_start",
|
||
+ "vruntime",
|
||
+ "sum_exec_runtime",
|
||
+ "switches",
|
||
+ "voluntary_switches",
|
||
+ "involuntary_switches"
|
||
+ ]
|
||
}
|
||
]
|
||
}
|
||
\ No newline at end of file
|
||
diff --git a/atune_collector/collect_data.py b/atune_collector/collect_data.py
|
||
index 1764304..3593db6 100755
|
||
--- a/atune_collector/collect_data.py
|
||
+++ b/atune_collector/collect_data.py
|
||
@@ -18,6 +18,7 @@ import argparse
|
||
import json
|
||
import os
|
||
import time
|
||
+import csv
|
||
|
||
from plugin.plugin import MPI
|
||
|
||
@@ -30,66 +31,80 @@ class Collector:
|
||
self.field_name = []
|
||
self.support_multi_block = ['storage']
|
||
self.support_multi_nic = ['network', 'network-err']
|
||
+ self.support_multi_app = ['process']
|
||
|
||
def parse_json(self):
|
||
"""parse json data"""
|
||
monitors = []
|
||
for item in self.data["collection_items"]:
|
||
- parameters = ["--interval=%s;" % self.data["interval"]]
|
||
- for metric in item["metrics"]:
|
||
- nics = self.data["network"].split(',')
|
||
- blocks = self.data["block"].split(',')
|
||
- if item["name"] in self.support_multi_nic and len(nics) > 1:
|
||
- for net in nics:
|
||
+ if item["name"] in self.support_multi_app and ('application' not in self.data or
|
||
+ self.data["application"] == ""):
|
||
+ continue
|
||
+ if item["name"] in self.support_multi_app:
|
||
+ applications = self.data["application"].split(',')
|
||
+ parameters = ["--interval=%s --app=%s;" %(self.data["interval"], self.data["application"])]
|
||
+ for application in applications:
|
||
+ for metric in item["metrics"]:
|
||
self.field_name.append(
|
||
- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net))
|
||
- elif item["name"] in self.support_multi_block and len(blocks) > 1:
|
||
- for block in blocks:
|
||
- self.field_name.append(
|
||
- "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block))
|
||
- else:
|
||
- self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric))
|
||
- parameters.append("--fields=%s" % metric)
|
||
- if "threshold" in item:
|
||
- parameters.append("--threshold=%s" % item["threshold"])
|
||
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, application))
|
||
+ parameters.append("--fields=%s" % metric)
|
||
+ else:
|
||
+ parameters = ["--interval=%s;" % self.data["interval"]]
|
||
+ for metric in item["metrics"]:
|
||
+ nics = self.data["network"].split(',')
|
||
+ blocks = self.data["block"].split(',')
|
||
+
|
||
+ if item["name"] in self.support_multi_nic and len(nics) > 1:
|
||
+ for net in nics:
|
||
+ self.field_name.append(
|
||
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, net))
|
||
+ elif item["name"] in self.support_multi_block and len(blocks) > 1:
|
||
+ for block in blocks:
|
||
+ self.field_name.append(
|
||
+ "%s.%s.%s#%s" % (item["module"], item["purpose"], metric, block))
|
||
+ else:
|
||
+ self.field_name.append("%s.%s.%s" % (item["module"], item["purpose"], metric))
|
||
+ parameters.append("--fields=%s" % metric)
|
||
+ if "threshold" in item:
|
||
+ parameters.append("--threshold=%s" % item["threshold"])
|
||
+
|
||
parameters.append("--nic=%s" % self.data["network"])
|
||
parameters.append("--device=%s" % self.data["block"])
|
||
monitors.append([item["module"], item["purpose"], " ".join(parameters)])
|
||
return monitors
|
||
|
||
- def save_csv(self, field_data):
|
||
- """save data to csv file"""
|
||
+ def collect_data(self):
|
||
+ """collect data"""
|
||
+ collect_num = self.data["sample_num"]
|
||
+ if int(collect_num) < 1:
|
||
+ os.abort("sample_num must be greater than 0")
|
||
+
|
||
+ mpi = MPI()
|
||
+ monitors = self.parse_json()
|
||
path = self.data["output_dir"]
|
||
if not os.path.exists(path):
|
||
os.makedirs(path, 0o750)
|
||
file_name = "{}-{}.csv".format(self.data.get("workload_type", "default"),
|
||
int(round(time.time() * 1000)))
|
||
- import csv
|
||
+
|
||
+ print("start to collect data, csv path is %s" % os.path.join(path, file_name))
|
||
+ print(" ".join(self.field_name))
|
||
with open(os.path.join(path, file_name), "w") as csvfile:
|
||
writer = csv.writer(csvfile)
|
||
self.field_name.insert(0, "TimeStamp")
|
||
writer.writerow(self.field_name)
|
||
- writer.writerows(field_data)
|
||
+ csvfile.flush()
|
||
+ for _ in range(collect_num):
|
||
+ raw_data = mpi.get_monitors_data(monitors)
|
||
+ float_data = [float(num) for num in raw_data]
|
||
+ str_data = [str(round(value, 3)) for value in float_data]
|
||
+ print(" ".join(str_data))
|
||
+ float_data.insert(0, time.strftime("%H:%M:%S"))
|
||
+ # field_data.append(float_data)
|
||
+ writer.writerow(float_data)
|
||
+ csvfile.flush()
|
||
print("finish to collect data, csv path is %s" % os.path.join(path, file_name))
|
||
|
||
- def collect_data(self):
|
||
- """collect data"""
|
||
- collect_num = self.data["sample_num"]
|
||
- if int(collect_num) < 1:
|
||
- os.abort("sample_num must be greater than 0")
|
||
- field_data = []
|
||
- mpi = MPI()
|
||
- monitors = self.parse_json()
|
||
- print(" ".join(self.field_name))
|
||
- for _ in range(collect_num):
|
||
- raw_data = mpi.get_monitors_data(monitors)
|
||
- float_data = [float(num) for num in raw_data]
|
||
- str_data = [str(round(value, 3)) for value in float_data]
|
||
- print(" ".join(str_data))
|
||
- float_data.insert(0, time.strftime("%H:%M:%S"))
|
||
- field_data.append(float_data)
|
||
- return field_data
|
||
-
|
||
|
||
if __name__ == "__main__":
|
||
default_json_path = "/etc/atune_collector/collect_data.json"
|
||
@@ -100,5 +115,7 @@ if __name__ == "__main__":
|
||
with open(ARGS.config, 'r') as file:
|
||
json_data = json.load(file)
|
||
collector = Collector(json_data)
|
||
- dataset = collector.collect_data()
|
||
- collector.save_csv(dataset)
|
||
+ try:
|
||
+ collector.collect_data()
|
||
+ except KeyboardInterrupt:
|
||
+ print("user stop collect data")
|
||
\ No newline at end of file
|
||
diff --git a/atune_collector/plugin/monitor/__init__.py b/atune_collector/plugin/monitor/__init__.py
|
||
index 6291401..9292071 100755
|
||
--- a/atune_collector/plugin/monitor/__init__.py
|
||
+++ b/atune_collector/plugin/monitor/__init__.py
|
||
@@ -19,6 +19,7 @@ __all__ = [
|
||
"memory",
|
||
"network",
|
||
"performance",
|
||
+ "process",
|
||
"processor",
|
||
"storage",
|
||
"common",
|
||
diff --git a/atune_collector/plugin/monitor/common.py b/atune_collector/plugin/monitor/common.py
|
||
index 12a07f2..d0aa60c 100755
|
||
--- a/atune_collector/plugin/monitor/common.py
|
||
+++ b/atune_collector/plugin/monitor/common.py
|
||
@@ -134,6 +134,7 @@ class Monitor(object):
|
||
"--interval=" to specify period of time
|
||
"--cpu=" to select which cpu
|
||
"--event=" to select which event
|
||
+ "--app" to select which applications
|
||
:returns value: Success, collected info string
|
||
:raises Exceptions: Fail, with info
|
||
"""
|
||
diff --git a/atune_collector/plugin/monitor/process/__init__.py b/atune_collector/plugin/monitor/process/__init__.py
|
||
new file mode 100644
|
||
index 0000000..4c4ceb3
|
||
--- /dev/null
|
||
+++ b/atune_collector/plugin/monitor/process/__init__.py
|
||
@@ -0,0 +1,20 @@
|
||
+#!/usr/bin/python3
|
||
+# -*- coding: utf-8 -*-
|
||
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
|
||
+# A-Tune is licensed under the Mulan PSL v2.
|
||
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
||
+# You may obtain a copy of Mulan PSL v2 at:
|
||
+# http://license.coscl.org.cn/MulanPSL2
|
||
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
|
||
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
|
||
+# PURPOSE.
|
||
+# See the Mulan PSL v2 for more details.
|
||
+# Create: 2022-10-14
|
||
+
|
||
+"""
|
||
+Init file.
|
||
+"""
|
||
+
|
||
+__all__ = ["sched"]
|
||
+
|
||
+from . import *
|
||
\ No newline at end of file
|
||
diff --git a/atune_collector/plugin/monitor/process/sched.py b/atune_collector/plugin/monitor/process/sched.py
|
||
new file mode 100644
|
||
index 0000000..5289d84
|
||
--- /dev/null
|
||
+++ b/atune_collector/plugin/monitor/process/sched.py
|
||
@@ -0,0 +1,124 @@
|
||
+#!/usr/bin/python3
|
||
+# -*- coding: utf-8 -*-
|
||
+# Copyright (c) 2022 Huawei Technologies Co., Ltd.
|
||
+# A-Tune is licensed under the Mulan PSL v2.
|
||
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
|
||
+# You may obtain a copy of Mulan PSL v2 at:
|
||
+# http://license.coscl.org.cn/MulanPSL2
|
||
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
|
||
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
|
||
+# PURPOSE.
|
||
+# See the Mulan PSL v2 for more details.
|
||
+# Create: 2022-10-14
|
||
+
|
||
+"""
|
||
+The sub class of the monitor, used to collect the process sched info
|
||
+"""
|
||
+import inspect
|
||
+import logging
|
||
+import subprocess
|
||
+import getopt
|
||
+import re
|
||
+from ..common import Monitor
|
||
+
|
||
+LOGGER = logging.getLogger(__name__)
|
||
+
|
||
+
|
||
+class ProcSched(Monitor):
|
||
+ """To collect the process sched info"""
|
||
+ _module = "PROCESS"
|
||
+ _purpose = "SCHED"
|
||
+ _option = "/proc/{}/sched"
|
||
+
|
||
+ def __init__(self, user=None):
|
||
+ Monitor.__init__(self, user)
|
||
+ self.__cmd = "cat"
|
||
+ self.__interval = 1
|
||
+ self.__applications = []
|
||
+ self.__pids = []
|
||
+
|
||
+ def _get(self, para=None):
|
||
+ output = ""
|
||
+ pids = []
|
||
+ if para is not None:
|
||
+ opts, _ = getopt.getopt(para.split(), None, ['interval=', 'app='])
|
||
+ for opt, val in opts:
|
||
+ if opt in '--interval':
|
||
+ if val.isdigit():
|
||
+ self.__interval = int(val)
|
||
+ else:
|
||
+ err = ValueError(
|
||
+ "Invalid parameter: {opt}={val}".format(
|
||
+ opt=opt, val=val))
|
||
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
|
||
+ inspect.stack()[0][3], str(err))
|
||
+ raise err
|
||
+ continue
|
||
+ elif opt in '--app':
|
||
+ if val is not None:
|
||
+ self.__applications = val.split(',')
|
||
+ else:
|
||
+ err = ValueError(
|
||
+ "{opt} parameter is none".format(
|
||
+ opt=opt))
|
||
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
|
||
+ inspect.stack()[0][3], str(err))
|
||
+ raise err
|
||
+
|
||
+ for app in self.__applications:
|
||
+ pid = subprocess.getoutput(
|
||
+ "ps -A | grep {} | awk '{{print $1}}'".format(app)).split()[0]
|
||
+ pids.append(pid)
|
||
+ self.__pids = pids
|
||
+
|
||
+ for pid in self.__pids:
|
||
+ out = subprocess.check_output(
|
||
+ "{cmd} {opt}".format(
|
||
+ cmd=self.__cmd,
|
||
+ opt=self._option.format(pid)).split())
|
||
+ output = output + "" + out.decode()
|
||
+ return output
|
||
+
|
||
+ def decode(self, info, para):
|
||
+ """
|
||
+ decode the result of the operation
|
||
+ :param info: content that needs to be decoded
|
||
+ :param para: command line argument
|
||
+ :returns ret: operation result
|
||
+ """
|
||
+
|
||
+ if para is None:
|
||
+ return info
|
||
+
|
||
+ start = 0
|
||
+ keys = []
|
||
+ ret = ""
|
||
+
|
||
+ opts, _ = getopt.getopt(para.split(), None, ['nic=', 'fields=', 'device='])
|
||
+ for opt, val in opts:
|
||
+ if opt in '--fields':
|
||
+ keys.append(val)
|
||
+ continue
|
||
+
|
||
+ pattern = re.compile(
|
||
+ r"(\w+)\ {1,}\:\ {1,}(\d+.\d+)",
|
||
+ re.I | re.UNICODE | re.MULTILINE)
|
||
+ search_obj = pattern.findall(info)
|
||
+ search_list = []
|
||
+ for obj in search_obj:
|
||
+ if obj[0][:3] == "nr_":
|
||
+ search_list.append(obj[0][3:])
|
||
+ else:
|
||
+ search_list.append(obj[0])
|
||
+ search_list.append(obj[1])
|
||
+ if len(search_obj) == 0:
|
||
+ err = LookupError("Fail to find data")
|
||
+ LOGGER.error("%s.%s: %s", self.__class__.__name__,
|
||
+ inspect.stack()[0][3], str(err))
|
||
+ raise err
|
||
+
|
||
+ while start <= len(self.__applications) * len(keys):
|
||
+ for key in keys:
|
||
+ ret = ret + " " + search_list[search_list.index(key, start)+1]
|
||
+ start = search_list.index(key, start) + 1
|
||
+ return ret
|
||
\ No newline at end of file
|
||
--
|
||
2.27.0
|
||
|