From fadb6489cfbc14c67ebcd9b34a032ad574a3d529 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Wed, 13 Jul 2022 13:05:46 -0600 Subject: [PATCH 4/8] Resource leak cleanup (#1556) Reference:https://github.com/canonical/cloud-init/commit/9cbd94dd57112083856ead0e0ff724e9d1c1f714 Conflict:test file. Add tox target for tracing for resource leaks, fix some leaks --- cloudinit/analyze/__main__.py | 13 +++++++++++++ cloudinit/analyze/tests/test_boot.py | 24 +++++++++++++----------- cloudinit/analyze/tests/test_dump.py | 4 ++-- cloudinit/cmd/cloud_id.py | 3 ++- tests/unittests/test_util.py | 3 ++- tox.ini | 9 +++++++++ 6 files changed, 41 insertions(+), 15 deletions(-) diff --git a/cloudinit/analyze/__main__.py b/cloudinit/analyze/__main__.py index 99e5c20..4ec609c 100644 --- a/cloudinit/analyze/__main__.py +++ b/cloudinit/analyze/__main__.py @@ -8,6 +8,7 @@ import sys from cloudinit.util import json_dumps from datetime import datetime +from typing import IO from . import dump from . import show @@ -136,6 +137,7 @@ def analyze_boot(name, args): } outfh.write(status_map[status_code].format(**kwargs)) + clean_io(infh, outfh) return status_code @@ -161,6 +163,7 @@ def analyze_blame(name, args): outfh.write('\n'.join(srecs) + '\n') outfh.write('\n') outfh.write('%d boot records analyzed\n' % (idx + 1)) + clean_io(infh, outfh) def analyze_show(name, args): @@ -193,12 +196,14 @@ def analyze_show(name, args): 'character.\n\n') outfh.write('\n'.join(record) + '\n') outfh.write('%d boot records analyzed\n' % (idx + 1)) + clean_io(infh, outfh) def analyze_dump(name, args): """Dump cloud-init events in json format""" (infh, outfh) = configure_io(args) outfh.write(json_dumps(_get_events(infh)) + '\n') + clean_io(infh, outfh) def _get_events(infile): @@ -232,6 +237,14 @@ def configure_io(args): return (infh, outfh) +def clean_io(*file_handles: IO) -> None: + """close filehandles""" + for file_handle in file_handles: + if file_handle in (sys.stdin, sys.stdout): + continue + file_handle.close() + + if __name__ == '__main__': parser = get_parser() args = parser.parse_args() diff --git a/cloudinit/analyze/tests/test_boot.py b/cloudinit/analyze/tests/test_boot.py index f69423c..6676676 100644 --- a/cloudinit/analyze/tests/test_boot.py +++ b/cloudinit/analyze/tests/test_boot.py @@ -117,17 +117,19 @@ class TestAnalyzeBoot(CiTestCase): analyze_boot(name_default, args) # now args have been tested, go into outfile and make sure error # message is in the outfile - outfh = open(args.outfile, 'r') - data = outfh.read() - err_string = 'Your Linux distro or container does not support this ' \ - 'functionality.\nYou must be running a Kernel ' \ - 'Telemetry supported distro.\nPlease check ' \ - 'https://cloudinit.readthedocs.io/en/latest/topics' \ - '/analyze.html for more information on supported ' \ - 'distros.\n' - - self.remove_dummy_file(path, log_path) - self.assertEqual(err_string, data) + with open(args.outfile, "r") as outfh: + data = outfh.read() + err_string = ( + "Your Linux distro or container does not support this " + "functionality.\nYou must be running a Kernel " + "Telemetry supported distro.\nPlease check " + "https://cloudinit.readthedocs.io/en/latest/topics" + "/analyze.html for more information on supported " + "distros.\n" + ) + + self.remove_dummy_file(path, log_path) + self.assertEqual(err_string, data) @mock.patch("cloudinit.util.is_container", return_value=True) @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py index dac1efb..27db1b1 100644 --- a/cloudinit/analyze/tests/test_dump.py +++ b/cloudinit/analyze/tests/test_dump.py @@ -184,8 +184,8 @@ class TestDumpEvents(CiTestCase): tmpfile = self.tmp_path('logfile') write_file(tmpfile, SAMPLE_LOGS) m_parse_from_date.return_value = 1472594005.972 - - events, data = dump_events(cisource=open(tmpfile)) + with open(tmpfile) as file: + events, data = dump_events(cisource=file) year = datetime.now().year dt1 = datetime.strptime( 'Nov 03 06:51:06.074410 %d' % year, '%b %d %H:%M:%S.%f %Y') diff --git a/cloudinit/cmd/cloud_id.py b/cloudinit/cmd/cloud_id.py index 9760892..985f9a2 100755 --- a/cloudinit/cmd/cloud_id.py +++ b/cloudinit/cmd/cloud_id.py @@ -53,7 +53,8 @@ def handle_args(name, args): @return: 0 on success, 1 otherwise. """ try: - instance_data = json.load(open(args.instance_data)) + with open(args.instance_data) as file: + instance_data = json.load(file) except IOError: return error( "File not found '%s'. Provide a path to instance data json file" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 0b01337..1185487 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -560,7 +560,8 @@ class TestMultiLog(helpers.FilesystemMockingTestCase): self._createConsole(self.root) logged_string = 'something very important' util.multi_log(logged_string) - self.assertEqual(logged_string, open('/dev/console').read()) + with open("/dev/console") as f: + self.assertEqual(logged_string, f.read()) def test_logs_dont_go_to_stdout_if_console_exists(self): self._createConsole(self.root) diff --git a/tox.ini b/tox.ini index 874d3f2..5360067 100644 --- a/tox.ini +++ b/tox.ini @@ -60,6 +60,15 @@ commands = {envpython} -m sphinx {posargs:doc/rtd doc/rtd_html} doc8 doc/rtd +#commands = {envpython} -X tracemalloc=40 -Werror::ResourceWarning:cloudinit -m pytest \ +[testenv:py3-leak] +deps = {[testenv:py3]deps} +commands = {envpython} -X tracemalloc=40 -Wall -m pytest \ + --durations 10 \ + {posargs:--cov=cloudinit --cov-branch \ + tests/unittests} + + [xenial-shared-deps] # The version of pytest in xenial doesn't work with Python 3.8, so we define # two xenial environments: [testenv:xenial] runs the tests with exactly the -- 2.40.0