From cb08a3c1703e4e2ab4e4db336ce38d5798db190b Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Sun, 7 Feb 2021 16:27:06 +0000 Subject: [PATCH 1/4] Show installed packages when running tests. --- noxfile.py | 4 ++ tools/env_listing.py | 124 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100755 tools/env_listing.py diff --git a/noxfile.py b/noxfile.py index fc6175bdf0..34464059ae 100644 --- a/noxfile.py +++ b/noxfile.py @@ -165,6 +165,10 @@ def tests(session): cache_cartopy(session) session.install("--no-deps", "--editable", ".") + + # Always echo the env content for CI test failure debugging. + session.run("python", "tools/env_listing.py") + session.run( "python", "-m", diff --git a/tools/env_listing.py b/tools/env_listing.py new file mode 100755 index 0000000000..fe81387bdd --- /dev/null +++ b/tools/env_listing.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Produce a listing output of the calling Python environment. + +Combine "conda list" with a "pip list" to describe all installed packages. + +""" +from subprocess import check_output +import beautifultable as bt + + +def sanitise_lines(lines, header_line_hint=None): + """ + Split string by spaces, removing leading, trailing and repeated whitespace. + Remove header lines up to the last one containing 'header_line_hint'. + + """ + pkg_info = [] + for line in lines: + line = line.strip() + while '\t' in line: + line = line.replace('\t', ' ') + while ' ' in line: + ... + line = line.replace(' ', ' ') + line = line.split(' ') + if len(line) > 0 and line[0] not in ('', '#'): + pkg_info.append(line) + + pkg_keys = [pkg[0] for pkg in pkg_info] + if header_line_hint is not None: + header_inds = [ind for ind, key in enumerate(pkg_keys) + if header_line_hint in key] + if len(header_inds) > 0: + pkg_info = pkg_info[header_inds[-1] + 1:] + + result = {pkg[0]: pkg[1:] for pkg in pkg_info} + return result + + +def scan_env(): + """ + Get package listings from conda and pip. + + Return: + package_names, conda_info, pip_info + + The package names are sorted. + 'pip_info' is a dict : name --> version + 'conda_info' is a dict : name --> version, build, channel + Any package may be present in either dict, or both. + + """ + conda_list_bytes = check_output(['conda list'], shell=True) + conda_list_lines = conda_list_bytes.decode().split('\n') + pip_list_bytes = check_output(['pip list'], shell=True) + pip_list_lines = pip_list_bytes.decode().split('\n') + + conda_info = sanitise_lines(conda_list_lines) + pip_info = sanitise_lines(pip_list_lines, header_line_hint='---') + conda_keys = set(conda_info.keys()) + pip_keys = set([pkg[0] for pkg in pip_info]) + all_keys = sorted(set(conda_info.keys()) | set(pip_info.keys())) + + return all_keys, conda_info, pip_info + + +def make_package_table(package_names, conda_info, pip_info): + """ + Turn the package info returned from 'scan_env' into a printable table. + + """ + table = bt.BeautifulTable() + table.columns.header = [ + 'Package', 'source', 'version(s)', 'conda-version', 'conda-channel'] + + def version_summary(package, pip_info, conda_info): + """ + Extract + combine info from pip and conda about 'package'. + Return a list of 5 strings : package, source, version, build, channel. + + """ + pipver = pip_info[package][0] if package in pip_info else None + condaver = conda_info[package][0] if package in conda_info else None + columns = [package] + [''] * 4 + if condaver: + conda_extra_columns = conda_info[package][1:] + assert len(conda_extra_columns) == 2 + columns[3:] = conda_extra_columns + if pipver is not None and condaver is None: + source, version = "pip", pipver + elif pipver is None and condaver is not None: + source, version = "conda", condaver + else: + if (pipver == condaver): + source, version = "both", pipver + else: + source, version = '**CONFLICT**', f'pip={pipver} conda={condaver}' + if 'CONFLICT' not in source: + version = '= ' + version + columns[1:3] = [source, version] + return columns + + for key in package_names: + table.rows.append(version_summary(key, + pip_info=pip_info, + conda_info=conda_info)) + + # Pre-style the table output. + table.maxwidth = 9999 + table.columns.alignment = bt.ALIGN_LEFT + table.set_style(bt.STYLE_COMPACT) + return table # Ready to print + + +if __name__ == '__main__': + env_info = scan_env() + table = make_package_table(*env_info) + print(str(table)) From 25af9c274f57610d0291e3c6df579de0c5f4ee3e Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Sun, 7 Feb 2021 16:42:33 +0000 Subject: [PATCH 2/4] Explicitly install beautifultable (for now ?). --- noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/noxfile.py b/noxfile.py index 34464059ae..92a892a3c3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -167,6 +167,7 @@ def tests(session): session.install("--no-deps", "--editable", ".") # Always echo the env content for CI test failure debugging. + session.install("beautifultable") session.run("python", "tools/env_listing.py") session.run( From 99220bd6f8698d70a8ea28818b06ecf7deec5f7a Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Sun, 7 Feb 2021 17:45:49 +0000 Subject: [PATCH 3/4] Tolerance for unexpected conda list results. --- tools/env_listing.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/env_listing.py b/tools/env_listing.py index fe81387bdd..cffd6a73fa 100755 --- a/tools/env_listing.py +++ b/tools/env_listing.py @@ -90,6 +90,13 @@ def version_summary(package, pip_info, conda_info): columns = [package] + [''] * 4 if condaver: conda_extra_columns = conda_info[package][1:] + # Normally will get ", build, channel' + # But it seems we may need to allow for more or less. + while len(conda_extra_columns) < 2: + conda_extra_columns += [''] + if len(conda_extra_columns) > 2: + conda_extra_columns[1] = ' '.join(conda_extra_columns[1:]) + conda_extra_columns = conda_extra_columns[:2] assert len(conda_extra_columns) == 2 columns[3:] = conda_extra_columns if pipver is not None and condaver is None: From 3f9296b1114d1eff98c7f4c37d54feb700423ad5 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Sun, 7 Feb 2021 19:03:05 +0000 Subject: [PATCH 4/4] Remove hashbang to satisfy licence header check. --- tools/env_listing.py | 1 - 1 file changed, 1 deletion(-) mode change 100755 => 100644 tools/env_listing.py diff --git a/tools/env_listing.py b/tools/env_listing.py old mode 100755 new mode 100644 index cffd6a73fa..a54bb947b7 --- a/tools/env_listing.py +++ b/tools/env_listing.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright Iris contributors # # This file is part of Iris and is released under the LGPL license.