TST: use env variable instead of pytest option for testing ArrayManager (#40222)

Author: jorisvandenbossche

.github/workflows/ci.yml

@@ -150,23 +150,25 @@ jobs:
       uses: ./.github/actions/setup
 
     - name: Run tests
+      env:
+        PANDAS_DATA_MANAGER: array
       run: |
         source activate pandas-dev
-        pytest pandas/tests/frame/methods --array-manager
-        pytest pandas/tests/frame/test_constructors.py --array-manager
-        pytest pandas/tests/frame/constructors/ --array-manager
-        pytest pandas/tests/frame/test_reductions.py --array-manager
-        pytest pandas/tests/reductions/ --array-manager
-        pytest pandas/tests/generic/test_generic.py --array-manager
-        pytest pandas/tests/arithmetic/ --array-manager
-        pytest pandas/tests/groupby/ --array-manager
-        pytest pandas/tests/resample/ --array-manager
-        pytest pandas/tests/reshape/merge --array-manager
+        pytest pandas/tests/frame/methods
+        pytest pandas/tests/frame/test_constructors.py
+        pytest pandas/tests/frame/constructors/
+        pytest pandas/tests/frame/test_reductions.py
+        pytest pandas/tests/reductions/
+        pytest pandas/tests/generic/test_generic.py
+        pytest pandas/tests/arithmetic/
+        pytest pandas/tests/groupby/
+        pytest pandas/tests/resample/
+        pytest pandas/tests/reshape/merge
 
         # indexing subset (temporary since other tests don't pass yet)
-        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean --array-manager
-        pytest pandas/tests/frame/indexing/test_where.py --array-manager
-        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_multi_index --array-manager
-        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns --array-manager
-        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups --array-manager
-        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column --array-manager
+        pytest pandas/tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean
+        pytest pandas/tests/frame/indexing/test_where.py
+        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_multi_index
+        pytest pandas/tests/frame/indexing/test_setitem.py::TestDataFrameSetItem::test_setitem_listlike_indexer_duplicate_columns
+        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_astype_assignment_with_dups
+        pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column

pandas/conftest.py

@@ -100,19 +100,6 @@ def pytest_addoption(parser):
         action="store_true",
         help="Fail if a test is skipped for missing data file.",
     )
-    parser.addoption(
-        "--array-manager",
-        "--am",
-        action="store_true",
-        help="Use the experimental ArrayManager as default data manager.",
-    )
-
-
-def pytest_sessionstart(session):
-    # Note: we need to set the option here and not in pytest_runtest_setup below
-    # to ensure this is run before creating fixture data
-    if session.config.getoption("--array-manager"):
-        pd.options.mode.data_manager = "array"
 
 
 def pytest_runtest_setup(item):

pandas/core/config_init.py

@@ -9,6 +9,7 @@
 module is imported, register them here rather than in the module.
 
 """
+import os
 import warnings
 
 import pandas._config.config as cf
@@ -483,18 +484,32 @@ def use_inf_as_na_cb(key):
     cf.register_option(
         "use_inf_as_null", False, use_inf_as_null_doc, cb=use_inf_as_na_cb
     )
-    cf.register_option(
-        "data_manager",
-        "block",
-        "Internal data manager type",
-        validator=is_one_of_factory(["block", "array"]),
-    )
+
 
 cf.deprecate_option(
     "mode.use_inf_as_null", msg=use_inf_as_null_doc, rkey="mode.use_inf_as_na"
 )
 
 
+data_manager_doc = """
+: string
+    Internal data manager type; can be "block" or "array". Defaults to "block",
+    unless overridden by the 'PANDAS_DATA_MANAGER' environment variable (needs
+    to be set before pandas is imported).
+"""
+
+
+with cf.config_prefix("mode"):
+    cf.register_option(
+        "data_manager",
+        # Get the default from an environment variable, if set, otherwise defaults
+        # to "block". This environment variable can be set for testing.
+        os.environ.get("PANDAS_DATA_MANAGER", "block"),
+        data_manager_doc,
+        validator=is_one_of_factory(["block", "array"]),
+    )
+
+
 # user warnings
 chained_assignment = """
 : string

pandas/util/_test_decorators.py

@@ -35,6 +35,8 @@ def test_foo():
 import numpy as np
 import pytest
 
+from pandas._config import get_option
+
 from pandas.compat import (
     IS64,
     is_platform_windows,
@@ -285,16 +287,11 @@ def async_mark():
     return async_mark
 
 
-# Note: we are using a string as condition (and not for example
-# `get_option("mode.data_manager") == "array"`) because this needs to be
-# evaluated at test time (otherwise this boolean condition gets evaluated
-# at import time, when the pd.options.mode.data_manager has not yet been set)
-
 skip_array_manager_not_yet_implemented = pytest.mark.skipif(
-    "config.getvalue('--array-manager')", reason="JSON C code relies on Blocks"
+    get_option("mode.data_manager") == "array", reason="JSON C code relies on Blocks"
 )
 
 skip_array_manager_invalid_test = pytest.mark.skipif(
-    "config.getvalue('--array-manager')",
+    get_option("mode.data_manager") == "array",
     reason="Test that relies on BlockManager internals or specific behaviour",
 )