KroneckerNormal has compilation problems with PyMC v3 on Windows #5253
Description
Description of your problem
test_distributions_random.py::TestScalarParameterSamples::test_kronecker_normal is failing the Windows tests on the v3 branch with compilation error.
Please provide the full traceback.
Complete error traceback
================================== FAILURES ===================================
______________ TestScalarParameterSamples.test_kronecker_normal _______________
self = <pymc3.tests.test_distributions_random.TestScalarParameterSamples object at 0x000001ED17E2D1C0>
def test_kronecker_normal(self):
def ref_rand(size, mu, covs, sigma):
cov = pm.math.kronecker(covs[0], covs[1]).eval()
cov += sigma ** 2 * np.identity(cov.shape[0])
return st.multivariate_normal.rvs(mean=mu, cov=cov, size=size)
def ref_rand_chol(size, mu, chols, sigma):
covs = [np.dot(chol, chol.T) for chol in chols]
return ref_rand(size, mu, covs, sigma)
def ref_rand_evd(size, mu, evds, sigma):
covs = []
for eigs, Q in evds:
covs.append(np.dot(Q, np.dot(np.diag(eigs), Q.T)))
return ref_rand(size, mu, covs, sigma)
sizes = [2, 3]
sigmas = [0, 1]
for n, sigma in zip(sizes, sigmas):
N = n ** 2
covs = [RandomPdMatrix(n), RandomPdMatrix(n)]
chols = list(map(np.linalg.cholesky, covs))
evds = list(map(np.linalg.eigh, covs))
dom = Domain([np.random.randn(N) * 0.1], edges=(None, None), shape=N)
mu = Domain([np.random.randn(N) * 0.1], edges=(None, None), shape=N)
std_args = {"mu": mu}
cov_args = {"covs": covs}
chol_args = {"chols": chols}
evd_args = {"evds": evds}
if sigma is not None and sigma != 0:
std_args["sigma"] = Domain([sigma], edges=(None, None))
else:
for args in [cov_args, chol_args, evd_args]:
args["sigma"] = sigma
> pymc3_random(
pm.KroneckerNormal,
std_args,
valuedomain=dom,
ref_rand=ref_rand,
extra_args=cov_args,
model_args=cov_args,
)
pymc3\tests\test_distributions_random.py:973:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
pymc3\tests\test_distributions_random.py:77: in pymc3_random
model = build_model(dist, valuedomain, paramdomains, extra_args)
pymc3\tests\test_distributions.py:224: in build_model
distfam("value", shape=valuedomain.shape, transform=None, **vals)
pymc3\distributions\distribution.py:122: in __new__
return model.Var(name, dist, data, total_size, dims=dims)
pymc3\model.py:1138: in Var
var = FreeRV(name=name, distribution=dist, total_size=total_size, model=self)
pymc3\model.py:1671: in __init__
self.logp_elemwiset = distribution.logp(self)
pymc3\distributions\multivariate.py:2081: in logp
quad, logdet = self._quaddist(value)
pymc3\distributions\multivariate.py:2062: in _quaddist
quad = tt.batched_dot(sqrt_quad.T, sqrt_quad.T)
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\tensor\basic.py:3771: in batched_dot
return theano.tensor.blas.BatchedDot()(a, b)
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\graph\op.py:253: in __call__
compute_test_value(node)
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\graph\op.py:126: in compute_test_value
thunk = node.op.make_thunk(node, storage_map, compute_map, no_recycling=[])
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\graph\op.py:634: in make_thunk
return self.make_c_thunk(node, storage_map, compute_map, no_recycling)
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\graph\op.py:600: in make_c_thunk
outputs = cl.make_thunk(
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\link\c\basic.py:1203: in make_thunk
cthunk, module, in_storage, out_storage, error_storage = self.__compile__(
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\link\c\basic.py:1138: in __compile__
thunk, module = self.cthunk_factory(
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\link\c\basic.py:1634: in cthunk_factory
module = get_module_cache().module_from_key(key=key, lnk=self)
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\link\c\cmodule.py:1191: in module_from_key
module = lnk.compile_cmodule(location)
C:\Miniconda\envs\pymc3-dev-py38\lib\site-packages\theano\link\c\basic.py:1543: in compile_cmodule
module = c_compiler.compile_str(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
module_name = 'm0525bfe4c83e8df7d237d165dfa3fa0c87d2c4cd4852014d3b9ac685f2f7f894'
src_code = '#include <Python.h>\n#include <iostream>\n#include "theano_mod_helper.h"\n#include <math.h>\n#include <numpy/arrayobj...d3b9ac685f2f7f894(void) {\n import_array();\n \n PyObject *m = PyModule_Create(&moduledef);\n return m;\n}\n'
location = 'C:\\Users\\runneradmin\\AppData\\Local\\Theano\\compiledir_Windows-10-10.0.17763-SP0-Intel64_Family_6_Model_85_Stepping_4_GenuineIntel-3.8.12-64\\tmpxkpkw1y4'
include_dirs = ['C:\\Miniconda\\envs\\pymc3-dev-py38\\lib\\site-packages\\numpy\\core\\include', 'C:\\Miniconda\\envs\\pymc3-dev-py38\\include', 'C:\\Miniconda\\envs\\pymc3-dev-py38\\lib\\site-packages\\theano\\link\\c\\c_code']
lib_dirs = ['C:\\Miniconda\\envs\\pymc3-dev-py38\\libs', 'C:\\Miniconda\\envs\\pymc3-dev-py38']
libs = ['python38']
preargs = ['-O3', '-fno-math-errno', '-Wno-unused-label', '-Wno-unused-variable', '-Wno-write-strings', '-march=core2', ...]
py_module = True, hide_symbols = True
@staticmethod
def compile_str(
module_name,
src_code,
location=None,
include_dirs=None,
lib_dirs=None,
libs=None,
preargs=None,
py_module=True,
hide_symbols=True,
):
"""
Parameters
----------
module_name : str
This has been embedded in the src_code.
src_code
A complete c or c++ source listing for the module.
location
A pre-existing filesystem directory where the cpp file and .so will
be written.
include_dirs
A list of include directory names (each gets prefixed with -I).
lib_dirs
A list of library search path directory names (each gets prefixed
with -L).
libs
A list of libraries to link with (each gets prefixed with -l).
preargs
A list of extra compiler arguments.
py_module
If False, compile to a shared library, but do not import it as a
Python module.
hide_symbols
If True (the default) all symbols will be hidden from the library
symbol table (which means that other objects can't use them).
Returns
-------
object
Dynamically-imported python module of the compiled code (unless
py_module is False, in that case returns None).
"""
# TODO: Do not do the dlimport in this function
if not config.cxx:
raise MissingGXX("g++ not available! We can't compile c code.")
if include_dirs is None:
include_dirs = []
if lib_dirs is None:
lib_dirs = []
if libs is None:
libs = []
if preargs is None:
preargs = []
# Remove empty string directory
include_dirs = [d for d in include_dirs if d]
lib_dirs = [d for d in lib_dirs if d]
include_dirs = include_dirs + std_include_dirs()
libs = libs + std_libs()
lib_dirs = lib_dirs + std_lib_dirs()
cppfilename = os.path.join(location, "mod.cpp")
with open(cppfilename, "w") as cppfile:
_logger.debug(f"Writing module C++ code to {cppfilename}")
cppfile.write(src_code)
# Avoid gcc warning "no newline at end of file".
if not src_code.endswith("\n"):
cppfile.write("\n")
if platform.python_implementation() == "PyPy":
suffix = "." + get_lib_extension()
dist_suffix = distutils.sysconfig.get_config_var("SO")
if dist_suffix is not None and dist_suffix != "":
suffix = dist_suffix
filepath = f"{module_name}{suffix}"
else:
filepath = f"{module_name}.{get_lib_extension()}"
lib_filename = os.path.join(location, filepath)
_logger.debug(f"Generating shared lib {lib_filename}")
cmd = [config.cxx, get_gcc_shared_library_arg(), "-g"]
if config.cmodule__remove_gxx_opt:
cmd.extend(p for p in preargs if not p.startswith("-O"))
else:
cmd.extend(preargs)
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
cmd.extend([f"-I{path_wrapper}{idir}{path_wrapper}" for idir in include_dirs])
cmd.extend([f"-L{path_wrapper}{ldir}{path_wrapper}" for ldir in lib_dirs])
if hide_symbols and sys.platform != "win32":
# This has been available since gcc 4.0 so we suppose it
# is always available. We pass it here since it
# significantly reduces the size of the symbol table for
# the objects we want to share. This in turns leads to
# improved loading times on most platforms (win32 is
# different, as usual).
cmd.append("-fvisibility=hidden")
cmd.extend(["-o", f"{path_wrapper}{lib_filename}{path_wrapper}"])
cmd.append(f"{path_wrapper}{cppfilename}{path_wrapper}")
cmd.extend([f"-l{l}" for l in libs])
# print >> sys.stderr, 'COMPILING W CMD', cmd
_logger.debug(f"Running cmd: {' '.join(cmd)}")
def print_command_line_error():
# Print command line when a problem occurred.
print(
("Problem occurred during compilation with the " "command line below:"),
file=sys.stderr,
)
print(" ".join(cmd), file=sys.stderr)
try:
p_out = output_subprocess_Popen(cmd)
compile_stderr = p_out[1].decode()
except Exception:
# An exception can occur e.g. if `g++` is not found.
print_command_line_error()
raise
status = p_out[2]
if status:
tf = tempfile.NamedTemporaryFile(
mode="w", prefix="theano_compilation_error_", delete=False
)
# gcc put its messages to stderr, so we add ours now
tf.write("===============================\n")
for i, l in enumerate(src_code.split("\n")):
tf.write(f"{i + 1}\t{l}\n")
tf.write("===============================\n")
tf.write(
"Problem occurred during compilation with the " "command line below:\n"
)
tf.write(" ".join(cmd))
# Print errors just below the command line.
tf.write(compile_stderr)
tf.close()
print("\nYou can find the C code in this temporary file: " + tf.name)
not_found_libraries = re.findall('-l["."-_a-zA-Z0-9]*', compile_stderr)
for nf_lib in not_found_libraries:
print("library " + nf_lib[2:] + " is not found.")
if re.search('-lPYTHON["."0-9]*', nf_lib, re.IGNORECASE):
py_string = re.search(
'-lpython["."0-9]*', nf_lib, re.IGNORECASE
).group()[8:]
if py_string != "":
print(
"Check if package python-dev "
+ py_string
+ " or python-devel "
+ py_string
+ " is installed."
)
else:
print(
"Check if package python-dev or python-devel is installed."
)
# We replace '\n' by '. ' in the error message because when Python
# prints the exception, having '\n' in the text makes it more
# difficult to read.
compile_stderr = compile_stderr.replace("\n", ". ")
> raise Exception(
f"Compilation failed (return status={status}): {compile_stderr}"
E Exception: ("Compilation failed (return status=1): C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\ccGjxZa6.o: In function `run':\r. C:/Users/runneradmin/AppData/Local/Theano/compiledir_Windows-10-10.0.17763-SP0-Intel64_Family_6_Model_85_Stepping_4_GenuineIntel-3.8.12-64/tmpxkpkw1y4/mod.cpp:1612: undefined reference to `dgemm_'\r. C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\ccGjxZa6.o:mod.cpp:(.rdata$.refptr.dgemm_[.refptr.dgemm_]+0x0): undefined reference to `dgemm_'\r. collect2.exe: error: ld returned 1 exit status\r. ", 'FunctionGraph(BatchedDot(<TensorType(float64, row)>, <TensorType(float64, row)>))')Versions and main components
- PyMC/PyMC3 Version:
v3 - Aesara/Theano Version:
1.1.2 - Python Version: 3.8.12
- Operating system: Windows