Merge pull request #827 from ComputationalCryoEM/develop

Develop ~~> Master v0.10.1 Updates

Author: garrettwrong

.github/workflows/daily_workflow.yml

@@ -0,0 +1,33 @@
+name: ASPIRE Python Daily Build CI
+
+on:
+  schedule:
+    -cron: '0 0 * * *'
+
+jobs:
+  dev_docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: develop
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.7'
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Build Sphinx docs
+        run: |
+          make clean
+          sphinx-apidoc -f -o ./source ../src -H Modules
+          make html
+        working-directory: ./docs
+      - name: Archive Sphinx docs
+        uses: actions/upload-artifact@v3
+        with:
+          name: sphinx-docs
+          path: docs
+          retention-days: 7

gallery/experiments/simulated_abinitio_pipeline.py

@@ -24,7 +24,7 @@
 from aspire.basis import FFBBasis2D, FFBBasis3D
 from aspire.classification import BFSReddyChatterjiAverager2D, RIRClass2D
 from aspire.denoising import DenoiserCov2D
-from aspire.noise import AnisotropicNoiseEstimator
+from aspire.noise import AnisotropicNoiseEstimator, CustomNoiseAdder
 from aspire.operators import FunctionFilter, RadialCTFFilter
 from aspire.reconstruction import MeanEstimator
 from aspire.source import Simulation
@@ -81,7 +81,7 @@ def noise_function(x, y):
     return (alpha * f1 + beta * f2) / 2.0
 
 
-custom_noise_filter = FunctionFilter(noise_function)
+custom_noise = CustomNoiseAdder(noise_filter=FunctionFilter(noise_function))
 
 logger.info("Initialize CTF filters.")
 # Create some CTF effects
@@ -104,7 +104,7 @@ def noise_function(x, y):
     L=v.resolution,
     n=num_imgs,
     vols=v,
-    noise_filter=custom_noise_filter,
+    noise_adder=custom_noise,
     unique_filters=ctf_filters,
     dtype=v.dtype,
 )

gallery/tutorials/basic_image_array.py

@@ -14,9 +14,8 @@
 from scipy import misc
 
 from aspire.image import Image
-from aspire.image.xform import NoiseAdder
-from aspire.noise import AnisotropicNoiseEstimator
-from aspire.operators import FunctionFilter, ScalarFilter
+from aspire.noise import AnisotropicNoiseEstimator, CustomNoiseAdder, WhiteNoiseAdder
+from aspire.operators import FunctionFilter
 from aspire.source import ArrayImageSource
 
 # %%
@@ -46,19 +45,18 @@
 # Construct the Image class by passing it an array of data.
 img = Image(stock_img)
 # Downsample (just to speeds things up)
-new_resolution = img.res // 4
+new_resolution = img.resolution // 4
 img = img.downsample(new_resolution)
 
 
 # We will begin processing by adding some noise.
 # We would like to create uniform noise for a 2d image with prescibed variance,
 noise_var = np.var(img.asnumpy()) * 5
-noise_filter = ScalarFilter(dim=2, value=noise_var)
 
-# Then create a NoiseAdder.
-noise = NoiseAdder(seed=123, noise_filter=noise_filter)
+# Then create a WhiteNoiseAdder.
+noise = WhiteNoiseAdder(var=noise_var, seed=123)
 
-# We can apply the NoiseAdder to our image data.
+# We can apply the WhiteNoiseAdder to our image data.
 img_with_noise = noise.forward(img)
 
 # We will plot the original and first noisy image,
@@ -83,7 +81,7 @@
 # with each image just being a copy of the data from ``img``.
 
 n_imgs = 128
-imgs_data = np.empty((n_imgs, img.res, img.res), dtype=np.float64)
+imgs_data = np.empty((n_imgs, img.resolution, img.resolution), dtype=np.float64)
 for i in range(n_imgs):
     imgs_data[i] = img[0]
 imgs = Image(imgs_data)
@@ -98,7 +96,7 @@ def noise_function(x, y):
 # We can create a custom filter from that function.
 f = FunctionFilter(noise_function)
 # And use the filter to add the noise to our stack of images.
-noise_adder = NoiseAdder(seed=123, noise_filter=f)
+noise_adder = CustomNoiseAdder(noise_filter=f, seed=123)
 imgs_with_noise = noise_adder.forward(imgs)
 
 # Let's see the first two noisy images.

gallery/tutorials/class_averaging.py

@@ -13,8 +13,7 @@
 
 from aspire.classification import RIRClass2D, TopClassSelector
 from aspire.image import Image
-from aspire.image.xform import NoiseAdder
-from aspire.operators import ScalarFilter
+from aspire.noise import WhiteNoiseAdder
 from aspire.source import ArrayImageSource  # Helpful hint if you want to BYO array.
 from aspire.utils import gaussian_2d
 
@@ -139,11 +138,8 @@
 var = np.var(src.images[:].asnumpy())
 noise_var = var * 2**4
 
-# We create a uniform noise to apply to the 2D images
-noise_filter = ScalarFilter(dim=2, value=noise_var)
-
-# Then create noise with the ``NoiseAdder`` class.
-noise = NoiseAdder(seed=123, noise_filter=noise_filter)
+# Then create noise with the ``WhiteNoiseAdder`` class.
+noise = WhiteNoiseAdder(var=noise_var, seed=123)
 
 # Add noise to the images by performing ``forward``
 noisy_im = noise.forward(src.images[:])

gallery/tutorials/configuration.py

@@ -54,7 +54,7 @@
 #     .. code-block:: yaml
 #
 #       logging:
-#         log_dir: /tmp/my_proj/logs
+#         log_dir: /tmp/my_proj/aspire_logs
 #
 # This directory must then be set before invoking any code.
 #
@@ -120,6 +120,60 @@
 # You can also resolve the config in code
 print(aspire.config.dump())
 
+# %%
+# Logging preferences
+# ---------
+# The "Logging" section contains options for controlling the verbosity and destination of log messages generated by ASPIRE.
+#
+# By default, the ``log_dir`` is a directory called ``logs`` within the current working directory from which ASPIRE was invoked, but
+# it can be changed to an absolute path (for instance, to go to the same folder as a custom configuration file stored elsewhere, such
+# as in the example above):
+#
+#    .. code-block:: yaml
+#
+#      logging:
+#          log_dir=/tmp/my_proj/aspire_logs
+#
+# ASPIRE directs output streams to both the console and to a log file during each session. The logging verbosity for both of these
+# can be specified individually via ``console_level`` and ``log_file_level``. These levels are passed through to Python's logging
+# system, so the labels are the same: ``"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"``. In general, it may make sense to have the log file
+# store more verbose output than the console. By default, only ``INFO`` and above level messages are printed to the terminal, while
+# detailed ``DEBUG`` messages are saved to disk:
+#
+#    .. code-block:: yaml
+#
+#      logging:
+#          console_level: INFO
+#          log_file_level: DEBUG
+#
+# The following modification would save all ``DEBUG`` messages to the log file, but only print ``ERROR`` and ``CRITICAL`` messages
+# in the terminal:
+#
+#    .. code-block:: yaml
+#
+#      logging:
+#          console_level: ERROR
+#          log_file_level: DEBUG
+#
+# By default, the filename of the log file is ``aspire-{%Y-%m-%dT%H-%M-%S.%f}.log``. The prefix before the timestamp can be customized
+# via the ``log_prefix`` option.
+#
+# More details on logging behavior in Python can be found in the `Python logging HOWTO`_.
+
+# %%
+# Advanced logging options
+# ---------
+# Users with sophisticated knowledge of Python's ``logging`` library are able to directly work with the ``logging.conf`` file stored in
+# ``src/aspire`` in the source.
+#
+# .. literalinclude:: ../../../src/aspire/logging.conf
+#
+# This file exposes options that ASPIRE's ``config.yaml`` does not, for example the precise formatting options for log messages. However,
+# values from ASPIRE's configuration are passed through this file, so caution is needed when modifying it.
+
+
+# %%
+# .. _Python logging HOWTO: https://docs.python.org/3/howto/logging.html
 
 # %%
 # .. _confuse library: https://confuse.readthedocs.io/en/latest/index.html

gallery/tutorials/cov2d_simulation.py

@@ -16,7 +16,8 @@
 
 from aspire.basis import FFBBasis2D
 from aspire.covariance import RotCov2D
-from aspire.operators import RadialCTFFilter, ScalarFilter
+from aspire.noise import WhiteNoiseAdder
+from aspire.operators import RadialCTFFilter
 from aspire.source.simulation import Simulation
 from aspire.utils import anorm
 from aspire.volume import Volume
@@ -52,7 +53,7 @@
 # and initial noise inside the Simulation class.
 
 noise_var = 1.3957e-4
-noise_filter = ScalarFilter(dim=2, value=noise_var)
+noise_adder = WhiteNoiseAdder(var=noise_var)
 
 # %%
 # Specify the CTF Parameters
@@ -98,7 +99,7 @@
     offsets=0.0,
     amplitudes=1.0,
     dtype=dtype,
-    noise_filter=noise_filter,
+    noise_adder=noise_adder,
 )
 
 

gallery/tutorials/generating_volume_projections.py

@@ -12,7 +12,7 @@
 
 import numpy as np
 
-from aspire.operators import ScalarFilter
+from aspire.noise import WhiteNoiseAdder
 from aspire.source.simulation import Simulation
 from aspire.utils import Rotation
 from aspire.volume import Volume
@@ -56,8 +56,8 @@
 
 noise_variance = 1e-10  # Normally this would be derived from a desired SNR.
 
-# Then create a constant filter based on that variance, which is passed to Simulation
-white_noise_filter = ScalarFilter(dim=2, value=noise_variance)
+# Then create a CustomNoiseAdder based on that variance, which is passed to Simulation.
+white_noise_adder = WhiteNoiseAdder(var=noise_variance)
 
 
 # %%
@@ -79,7 +79,7 @@
     amplitudes=amplitudes,  # amplification ( 1 is identity)
     seed=12345,  # RNG seed for reproducibility
     dtype=v.dtype,  # match our datatype to the Volume.
-    noise_filter=white_noise_filter,  # optionally prescribe noise
+    noise_adder=white_noise_adder,  # optionally prescribe noise
 )
 
 # %%

gallery/tutorials/lecture_feature_demo.py

@@ -38,8 +38,13 @@
 import numpy as np
 
 from aspire.abinitio import CLSyncVoting
-from aspire.noise import AnisotropicNoiseEstimator, WhiteNoiseEstimator
-from aspire.operators import FunctionFilter, RadialCTFFilter, ScalarFilter
+from aspire.noise import (
+    AnisotropicNoiseEstimator,
+    CustomNoiseAdder,
+    WhiteNoiseAdder,
+    WhiteNoiseEstimator,
+)
+from aspire.operators import FunctionFilter, RadialCTFFilter
 from aspire.source import RelionSource, Simulation
 from aspire.utils import (
     Rotation,
@@ -239,10 +244,10 @@
 noise_variance = 100.0 * var
 logger.info(f"noise var {noise_variance}")
 
-# Then create a constant filter based on that variance
-white_noise_filter = ScalarFilter(dim=2, value=noise_variance)
+# Then create a CustomNoiseAdder based on that variance.
+white_noise_adder = WhiteNoiseAdder(var=noise_variance)
 # We can create a similar simulation with this additional noise_filter argument:
-sim3 = Simulation(L=v2.resolution, n=num_imgs, vols=v2, noise_filter=white_noise_filter)
+sim3 = Simulation(L=v2.resolution, n=num_imgs, vols=v2, noise_adder=white_noise_adder)
 sim3.images[:10].show()
 # These should be rather noisy now ...
 
@@ -314,7 +319,7 @@
 
 # Create another Simulation source to tinker with.
 sim_wht = Simulation(
-    L=v2.resolution, n=num_imgs, vols=v2, noise_filter=white_noise_filter
+    L=v2.resolution, n=num_imgs, vols=v2, noise_adder=white_noise_adder
 )
 
 # Estimate the white noise.
@@ -337,12 +342,10 @@ def noise_function(x, y):
 # In python, functions are first class objects.
 # We take advantage of that to pass this function around as a variable.
 # It will be evaluated later...
-custom_noise_filter = FunctionFilter(noise_function)
+custom_noise = CustomNoiseAdder(noise_filter=FunctionFilter(noise_function))
 
 # Create yet another Simulation source to tinker with.
-sim4 = Simulation(
-    L=v2.resolution, n=num_imgs, vols=v2, noise_filter=custom_noise_filter
-)
+sim4 = Simulation(L=v2.resolution, n=num_imgs, vols=v2, noise_adder=custom_noise)
 sim4.images[:10].show()
 
 # %%
@@ -446,7 +449,7 @@ def noise_function(x, y):
     n=num_imgs,
     vols=v2,
     unique_filters=filters,
-    noise_filter=custom_noise_filter,
+    noise_adder=custom_noise,
 )
 sim6.images[:10].show()
 

gallery/tutorials/pipeline_demo.py

@@ -72,12 +72,13 @@ def download(url, save_path, chunk_size=1024 * 1024):
 # Gaussian white noise and ``RadialCTFFilter`` to generate a set of CTF filters with various defocus values.
 
 # Create noise and CTF filters
-from aspire.operators import RadialCTFFilter, ScalarFilter
+from aspire.noise import WhiteNoiseAdder
+from aspire.operators import RadialCTFFilter
 
 # Gaussian noise filter.
-# Note, the value supplied to the ``ScalarFilter``, chosen based on other parameters
+# Note, the value supplied to the ``WhiteNoiseAdder``, chosen based on other parameters
 # for this quick tutorial, can be changed to adjust the power of the noise.
-noise_filter = ScalarFilter(value=1e-5)
+noise_adder = WhiteNoiseAdder(var=1e-5)
 
 # Radial CTF Filter
 defocus_min = 15000  # unit is angstroms
@@ -108,7 +109,7 @@ def download(url, save_path, chunk_size=1024 * 1024):
     n=n_imgs,  # number of projections
     vols=vol,  # volume source
     offsets=np.zeros((n_imgs, 2)),  # Default: images are randomly shifted
-    noise_filter=noise_filter,
+    noise_adder=noise_adder,
     unique_filters=ctf_filters,
 )
 

gallery/tutorials/preprocess_imgs_sim.py

@@ -11,8 +11,8 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-from aspire.noise import WhiteNoiseEstimator
-from aspire.operators import RadialCTFFilter, ScalarFilter
+from aspire.noise import WhiteNoiseAdder, WhiteNoiseEstimator
+from aspire.operators import RadialCTFFilter
 from aspire.source.simulation import Simulation
 from aspire.volume import Volume
 
@@ -32,7 +32,7 @@
 
 # Set the noise variance and build the noise filter
 noise_variance = 4e-1
-noise_filter = ScalarFilter(dim=2, value=noise_variance)
+noise_adder = WhiteNoiseAdder(var=noise_variance)
 
 # Specify the CTF parameters not used for this example
 # but necessary for initializing the simulation object
@@ -71,7 +71,7 @@
     n=num_imgs,
     vols=vols,
     unique_filters=ctf_filters,
-    noise_filter=noise_filter,
+    noise_adder=noise_adder,
 )
 
 # %%