From 3fde650c2a868272d374d6f3e8c66655e75bc6b3 Mon Sep 17 00:00:00 2001 From: Fulton Wang Date: Mon, 19 Dec 2022 12:03:43 -0800 Subject: [PATCH] update tracin influence API (#1072) Summary: Pull Request resolved: https://github.com/pytorch/captum/pull/1072 This diff changes the API for implementations of `TracInCPBase` as discussed in https://fb.quip.com/JbpnAiWluZmI. In particular, the arguments representing test data of the `influence` method are changed from `inputs: Tuple, targets: Optional[Tensor]` to `inputs: Union[Tuple[Any], DataLoader]`, which is either a single batch, or a dataloader yielding batches. In both cases, `model(*batch)` is assumed to produce the predictions for a batch, and `batch[-1]` is assumed to be the labels for a batch. This is the same format assumed of the batches yielded by `train_dataloader`. We make this change for 2 reasons - it unifies the assumptions made of the test data and the assumptions made of the training data - for some implementations, we want to allow the test data to be represented by a dataloader. with the old API, there was no clean way to allow both a single as well as a dataloader to be passed in, since a batch required 2 arguments, but a dataloader only requires 1. For now, all implementations only allow `inputs` to be a tuple (and not a dataloader). This is okay due to inheritance rules. Later on, we will allow some implementations (i.e. `TracInCP`) to accept a dataloader as `inputs`. Other changes: - changes to make documentation. for example, documentation in `TracInCPBase.influence` now refers to the "test dataset" instead of test batch. - the `unpack_inputs` argument is no longer needed for the `influence` methods, and is removed - the usage of `influence` in all the tests is changed to match new API. - signature of helper methods `_influence_batch_tracincp` and `_influence_batch_tracincp_fast` are changed to match new representation of batches. Reviewed By: cyrjano Differential Revision: D41324297 fbshipit-source-id: 827350795bf2e5c6c1fab2e5ef8f2db1473dfe3d --- captum/influence/_core/influence.py | 8 +- captum/influence/_core/tracincp.py | 461 ++++++++--------- .../_core/tracincp_fast_rand_proj.py | 470 ++++++++---------- captum/influence/_utils/common.py | 14 +- tests/influence/_core/test_dataloader.py | 7 +- .../test_tracin_intermediate_quantities.py | 21 +- .../_core/test_tracin_k_most_influential.py | 8 +- .../influence/_core/test_tracin_regression.py | 28 +- .../_core/test_tracin_self_influence.py | 7 +- .../_core/test_tracin_show_progress.py | 9 +- .../influence/_core/test_tracin_validation.py | 2 +- tests/influence/_core/test_tracin_xor.py | 6 +- tests/influence/_utils/common.py | 12 +- tutorials/TracInCP_Tutorial.ipynb | 49 +- 14 files changed, 496 insertions(+), 606 deletions(-) diff --git a/captum/influence/_core/influence.py b/captum/influence/_core/influence.py index f8ef1eb882..553ab38abb 100644 --- a/captum/influence/_core/influence.py +++ b/captum/influence/_core/influence.py @@ -12,13 +12,11 @@ class DataInfluence(ABC): An abstract class to define model data influence skeleton. """ - def __init_( - self, model: Module, influence_src_dataset: Dataset, **kwargs: Any - ) -> None: + def __init_(self, model: Module, train_dataset: Dataset, **kwargs: Any) -> None: r""" Args: model (torch.nn.Module): An instance of pytorch model. - influence_src_dataset (torch.utils.data.Dataset): PyTorch Dataset that is + train_dataset (torch.utils.data.Dataset): PyTorch Dataset that is used to create a PyTorch Dataloader to iterate over the dataset and its labels. This is the dataset for which we will be seeking for influential instances. In most cases this is the training dataset. @@ -26,7 +24,7 @@ def __init_( implementation of `DataInfluence` abstract class. """ self.model = model - self.influence_src_dataset = influence_src_dataset + self.train_dataset = train_dataset @abstractmethod def influence(self, inputs: Any = None, **kwargs: Any) -> Any: diff --git a/captum/influence/_core/tracincp.py b/captum/influence/_core/tracincp.py index 02c388378c..f3119e8443 100644 --- a/captum/influence/_core/tracincp.py +++ b/captum/influence/_core/tracincp.py @@ -18,7 +18,7 @@ import torch from captum._utils.av import AV -from captum._utils.common import _format_inputs, _get_module_from_name, _parse_version +from captum._utils.common import _get_module_from_name, _parse_version from captum._utils.gradient import ( _compute_jacobian_wrt_params, _compute_jacobian_wrt_params_with_sample_wise_trick, @@ -74,13 +74,13 @@ class KMostInfluentialResults(NamedTuple): This namedtuple stores the results of using the `influence` method. This method is implemented by all subclasses of `TracInCPBase` to calculate proponents / opponents. The `indices` field stores the indices of the - proponents / opponents for each example in the test batch. For example, if finding - opponents, `indices[i][j]` stores the index in the training data of the example - with the `j`-th highest influence score on the `i`-th example in the test batch. - Similarly, the `influence_scores` field stores the actual influence scores, so that - `influence_scores[i][j]` is the influence score of example `indices[i][j]` in the - training data on example `i` of the test batch. Please see `TracInCPBase.influence` - for more details. + proponents / opponents for each example in the test dataset. For example, if + finding opponents, `indices[i][j]` stores the index in the training data of the + example with the `j`-th highest influence score on the `i`-th example in the test + dataset. Similarly, the `influence_scores` field stores the actual influence scores, + so that `influence_scores[i][j]` is the influence score of example `indices[i][j]` + in the training data on example `i` of the test dataset. Please see + `TracInCPBase.influence` for more details. """ indices: Tensor @@ -111,9 +111,8 @@ def __init__( model (torch.nn.Module): An instance of pytorch model. This model should define all of its layers as attributes of the model. train_dataset (torch.utils.data.Dataset or torch.utils.data.DataLoader): - In the `influence` method, we either compute the influence score of - training examples on examples in a test batch, or self influence - scores for those training examples, depending on which mode is used. + In the `influence` method, we compute the influence score of + training examples on examples in a test batch. This argument represents the training dataset containing those training examples. In order to compute those influence scores, we will create a Pytorch DataLoader yielding batches of training @@ -213,24 +212,24 @@ def __init__( @abstractmethod def self_influence( self, - inputs_dataset: Optional[Union[Tuple[Any, ...], DataLoader]] = None, + inputs: Optional[Union[Tuple[Any, ...], DataLoader]] = None, show_progress: bool = False, ) -> Tensor: """ - If `inputs_dataset` is not specified calculates the self influence + If `inputs` is not specified calculates the self influence scores for the training dataset `train_dataset`. Otherwise, computes - self influence scores for the examples in `inputs_dataset`, + self influence scores for the examples in `inputs`, which is either a single batch or a Pytorch `DataLoader` that yields batches. Therefore, in this case, the computed self influence scores are *not* for the examples in training dataset `train_dataset`. - Note that if `inputs_dataset` is a single batch, this - will call `model` on that single batch, and if `inputs_dataset` yields + Note that if `inputs` is a single batch, this + will call `model` on that single batch, and if `inputs` yields batches, this will call `model` on each batch that is yielded. Therefore, please ensure that for both cases, the batch(es) that `model` is called with are not too large, so that there will not be an out-of-memory error. Args: - inputs_dataset (tuple or DataLoader, optional): This specifies the + inputs (tuple or DataLoader, optional): This specifies the dataset for which self influence scores will be computed. Either a single tuple of any, or a `DataLoader`, where each batch yielded is a tuple of type any. In either case, the tuple @@ -246,15 +245,15 @@ def self_influence( above assumptions. Default: None. show_progress (bool, optional): Computation of self influence scores can - take a long time if `inputs_dataset` represents many examples. If + take a long time if `inputs` represents many examples. If `show_progress` is true, the progress of this computation will be displayed. In more detail, this computation will iterate over all checkpoints (provided as the `checkpoints` initialization argument) in an outer loop, and iterate over all batches that - `inputs_dataset` represents in an inner loop. Therefore, the + `inputs` represents in an inner loop. Therefore, the total number of (checkpoint, batch) combinations that need to be iterated over is - (# of checkpoints x # of batches that `inputs_dataset` represents). + (# of checkpoints x # of batches that `inputs` represents). If `show_progress` is True, the total progress of both the outer iteration over checkpoints and the inner iteration over batches is displayed. It will try to use tqdm if available for advanced @@ -264,7 +263,7 @@ def self_influence( Returns: self_influence_scores (Tensor): This is a 1D tensor containing the self - influence scores of all examples in `inputs_dataset`, regardless of + influence scores of all examples in `inputs`, regardless of whether it represents a single batch or a `DataLoader` that yields batches. """ @@ -274,7 +273,6 @@ def self_influence( def _get_k_most_influential( self, inputs: Tuple[Any, ...], - targets: Optional[Tensor] = None, k: int = 5, proponents: bool = True, show_progress: bool = False, @@ -282,11 +280,13 @@ def _get_k_most_influential( r""" Args: - inputs (tuple[Any, ...]): A tuple that represents a batch of examples. It - does not represent labels, which are passed as `targets`. - targets (Tensor, optional): If computing influence scores on a loss - function, these are the labels corresponding to the batch `inputs`. - Default: None + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. k (int, optional): The number of proponents or opponents to return per test example. Default: 5 @@ -316,8 +316,7 @@ def _get_k_most_influential( influence scores for a different test example, in sorted order. In particular, `influence_scores[i][j]` is the influence score of example `indices[i][j]` in training dataset `train_dataset` - on example `i` in the test batch represented by `inputs` and - `targets`. + on example `i` in the test batch represented by `inputs`. """ pass @@ -325,26 +324,18 @@ def _get_k_most_influential( def _influence( self, inputs: Tuple[Any, ...], - targets: Optional[Tensor] = None, show_progress: bool = False, ) -> Tensor: r""" Args: - inputs (tuple[Any, ...]): A batch of examples. Does not represent labels, - which are passed as `targets`. The assumption is that - `model(*inputs)` produces the predictions for the batch. - targets (Tensor, optional): If computing influence scores on a loss - function, these are the labels corresponding to the batch - `inputs`. - Default: None - - Returns: - influence_scores (Tensor): Influence scores over the entire - training dataset `train_dataset`. Dimensionality is - (inputs_batch_size, src_dataset_size). For example: - influence_scores[i][j] = the influence score for the j-th training - example to the i-th input example. + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. show_progress (bool, optional): To compute the influence of examples in training dataset `train_dataset`, we compute the influence of each batch. If `show_progress` is true, the progress of this @@ -354,53 +345,52 @@ def _influence( estimation). Otherwise, it will fallback to a simple output of progress. Default: False + + Returns: + influence_scores (Tensor): Influence scores over the entire + training dataset `train_dataset`. Dimensionality is + (inputs_batch_size, src_dataset_size). For example: + influence_scores[i][j] = the influence score for the j-th training + example to the i-th example in the test batch. """ pass @abstractmethod def influence( # type: ignore[override] self, - inputs: Any, - targets: Optional[Tensor] = None, + inputs: Tuple[Any, ...], k: Optional[int] = None, proponents: bool = True, unpack_inputs: bool = True, show_progress: bool = False, ) -> Union[Tensor, KMostInfluentialResults]: r""" - This is the key method of this class, and can be run in 3 different modes, + This is the key method of this class, and can be run in 2 different modes, where the mode that is run depends on the arguments passed to this method: - - influence score mode: This mode is used if `inputs` is not None, and `k` is - None. This mode computes the influence score of every example in - training dataset `train_dataset` on every example in the test - batch represented by `inputs` and `targets`. - - k-most influential mode: This mode is used if `inputs` is not None, and - `k` is not None, and an int. This mode computes the proponents or - opponents of every example in the test batch represented by `inputs` - and `targets`. In particular, for each test example in the test batch, - this mode computes its proponents (resp. opponents), which are the - indices in the training dataset `train_dataset` of the training - examples with the `k` highest (resp. lowest) influence scores on the - test example. Proponents are computed if `proponents` is True. - Otherwise, opponents are computed. For each test example, this method - also returns the actual influence score of each proponent (resp. - opponent) on the test example. + - influence score mode: This mode is used if `k` is None. This mode computes + the influence score of every example in training dataset `train_dataset` + on every example in the test batch represented by `inputs`. + - k-most influential mode: This mode is used if `k` is not None, and an int. + This mode computes the proponents or opponents of every example in the + test batch represented by `inputs`. In particular, for each test example in + the test batch, this mode computes its proponents (resp. opponents), + which are the indices in the training dataset `train_dataset` of the + training examples with the `k` highest (resp. lowest) influence scores on the + test example. Proponents are computed if `proponents` is True. Otherwise, + opponents are computed. For each test example, this method also returns the + actual influence score of each proponent (resp. opponent) on the test + example. Args: - inputs (Any): `inputs` is the test batch that will be - used when running in either influence score or k-most influential - mode. If the argument `unpack_inputs` is False, the - assumption is that `model(inputs)` produces the predictions - for a batch, and `inputs` can be of any type. Otherwise if the - argument `unpack_inputs` is True, the assumption is that - `model(*inputs)` produces the predictions for a batch, and - `inputs` will need to be a tuple. In other words, `inputs` will be - unpacked as an argument when passing to `model`. - targets (Tensor, optional): If computing influence scores on a loss - function, these are the labels corresponding to the batch `inputs`. - Default: None + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. k (int, optional): If not provided or `None`, the influence score mode will be run. Otherwise, the k-most influential mode will be run, and `k` is the number of proponents / opponents to return per @@ -410,10 +400,6 @@ def influence( # type: ignore[override] or opponents (`proponents=False`), if running in k-most influential mode. Default: True - unpack_inputs (bool, optional): Whether to unpack the `inputs` argument to - when passing it to `model`, if `inputs` is a tuple (no unpacking - done otherwise). - Default: True show_progress (bool, optional): For all modes, computation of results requires "training dataset computations": computations for each batch in the training dataset `train_dataset`, which may @@ -428,28 +414,24 @@ def influence( # type: ignore[override] Returns: The return value of this method depends on which mode is run. - - influence score mode: if this mode is run (`inputs is not None, `k` is - None), returns a 2D tensor `influence_scores` of shape - `(input_size, train_dataset_size)`, where `input_size` is - the number of examples in the test batch, and - `train_dataset_size` is the number of examples in - training dataset `train_dataset`. In other words, - `influence_scores[i][j]` is the influence score of the `j`-th - example in `train_dataset` on the `i`-th example in the - test batch. - - k-most influential mode: if this mode is run (`inputs` is not None, - `k` is an int), returns a namedtuple `(indices, influence_scores)`. - `indices` is a 2D tensor of shape `(input_size, k)`, where - `input_size` is the number of examples in the test batch. If - computing proponents (resp. opponents), `indices[i][j]` is the - index in training dataset `train_dataset` of the example - with the `j`-th highest (resp. lowest) influence score (out of the - examples in `train_dataset`) on the `i`-th example in the - test batch. `influence_scores` contains the corresponding influence - scores. In particular, `influence_scores[i][j]` is the influence - score of example `indices[i][j]` in `train_dataset` on - example `i` in the test batch represented by `inputs` and - `targets`. + - influence score mode: if this mode is run (`k` is None), returns a 2D + tensor `influence_scores` of shape `(input_size, train_dataset_size)`, + where `input_size` is the number of examples in the test dataset, and + `train_dataset_size` is the number of examples in training dataset + `train_dataset`. In other words, `influence_scores[i][j]` is the + influence score of the `j`-th example in `train_dataset` on the `i`-th + example in the test batch. + - k-most influential mode: if this mode is run (`k` is an int), returns + a namedtuple `(indices, influence_scores)`. `indices` is a 2D tensor of + shape `(input_size, k)`, where `input_size` is the number of examples in + the test batch. If computing proponents (resp. opponents), + `indices[i][j]` is the index in training dataset `train_dataset` of the + example with the `j`-th highest (resp. lowest) influence score (out of + the examples in `train_dataset`) on the `i`-th example in the test + dataset. `influence_scores` contains the corresponding influence scores. + In particular, `influence_scores[i][j]` is the influence score of example + `indices[i][j]` in `train_dataset` on example `i` in the test batch + represented by `inputs`. """ pass @@ -468,12 +450,10 @@ def get_name(cls: Type["TracInCPBase"]) -> str: def _influence_route_to_helpers( influence_instance: TracInCPBase, - inputs: Any, - targets: Optional[Tensor] = None, + inputs: Tuple[Any, ...], k: Optional[int] = None, proponents: bool = True, - unpack_inputs: bool = True, - show_progress: bool = False, + **kwargs, ) -> Union[Tensor, KMostInfluentialResults]: """ This is a helper function called by `TracInCP.influence` and @@ -485,14 +465,14 @@ def _influence_route_to_helpers( same as the `influence` method. Note that `influence_instance` refers to the instance for which the `influence` method was called. """ - - _inputs = _format_inputs(inputs, unpack_inputs) - if k is None: - return influence_instance._influence(_inputs, targets, show_progress) + return influence_instance._influence(inputs, **kwargs) else: return influence_instance._get_k_most_influential( - _inputs, targets, k, proponents, show_progress + inputs, + k, + proponents, + **kwargs, ) @@ -515,9 +495,8 @@ def __init__( model (torch.nn.Module): An instance of pytorch model. This model should define all of its layers as attributes of the model. train_dataset (torch.utils.data.Dataset or torch.utils.data.DataLoader): - In the `influence` method, we either compute the influence score of - training examples on examples in a test batch, or self influence - scores for those training examples, depending on which mode is used. + In the `influence` method, we compute the influence score of + training examples on examples in a test batch. This argument represents the training dataset containing those training examples. In order to compute those influence scores, we will create a Pytorch DataLoader yielding batches of training @@ -672,47 +651,38 @@ def __init__( @log_usage() def influence( # type: ignore[override] self, - inputs: Any, - targets: Optional[Tensor] = None, + inputs: Tuple[Any, ...], k: Optional[int] = None, proponents: bool = True, - unpack_inputs: bool = True, show_progress: bool = False, ) -> Union[Tensor, KMostInfluentialResults]: r""" - This is the key method of this class, and can be run in 3 different modes, + This is the key method of this class, and can be run in 2 different modes, where the mode that is run depends on the arguments passed to this method: - - influence score mode: This mode is used if `k` is - None. This mode computes the influence score of every example in - training dataset `train_dataset` on every example in the test - batch represented by `inputs` and `targets`. - - k-most influential mode: This mode is used if `k` is not None, - and an int. This mode computes the proponents or - opponents of every example in the test batch represented by `inputs` - and `targets`. In particular, for each test example in the test batch, - this mode computes its proponents (resp. opponents), which are the - indices in the training dataset `train_dataset` of the training - examples with the `k` highest (resp. lowest) influence scores on the - test example. Proponents are computed if `proponents` is True. - Otherwise, opponents are computed. For each test example, this method - also returns the actual influence score of each proponent (resp. - opponent) on the test example. + - influence score mode: This mode is used if `k` is None. This mode computes + the influence score of every example in training dataset `train_dataset` + on every example in the test batch represented by `inputs`. + - k-most influential mode: This mode is used if `k` is not None, and an int. + This mode computes the proponents or opponents of every example in the + test batch represented by `inputs`. In particular, for each test example in + the test batch, this mode computes its proponents (resp. opponents), + which are the indices in the training dataset `train_dataset` of the + training examples with the `k` highest (resp. lowest) influence scores on the + test example. Proponents are computed if `proponents` is True. Otherwise, + opponents are computed. For each test example, this method also returns the + actual influence score of each proponent (resp. opponent) on the test + example. Args: - inputs (Any): `inputs` is the test batch that will be - used when running in either influence score or k-most influential - mode. If the argument `unpack_inputs` is False, the - assumption is that `model(inputs)` produces the predictions - for a batch, and `inputs` can be of any type. Otherwise if the - argument `unpack_inputs` is True, the assumption is that - `model(*inputs)` produces the predictions for a batch, and - `inputs` will need to be a tuple. In other words, `inputs` will be - unpacked as an argument when passing to `model`. - targets (Tensor, optional): If computing influence scores on a loss - function, these are the labels corresponding to the batch `inputs`. - Default: None + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. k (int, optional): If not provided or `None`, the influence score mode will be run. Otherwise, the k-most influential mode will be run, and `k` is the number of proponents / opponents to return per @@ -722,44 +692,38 @@ def influence( # type: ignore[override] or opponents (`proponents=False`), if running in k-most influential mode. Default: True - unpack_inputs (bool, optional): Whether to unpack the `inputs` argument to - when passing it to `model`, if `inputs` is a tuple (no unpacking - done otherwise). - Default: True show_progress (bool, optional): For all modes, computation of results requires "training dataset computations": computations for each batch in the training dataset `train_dataset`, which may take a long time. If `show_progress` is true, the progress of - "training dataset computations" will be displayed. It will try to - use tqdm if available for advanced features (e.g. time estimation). - Otherwise, it will fallback to a simple output of progress. + "training dataset computations" will be displayed. In particular, + the number of batches for which computations have been performed + will be displayed. It will try to use tqdm if available for + advanced features (e.g. time estimation). Otherwise, it will + fallback to a simple output of progress. Default: False Returns: The return value of this method depends on which mode is run. - - influence score mode: if this mode is run (`inputs is not None, `k` is - None), returns a 2D tensor `influence_scores` of shape - `(input_size, train_dataset_size)`, where `input_size` is - the number of examples in the test batch, and - `train_dataset_size` is the number of examples in - training dataset `train_dataset`. In other words, - `influence_scores[i][j]` is the influence score of the `j`-th - example in `train_dataset` on the `i`-th example in the - test batch. - - k-most influential mode: if this mode is run (`inputs` is not None, - `k` is an int), returns a namedtuple `(indices, influence_scores)`. - `indices` is a 2D tensor of shape `(input_size, k)`, where - `input_size` is the number of examples in the test batch. If - computing proponents (resp. opponents), `indices[i][j]` is the - index in training dataset `train_dataset` of the example - with the `j`-th highest (resp. lowest) influence score (out of the - examples in `train_dataset`) on the `i`-th example in the - test batch. `influence_scores` contains the corresponding influence - scores. In particular, `influence_scores[i][j]` is the influence - score of example `indices[i][j]` in `train_dataset` on - example `i` in the test batch represented by `inputs` and - `targets`. + - influence score mode: if this mode is run (`k` is None), returns a 2D + tensor `influence_scores` of shape `(input_size, train_dataset_size)`, + where `input_size` is the number of examples in the test batch, and + `train_dataset_size` is the number of examples in training dataset + `train_dataset`. In other words, `influence_scores[i][j]` is the + influence score of the `j`-th example in `train_dataset` on the `i`-th + example in the test batch. + - k-most influential mode: if this mode is run (`k` is an int), returns + a namedtuple `(indices, influence_scores)`. `indices` is a 2D tensor of + shape `(input_size, k)`, where `input_size` is the number of examples in + the test batch. If computing proponents (resp. opponents), + `indices[i][j]` is the index in training dataset `train_dataset` of the + example with the `j`-th highest (resp. lowest) influence score (out of + the examples in `train_dataset`) on the `i`-th example in the test + batch. `influence_scores` contains the corresponding influence scores. + In particular, `influence_scores[i][j]` is the influence score of example + `indices[i][j]` in `train_dataset` on example `i` in the test batch + represented by `inputs`. """ assert inputs is not None, ( @@ -770,26 +734,24 @@ def influence( # type: ignore[override] return _influence_route_to_helpers( self, inputs, - targets, k, proponents, - unpack_inputs, - show_progress, + show_progress=show_progress, ) def _sum_jacobians( self, - inputs_dataset: DataLoader, + inputs: DataLoader, loss_fn: Optional[Union[Module, Callable]] = None, reduction_type: Optional[str] = None, ): """ - sums the jacobians of all examples in `inputs_dataset`. result is of the + sums the jacobians of all examples in `inputs`. result is of the same format as layer_jacobians, but the batch dimension has size 1 """ - inputs_dataset_iter = iter(inputs_dataset) + inputs_iter = iter(inputs) - inputs_batch = next(inputs_dataset_iter) + inputs_batch = next(inputs_iter) def get_batch_contribution(inputs_batch): _input_jacobians = self._basic_computation_tracincp( @@ -805,7 +767,7 @@ def get_batch_contribution(inputs_batch): inputs_jacobians = get_batch_contribution(inputs_batch) - for inputs_batch in inputs_dataset_iter: + for inputs_batch in inputs_iter: inputs_batch_jacobians = get_batch_contribution(inputs_batch) inputs_jacobians = tuple( [ @@ -820,7 +782,7 @@ def get_batch_contribution(inputs_batch): def _concat_jacobians( self, - inputs_dataset: DataLoader, + inputs: DataLoader, loss_fn: Optional[Union[Module, Callable]] = None, reduction_type: Optional[str] = None, ): @@ -831,7 +793,7 @@ def _concat_jacobians( loss_fn, reduction_type, ) - for inputs_batch in inputs_dataset + for inputs_batch in inputs ] return tuple( @@ -842,7 +804,7 @@ def _concat_jacobians( @log_usage() def compute_intermediate_quantities( self, - inputs_dataset: Union[Tuple[Any, ...], DataLoader], + inputs: Union[Tuple[Any, ...], DataLoader], aggregate: bool = False, ) -> Tensor: """ @@ -867,7 +829,7 @@ def compute_intermediate_quantities( then taking the sum) allows memory usage to be reduced. Args: - inputs_dataset (Tuple, or DataLoader): Either a single tuple of any, or a + inputs (Tuple, or DataLoader): Either a single tuple of any, or a `DataLoader`, where each batch yielded is a tuple of any. In either case, the tuple represents a single batch, where the last element is assumed to be the labels for the batch. That is, @@ -881,7 +843,7 @@ def compute_intermediate_quantities( Returns: intermediate_quantities (Tensor): A tensor of dimension (N, D * C). Here, N is the total number of examples in - `inputs_dataset` if `aggregate` is False, and 1, otherwise (so that + `inputs` if `aggregate` is False, and 1, otherwise (so that a 2D tensor is always returned). C is the number of checkpoints passed as the `checkpoints` argument of `TracInCP.__init__`, and each row represents the vector for an example. Regarding D: Let I @@ -898,8 +860,8 @@ def compute_intermediate_quantities( the variable d in the top of page 15 of the TracIn paper: https://arxiv.org/pdf/2002.08484.pdf. """ - # If `inputs_dataset` is not a `DataLoader`, turn it into one. - inputs_dataset = _format_inputs_dataset(inputs_dataset) + # If `inputs` is not a `DataLoader`, turn it into one. + inputs = _format_inputs_dataset(inputs) def get_checkpoint_contribution(checkpoint): assert ( @@ -910,11 +872,11 @@ def get_checkpoint_contribution(checkpoint): # get jacobians as tuple of tensors if aggregate: inputs_jacobians = self._sum_jacobians( - inputs_dataset, self.loss_fn, self.reduction_type + inputs, self.loss_fn, self.reduction_type ) else: inputs_jacobians = self._concat_jacobians( - inputs_dataset, self.loss_fn, self.reduction_type + inputs, self.loss_fn, self.reduction_type ) # flatten into single tensor return learning_rate * torch.cat( @@ -935,9 +897,8 @@ def get_checkpoint_contribution(checkpoint): def _influence_batch_tracincp( self, - inputs: Tuple[Any, ...], - targets: Optional[Tensor], - batch: Tuple[Any, ...], + test_batch: Tuple[Any, ...], + train_batch: Tuple[Any, ...], ): """ computes influence scores for a single training batch @@ -952,8 +913,8 @@ def get_checkpoint_contribution(checkpoint): learning_rate = self.checkpoints_load_func(self.model, checkpoint) input_jacobians = self._basic_computation_tracincp( - inputs, - targets, + test_batch[0:-1], + test_batch[-1], self.test_loss_fn, self.test_reduction_type, ) @@ -961,7 +922,10 @@ def get_checkpoint_contribution(checkpoint): _gradient_dot_product( input_jacobians, self._basic_computation_tracincp( - batch[0:-1], batch[-1], self.loss_fn, self.reduction_type + train_batch[0:-1], + train_batch[-1], + self.loss_fn, + self.reduction_type, ), ) * learning_rate @@ -977,24 +941,24 @@ def get_checkpoint_contribution(checkpoint): def _influence( self, inputs: Tuple[Any, ...], - targets: Optional[Tensor] = None, show_progress: bool = False, ) -> Tensor: r""" Computes the influence of examples in training dataset `train_dataset` - on the examples in the test batch represented by `inputs` and `targets`. + on the examples in the test batch represented by `inputs`. This implementation does not require knowing the number of training examples in advance. Instead, the number of training examples is inferred from the output of `self._basic_computation_tracincp`. Args: - inputs (tuple[Any, ...]): A test batch of examples. Does not represent - labels, which are passed as `targets`. The assumption is that - `model(*inputs)` produces the predictions for the batch. - targets (Tensor, optional): If computing influence scores on a loss - function, these are the labels corresponding to the batch `inputs`. - Default: None + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. show_progress (bool, optional): To compute the influence of examples in training dataset `train_dataset`, we compute the influence of each batch. If `show_progress` is true, the progress of this @@ -1012,7 +976,7 @@ def _influence( `train_dataset_size` is the number of examples in training dataset `train_dataset`. For example: `influence_scores[i][j]` is the influence score for the j-th training - example to the i-th input example. + example to the i-th example in the test batch. """ train_dataloader = self.train_dataloader @@ -1028,7 +992,7 @@ def _influence( return torch.cat( [ - self._influence_batch_tracincp(inputs, targets, batch) + self._influence_batch_tracincp(inputs, batch) for batch in train_dataloader ], dim=1, @@ -1037,7 +1001,6 @@ def _influence( def _get_k_most_influential( self, inputs: Tuple[Any, ...], - targets: Optional[Tensor] = None, k: int = 5, proponents: bool = True, show_progress: bool = False, @@ -1045,11 +1008,13 @@ def _get_k_most_influential( r""" Args: - inputs (tuple[Any, ...]): A tuple that represents a batch of examples. It - does not represent labels, which are passed as `targets`. - targets (Tensor, optional): If computing influence scores on a loss - function, these are the labels corresponding to the batch `inputs`. - Default: None + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. k (int, optional): The number of proponents or opponents to return per test example. Default: 5 @@ -1079,8 +1044,7 @@ def _get_k_most_influential( influence scores for a different test example, in sorted order. In particular, `influence_scores[i][j]` is the influence score of example `indices[i][j]` in training dataset `train_dataset` - on example `i` in the test batch represented by `inputs` and - `targets`. + on example `i` in the test batch represented by `inputs`. """ desc = ( None @@ -1098,7 +1062,6 @@ def _get_k_most_influential( self.train_dataloader, self._influence_batch_tracincp, inputs, - targets, k, proponents, show_progress, @@ -1108,21 +1071,21 @@ def _get_k_most_influential( def _self_influence_by_checkpoints( self, - inputs_dataset: Union[Tuple[Any, ...], DataLoader], + inputs: Union[Tuple[Any, ...], DataLoader], show_progress: bool = False, ) -> Tensor: """ - Computes self influence scores for the examples in `inputs_dataset`, which is + Computes self influence scores for the examples in `inputs`, which is either a single batch or a Pytorch `DataLoader` that yields batches. Therefore, the computed self influence scores are *not* for the examples in training dataset `train_dataset` (unlike when computing self influence scores using the - `influence` method). Note that if `inputs_dataset` is a single batch, this - will call `model` on that single batch, and if `inputs_dataset` yields + `influence` method). Note that if `inputs` is a single batch, this + will call `model` on that single batch, and if `inputs` yields batches, this will call `model` on each batch that is yielded. Therefore, please ensure that for both cases, the batch(es) that `model` is called with are not too large, so that there will not be an out-of-memory error. This implementation performs an outer iteration over checkpoints, and an inner - iteration over all batches that `inputs_dataset` represents. The pros of this + iteration over all batches that `inputs` represents. The pros of this implementation are that the checkpoints do not need to be loaded too many times. @@ -1138,12 +1101,12 @@ def _self_influence_by_checkpoints( `train_dataset` argument to `TracInCP.__init__` for more details on the assumed structure of a batch. show_progress (bool, optional): Computation of self influence scores can - take a long time if `inputs_dataset` represents many examples. If + take a long time if `inputs` represents many examples. If `show_progress` is true, the progress of this computation will be displayed. In more detail, this computation will iterate over all checkpoints (provided as the `checkpoints` initialization argument) in an outer loop, and iterate over all batches that - `inputs_dataset` represents in an inner loop. Thus if + `inputs` represents in an inner loop. Thus if `show_progress` is True, the progress of both the outer iteration and the inner iterations will be displayed. To show progress, it will try to use tqdm if available for advanced @@ -1153,24 +1116,24 @@ def _self_influence_by_checkpoints( Returns: self_influence_scores (Tensor): This is a 1D tensor containing the self - influence scores of all examples in `inputs_dataset`, regardless of + influence scores of all examples in `inputs`, regardless of whether it represents a single batch or a `DataLoader` that yields batches. """ - # If `inputs_dataset` is not a `DataLoader`, turn it into one. - inputs_dataset = _format_inputs_dataset(inputs_dataset) + # If `inputs` is not a `DataLoader`, turn it into one. + inputs = _format_inputs_dataset(inputs) # If `show_progress` is true, create an outer progress bar that keeps track of # how many checkpoints have been processed if show_progress: # Try to determine length of inner progress bar if possible, with a default # of `None`. - inputs_dataset_len = None + inputs_len = None try: - inputs_dataset_len = len(inputs_dataset) + inputs_len = len(inputs) except TypeError: warnings.warn( - "Unable to determine the number of batches in `inputs_dataset`. " + "Unable to determine the number of batches in `inputs`. " "Therefore, if showing the progress of the computation of self " "influence scores, only the number of batches processed can be " "displayed, and not the percentage completion of the computation, " @@ -1193,8 +1156,8 @@ def calculate_via_flatten(layer_jacobian): def get_checkpoint_contribution(checkpoint): # This function returns a 1D tensor representing the contribution to the # self influence score for the given checkpoint, for all batches in - # `inputs_dataset`. The length of the 1D tensor is the total number of - # examples in `inputs_dataset`. + # `inputs`. The length of the 1D tensor is the total number of + # examples in `inputs`. assert ( checkpoint is not None ), "None returned from `checkpoints`, cannot load." @@ -1203,24 +1166,24 @@ def get_checkpoint_contribution(checkpoint): # This will store a list of the contribution of the self influence score # from each batch. Each element is a 1D tensor of length batch_size - the - # batch size of each batch in `inputs_dataset` (they do not need to be all + # batch size of each batch in `inputs` (they do not need to be all # the same) checkpoint_contribution = [] - _inputs_dataset = inputs_dataset + _inputs = inputs # If `show_progress` is true, create an inner progress bar that keeps track # of how many batches have been processed for the current checkpoint if show_progress: - _inputs_dataset = progress( - inputs_dataset, + _inputs = progress( + inputs, desc=( f"Using {self.get_name()} to compute self " "influence. Processing batch" ), - total=inputs_dataset_len, + total=inputs_len, ) - for batch in _inputs_dataset: + for batch in _inputs: layer_jacobians = self._basic_computation_tracincp( batch[0:-1], @@ -1258,7 +1221,7 @@ def get_checkpoint_contribution(checkpoint): ) # We concatenate the contributions from each batch into a single 1D tensor, - # which represents the contributions for all batches in `inputs_dataset` + # which represents the contributions for all batches in `inputs` return torch.cat(checkpoint_contribution, dim=0) @@ -1289,22 +1252,22 @@ def get_checkpoint_contribution(checkpoint): @log_usage() def self_influence( self, - inputs_dataset: Optional[Union[Tuple[Any, ...], DataLoader]] = None, + inputs: Optional[Union[Tuple[Any, ...], DataLoader]] = None, show_progress: bool = False, outer_loop_by_checkpoints: bool = False, ) -> Tensor: """ - Computes self influence scores for the examples in `inputs_dataset`, which is + Computes self influence scores for the examples in `inputs`, which is either a single batch or a Pytorch `DataLoader` that yields batches. - If `inputs_dataset` is not specified or `None` calculates self influence - score for the training dataset `train_dataset`. Note that if `inputs_dataset` + If `inputs` is not specified or `None` calculates self influence + score for the training dataset `train_dataset`. Note that if `inputs` is a single batch, this will call `model` on that single batch, and if - `inputs_dataset` yields batches, this will call `model` on each batch that is + `inputs` yields batches, this will call `model` on each batch that is yielded. Therefore, please ensure that for both cases, the batch(es) that `model` is called with are not too large, so that there will not be an out-of-memory error. Internally, this computation requires iterating both over the batches in - `inputs_dataset`, as well as different model checkpoints. There are two ways + `inputs`, as well as different model checkpoints. There are two ways this iteration can be done. If `outer_loop_by_checkpoints` is False, the outer iteration will be over batches, and the inner iteration will be over checkpoints. This has the pro that displaying the progress of the computation @@ -1315,7 +1278,7 @@ def self_influence( for each batch. For large models, loading checkpoints can be time-intensive. Args: - inputs_dataset (tuple or DataLoader, optional): This specifies the + inputs (tuple or DataLoader, optional): This specifies the dataset for which self influence scores will be computed. Either a single tuple of any, or a `DataLoader`, where each batch yielded is a tuple of type any. In either case, the tuple @@ -1331,7 +1294,7 @@ def self_influence( above assumptions. Default: None. show_progress (bool, optional): Computation of self influence scores can - take a long time if `inputs_dataset` represents many examples. If + take a long time if `inputs` represents many examples. If `show_progress`is true, the progress of this computation will be displayed. In more detail, if `outer_loop_by_checkpoints` is False, this computation will iterate over all batches in an outer loop. @@ -1340,7 +1303,7 @@ def self_influence( `outer_loop_by_checkpoints` is True, this computation will iterate over all checkpoints (provided as the `checkpoints` initialization argument) in an outer loop, and iterate over all batches that - `inputs_dataset` represents in an inner loop. Thus if + `inputs` represents in an inner loop. Thus if `show_progress` is True, the progress of both the outer iteration and the inner iterations will be displayed. To show progress, it will try to use tqdm if available for advanced @@ -1352,15 +1315,13 @@ def self_influence( details. Default: False """ - inputs_dataset = ( - inputs_dataset if inputs_dataset is not None else self.train_dataloader - ) + inputs = inputs if inputs is not None else self.train_dataloader if outer_loop_by_checkpoints: - return self._self_influence_by_checkpoints(inputs_dataset, show_progress) + return self._self_influence_by_checkpoints(inputs, show_progress) return _self_influence_by_batches_helper( self._self_influence_by_checkpoints, self.get_name(), - inputs_dataset, + inputs, show_progress, ) diff --git a/captum/influence/_core/tracincp_fast_rand_proj.py b/captum/influence/_core/tracincp_fast_rand_proj.py index 5b3c309ec4..07e3458b38 100644 --- a/captum/influence/_core/tracincp_fast_rand_proj.py +++ b/captum/influence/_core/tracincp_fast_rand_proj.py @@ -6,7 +6,7 @@ from typing import Any, Callable, cast, Dict, Iterator, List, Optional, Tuple, Union import torch -from captum._utils.common import _format_inputs, _get_module_from_name, _sort_key_list +from captum._utils.common import _get_module_from_name, _sort_key_list from captum._utils.gradient import _gather_distributed_tensors from captum._utils.progress import NullProgress, progress @@ -17,7 +17,6 @@ ) from captum.influence._utils.common import ( _check_loss_fn, - _DatasetFromList, _format_inputs_dataset, _get_k_most_influential_helper, _jacobian_loss_wrt_inputs, @@ -103,9 +102,8 @@ def __init__( fully qualified name of the layer if it is a defined attribute of the passed `model`. train_dataset (torch.utils.data.Dataset or torch.utils.data.DataLoader): - In the `influence` method, we either compute the influence score of - training examples on examples in a test batch, or self influence - scores for those training examples, depending on which mode is used. + In the `influence` method, we compute the influence score of + training examples on examples in a test batch. This argument represents the training dataset containing those training examples. In order to compute those influence scores, we will create a Pytorch DataLoader yielding batches of training @@ -208,48 +206,38 @@ def __init__( @log_usage() def influence( # type: ignore[override] self, - inputs: Any, - targets: Optional[Tensor] = None, + inputs: Tuple[Any, ...], k: Optional[int] = None, proponents: bool = True, - unpack_inputs: bool = True, show_progress: bool = False, ) -> Union[Tensor, KMostInfluentialResults]: r""" This is the key method of this class, and can be run in 2 different modes, where the mode that is run depends on the arguments passed to this method: - - influence score mode: This mode is used if `k` is - None. This mode computes the influence score of every example in - training dataset `train_dataset` on every example in the test - batch represented by `inputs` and `targets`. - - k-most influential mode: This mode is used if `k` is not None, - and an int. This mode computes the proponents or - opponents of every example in the test batch represented by `inputs` - and `targets`. In particular, for each test example in the test batch, - this mode computes its proponents (resp. opponents), which are the - indices in the training dataset `train_dataset` of the training - examples with the `k` highest (resp. lowest) influence scores on the - test example. Proponents are computed if `proponents` is True. - Otherwise, opponents are computed. For each test example, this method - also returns the actual influence score of each proponent (resp. - opponent) on the test example. + - influence score mode: This mode is used if `k` is None. This mode computes + the influence score of every example in training dataset `train_dataset` + on every example in the test batch represented by `inputs`. + - k-most influential mode: This mode is used if `k` is not None, and an int. + This mode computes the proponents or opponents of every example in the + test batch represented by `inputs`. In particular, for each test example in + the test batch, this mode computes its proponents (resp. opponents), + which are the indices in the training dataset `train_dataset` of the + training examples with the `k` highest (resp. lowest) influence scores on the + test example. Proponents are computed if `proponents` is True. Otherwise, + opponents are computed. For each test example, this method also returns the + actual influence score of each proponent (resp. opponent) on the test + example. Args: - inputs (Any): `inputs` is the test batch that will be - used when running in either influence score or k-most influential - mode. If the argument `unpack_inputs` is False, the - assumption is that `model(inputs)` produces the predictions - for a batch, and `inputs` can be of any type. Otherwise if the - argument `unpack_inputs` is True, the assumption is that - `model(*inputs)` produces the predictions for a batch, and - `inputs` will need to be a tuple. In other words, `inputs` will be - unpacked as an argument when passing to `model`. - targets (Tensor, optional): The labels corresponding to the batch `inputs`. - This method is designed to be applied for a loss function, so - `targets` is required, unless running in "self influence" mode. - Default: None + inputs (tuple or DataLoader): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. k (int, optional): If not provided or `None`, the influence score mode will be run. Otherwise, the k-most influential mode will be run, and `k` is the number of proponents / opponents to return per @@ -259,49 +247,38 @@ def influence( # type: ignore[override] or opponents (`proponents=False`), if running in k-most influential mode. Default: True - unpack_inputs (bool, optional): Whether to unpack the `inputs` argument to - when passing it to `model`, if `inputs` is a tuple (no unpacking - done otherwise). - Default: True show_progress (bool, optional): For all modes, computation of results requires "training dataset computations": computations for each batch in the training dataset `train_dataset`, which may take a long time. If `show_progress` is true, the progress of - "training dataset computations" will be displayed. It will try to - use tqdm if available for advanced features (e.g. time estimation). - Otherwise, it will fallback to a simple output of progress. + "training dataset computations" will be displayed. In particular, + the number of batches for which computations have been performed + will be displayed. It will try to use tqdm if available for + advanced features (e.g. time estimation). Otherwise, it will + fallback to a simple output of progress. Default: False Returns: The return value of this method depends on which mode is run. - - self influence mode: if this mode is run (`inputs` is None), returns a 1D - tensor of self influence scores over training dataset - `train_dataset`. The length of this tensor is the number of - examples in `train_dataset`, regardless of whether it is a - Dataset or DataLoader. - - influence score mode: if this mode is run (`inputs` is not None, `k` is - None), returns a 2D tensor `influence_scores` of shape - `(input_size, train_dataset_size)`, where `input_size` is - the number of examples in the test batch, and - `train_dataset_size` is the number of examples in - training dataset `train_dataset`. In other words, - `influence_scores[i][j]` is the influence score of the `j`-th - example in `train_dataset` on the `i`-th example in the - test batch. - - k-most influential mode: if this mode is run (`inputs` is not None, - `k` is an int), returns a namedtuple `(indices, influence_scores)`. - `indices` is a 2D tensor of shape `(input_size, k)`, where - `input_size` is the number of examples in the test batch. If - computing proponents (resp. opponents), `indices[i][j]` is the - index in training dataset `train_dataset` of the example - with the `j`-th highest (resp. lowest) influence score (out of the - examples in `train_dataset`) on the `i`-th example in the - test batch. `influence_scores` contains the corresponding influence - scores. In particular, `influence_scores[i][j]` is the influence - score of example `indices[i][j]` in `train_dataset` on - example `i` in the test batch represented by `inputs` and - `targets`. + - influence score mode: if this mode is run (`k` is None), returns a 2D + tensor `influence_scores` of shape `(input_size, train_dataset_size)`, + where `input_size` is the number of examples in the test batch, and + `train_dataset_size` is the number of examples in training dataset + `train_dataset`. In other words, `influence_scores[i][j]` is the + influence score of the `j`-th example in `train_dataset` on the `i`-th + example in the test batch. + - k-most influential mode: if this mode is run (`k` is an int), returns + a namedtuple `(indices, influence_scores)`. `indices` is a 2D tensor of + shape `(input_size, k)`, where `input_size` is the number of examples in + the test batch. If computing proponents (resp. opponents), + `indices[i][j]` is the index in training dataset `train_dataset` of the + example with the `j`-th highest (resp. lowest) influence score (out of + the examples in `train_dataset`) on the `i`-th example in the test + batch. `influence_scores` contains the corresponding influence scores. + In particular, `influence_scores[i][j]` is the influence score of example + `indices[i][j]` in `train_dataset` on example `i` in the test batch + represented by `inputs`. """ assert inputs is not None, ( "`inputs` argument is required." @@ -311,18 +288,15 @@ def influence( # type: ignore[override] return _influence_route_to_helpers( self, inputs, - targets, k, proponents, - unpack_inputs, - show_progress, + show_progress=show_progress, ) def _influence_batch_tracincp_fast( self, - inputs: Tuple[Any, ...], - targets: Tensor, - batch: Tuple[Any, ...], + test_batch: Tuple[Any, ...], + train_batch: Tuple[Any, ...], ): """ computes influence scores for a single training batch, when only considering @@ -340,16 +314,16 @@ def get_checkpoint_contribution(checkpoint): input_jacobians, input_layer_inputs = _basic_computation_tracincp_fast( self, - inputs, - targets, + test_batch[0:-1], + test_batch[-1], self.test_loss_fn, self.test_reduction_type, ) src_jacobian, src_layer_input = _basic_computation_tracincp_fast( self, - batch[0:-1], - batch[-1], + train_batch[0:-1], + train_batch[-1], self.loss_fn, self.reduction_type, ) @@ -376,24 +350,24 @@ def get_checkpoint_contribution(checkpoint): def _influence( # type: ignore[override] self, inputs: Tuple[Any, ...], - targets: Tensor, show_progress: bool = False, ) -> Tensor: r""" Computes the influence of examples in training dataset `train_dataset` - on the examples in the test batch represented by `inputs` and `targets`. + on the examples in the test batch represented by `inputs`. This implementation does not require knowing the number of training examples in advance. Instead, the number of training examples is inferred from the output of `_basic_computation_tracincp_fast`. Args: - inputs (tuple[Any, ...]): A batch of examples. Does not represent labels, - which are passed as `targets`. The assumption is that - `model(*inputs)` produces the predictions for the batch. - targets (Tensor): The labels corresponding to the batch `inputs`. This - method is designed to be applied for a loss function, so labels - are required. + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. show_progress (bool, optional): To compute the influence of examples in training dataset `train_dataset`, we compute the influence of each batch. If `show_progress` is true, the progress of this @@ -405,15 +379,14 @@ def _influence( # type: ignore[override] Default: False Returns: - influence_scores (Tensor): Influence scores from the TracInCPFast method. + influence_scores (Tensor): Influence scores from the `TracInCPFast` method. Its shape is `(input_size, train_dataset_size)`, where `input_size` is the number of examples in the test batch, and `train_dataset_size` is the number of examples in training dataset `train_dataset`. For example: `influence_scores[i][j]` is the influence score for the j-th training - example to the i-th input example. + example to the i-th example in the test batch. """ - assert targets is not None train_dataloader = self.train_dataloader @@ -429,7 +402,7 @@ def _influence( # type: ignore[override] return torch.cat( [ - self._influence_batch_tracincp_fast(inputs, targets, batch) + self._influence_batch_tracincp_fast(inputs, batch) for batch in train_dataloader ], dim=1, @@ -438,7 +411,6 @@ def _influence( # type: ignore[override] def _get_k_most_influential( # type: ignore[override] self, inputs: Tuple[Any, ...], - targets: Tensor, k: int = 5, proponents: bool = True, show_progress: bool = False, @@ -446,11 +418,13 @@ def _get_k_most_influential( # type: ignore[override] r""" Args: - inputs (tuple[Any, ...]): A tuple that represents a batch of examples. It - does not represent labels, which are passed as `targets`. - targets (Tensor): The labels corresponding to the batch `inputs`. This - method is designed to be applied for a loss function, so labels - are required. + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. k (int, optional): The number of proponents or opponents to return per test example. Default: 5 @@ -480,8 +454,7 @@ def _get_k_most_influential( # type: ignore[override] influence scores for a different test example, in sorted order. In particular, `influence_scores[i][j]` is the influence score of example `indices[i][j]` in training dataset `train_dataset` - on example `i` in the test batch represented by `inputs` and - `targets`. + on example `i` in the test batch represented by `inputs`. """ desc = ( None @@ -499,7 +472,6 @@ def _get_k_most_influential( # type: ignore[override] self.train_dataloader, self._influence_batch_tracincp_fast, inputs, - targets, k, proponents, show_progress, @@ -509,21 +481,21 @@ def _get_k_most_influential( # type: ignore[override] def _self_influence_by_checkpoints( self, - inputs_dataset: Union[Tuple[Any, ...], DataLoader], + inputs: Union[Tuple[Any, ...], DataLoader], show_progress: bool = False, ) -> Tensor: """ - Computes self influence scores for the examples in `inputs_dataset`, which is + Computes self influence scores for the examples in `inputs`, which is either a single batch or a Pytorch `DataLoader` that yields batches. Therefore, the computed self influence scores are *not* for the examples in training dataset `train_dataset` (unlike when computing self influence scores using the - `influence` method). Note that if `inputs_dataset` is a single batch, this - will call `model` on that single batch, and if `inputs_dataset` yields + `influence` method). Note that if `inputs` is a single batch, this + will call `model` on that single batch, and if `inputs` yields batches, this will call `model` on each batch that is yielded. Therefore, please ensure that for both cases, the batch(es) that `model` is called with are not too large, so that there will not be an out-of-memory error. This implementation performs an outer iteration over checkpoints, and an inner - iteration over all batches that `inputs_dataset` represents. The pros of this + iteration over all batches that `inputs` represents. The pros of this implementation are that the checkpoints do not need to be loaded too many times. @@ -539,12 +511,12 @@ def _self_influence_by_checkpoints( `train_dataset` argument to `TracInCP.__init__` for more details on the assumed structure of a batch. show_progress (bool, optional): Computation of self influence scores can - take a long time if `inputs_dataset` represents many examples. If + take a long time if `inputs` represents many examples. If `show_progress` is true, the progress of this computation will be displayed. In more detail, this computation will iterate over all checkpoints (provided as the `checkpoints` initialization argument) in an outer loop, and iterate over all batches that - `inputs_dataset` represents in an inner loop. Thus if + `inputs` represents in an inner loop. Thus if `show_progress` is True, the progress of both the outer iteration and the inner iterations will be displayed. To show progress, it will try to use tqdm if available for advanced @@ -554,24 +526,24 @@ def _self_influence_by_checkpoints( Returns: self_influence_scores (Tensor): This is a 1D tensor containing the self - influence scores of all examples in `inputs_dataset`, regardless of + influence scores of all examples in `inputs`, regardless of whether it represents a single batch or a `DataLoader` that yields batches. """ - # If `inputs_dataset` is not a `DataLoader`, turn it into one. - inputs_dataset = _format_inputs_dataset(inputs_dataset) + # If `inputs` is not a `DataLoader`, turn it into one. + inputs = _format_inputs_dataset(inputs) # If `show_progress` is true, create an outer progress bar that keeps track of # how many checkpoints have been processed if show_progress: # Try to determine length of inner progress bar if possible, with a default # of `None`. - inputs_dataset_len = None + inputs_len = None try: - inputs_dataset_len = len(inputs_dataset) + inputs_len = len(inputs) except TypeError: warnings.warn( - "Unable to determine the number of batches in `inputs_dataset`. " + "Unable to determine the number of batches in `inputs`. " "Therefore, if showing the progress of the computation of self " "influence scores, only the number of batches processed can be " "displayed, and not the percentage completion of the computation, " @@ -581,8 +553,8 @@ def _self_influence_by_checkpoints( def get_checkpoint_contribution(checkpoint): # This function returns a 1D tensor representing the contribution to the # self influence score for the given checkpoint, for all batches in - # `inputs_dataset`. The length of the 1D tensor is the total number of - # examples in `inputs_dataset`. + # `inputs`. The length of the 1D tensor is the total number of + # examples in `inputs`. assert ( checkpoint is not None ), "None returned from `checkpoints`, cannot load." @@ -591,24 +563,24 @@ def get_checkpoint_contribution(checkpoint): # This will store a list of the contribution of the self influence score # from each batch. Each element is a 1D tensor of length batch_size - the - # batch size of each batch in `inputs_dataset` (they do not need to be all + # batch size of each batch in `inputs` (they do not need to be all # the same) checkpoint_contribution = [] - _inputs_dataset = inputs_dataset + _inputs = inputs # If `show_progress` is true, create an inner progress bar that keeps track # of how many batches have been processed for the current checkpoint if show_progress: - _inputs_dataset = progress( - inputs_dataset, + _inputs = progress( + inputs, desc=( f"Using {self.get_name()} to compute self " "influence. Processing batch" ), - total=inputs_dataset_len, + total=inputs_len, ) - for batch in _inputs_dataset: + for batch in _inputs: batch_jacobian, batch_layer_input = _basic_computation_tracincp_fast( self, @@ -625,7 +597,7 @@ def get_checkpoint_contribution(checkpoint): ) # We concatenate the contributions from each batch into a single 1D tensor, - # which represents the contributions for all batches in `inputs_dataset` + # which represents the contributions for all batches in `inputs` return torch.cat(checkpoint_contribution, dim=0) if show_progress: @@ -655,22 +627,22 @@ def get_checkpoint_contribution(checkpoint): @log_usage() def self_influence( self, - inputs_dataset: Optional[Union[Tuple[Any, ...], DataLoader]] = None, + inputs: Optional[Union[Tuple[Any, ...], DataLoader]] = None, show_progress: bool = False, outer_loop_by_checkpoints: bool = False, ) -> Tensor: """ - Computes self influence scores for the examples in `inputs_dataset`, which is + Computes self influence scores for the examples in `inputs`, which is either a single batch or a Pytorch `DataLoader` that yields batches. - If `inputs_dataset` is not specified or `None` calculates self influence - score for the training dataset `train_dataset`. Note that if `inputs_dataset` + If `inputs` is not specified or `None` calculates self influence + score for the training dataset `train_dataset`. Note that if `inputs` is a single batch, this will call `model` on that single batch, - and if `inputs_dataset` yields batches, this will call `model` + and if `inputs` yields batches, this will call `model` on each batch that is yielded. Therefore, please ensure that for both cases, the batch(es) that `model` is called with are not too large, so that there will not be an out-of-memory error. Internally, this computation requires iterating both over the batches in - `inputs_dataset`, as well as different model checkpoints. There are two ways + `inputs`, as well as different model checkpoints. There are two ways this iteration can be done. If `outer_loop_by_checkpoints` is False, the outer iteration will be over batches, and the inner iteration will be over checkpoints. This has the pro that displaying the progress of the computation @@ -681,7 +653,7 @@ def self_influence( for each batch. For large models, loading checkpoints can be time-intensive. Args: - inputs_dataset (tuple or DataLoader, optional): This specifies the + inputs (tuple or DataLoader, optional): This specifies the dataset for which self influence scores will be computed. Either a single tuple of any, or a `DataLoader`, where each batch yielded is a tuple of type any. In either case, the tuple @@ -697,7 +669,7 @@ def self_influence( above assumptions. Default: None. show_progress (bool, optional): Computation of self influence scores can - take a long time if `inputs_dataset` represents many examples. If + take a long time if `inputs` represents many examples. If `show_progress`is true, the progress of this computation will be displayed. In more detail, if `outer_loop_by_checkpoints` is False, this computation will iterate over all batches in an outer loop. @@ -706,7 +678,7 @@ def self_influence( `outer_loop_by_checkpoints` is True, this computation will iterate over all checkpoints (provided as the `checkpoints` initialization argument) in an outer loop, and iterate over all batches that - `inputs_dataset` represents in an inner loop. Thus if + `inputs` represents in an inner loop. Thus if `show_progress` is True, the progress of both the outer iteration and the inner iterations will be displayed. To show progress, it will try to use tqdm if available for advanced @@ -718,15 +690,13 @@ def self_influence( details. Default: False """ - inputs_dataset = ( - inputs_dataset if inputs_dataset is not None else self.train_dataloader - ) + inputs = inputs if inputs is not None else self.train_dataloader if outer_loop_by_checkpoints: - return self._self_influence_by_checkpoints(inputs_dataset, show_progress) + return self._self_influence_by_checkpoints(inputs, show_progress) return _self_influence_by_batches_helper( self._self_influence_by_checkpoints, self.get_name(), - inputs_dataset, + inputs, show_progress, ) @@ -905,9 +875,8 @@ def __init__( fully qualified name of the layer if it is a defined attribute of the passed `model`. train_dataset (torch.utils.data.Dataset or torch.utils.data.DataLoader): - In the `influence` method, we either compute the influence score of - training examples on examples in a test batch, or self influence - scores for those training examples, depending on which mode is used. + In the `influence` method, we compute the influence score of + training examples on examples in a test batch. This argument represents the training dataset containing those training examples. In order to compute those influence scores, we will create a Pytorch DataLoader yielding batches of training @@ -1049,34 +1018,31 @@ def __init__( def _influence( # type: ignore[override] self, inputs: Tuple[Any, ...], - targets: Tensor, ) -> Tensor: r""" Args: - inputs (tuple[Any, ...]): A batch of examples. Does not represent labels, - which are passed as `targets`. The assumption is that - `model(*inputs)` produces the predictions for the batch. - targets (Tensor): The labels corresponding to the batch `inputs`. This - method is designed to be applied for a loss function, so labels - are required. + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. Returns: - influence_scores (Tensor): Influence scores from the - TracInCPFastRandProj method. Its shape is - `(input_size, train_dataset_size)`, where `input_size` is the - number of examples in the test batch, and `train_dataset_size` is - the number of examples in training dataset `train_dataset`. For - example, `influence_scores[i][j]` is the influence score for the j-th - training example to the i-th input example. + influence_scores (Tensor): Influence scores from the `TracInCPFastRandProj` + method. Its shape is `(input_size, train_dataset_size)`, where `input_size` + is the number of examples in the test batch, and + `train_dataset_size` is the number of examples in + training dataset `train_dataset`. For example: + `influence_scores[i][j]` is the influence score for the j-th training + example to the i-th example in the test batch. """ # TODO: after D35721609 lands, use helper function # `TracInCP._influence_rand_proj` here to avoid duplicated logic - inputs_batch = (*inputs, targets) input_projections = self._get_intermediate_quantities_tracincp_fast_rand_proj( - DataLoader( - _DatasetFromList([inputs_batch]), shuffle=False, batch_size=None - ), + inputs, self.projection_quantities, test=True, ) @@ -1088,18 +1054,19 @@ def _influence( # type: ignore[override] def _get_k_most_influential( # type: ignore[override] self, inputs: Tuple[Any, ...], - targets: Tensor, k: int = 5, proponents: bool = True, ) -> KMostInfluentialResults: r""" Args: - inputs (tuple[Any, ...]): A tuple that represents a batch of examples. It - does not represent labels, which are passed as `targets`. - targets (Tensor): The labels corresponding to the batch `inputs`. This - method is designed to be applied for a loss function, so labels - are required. + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. k (int, optional): The number of proponents or opponents to return per test example. Default: 5 @@ -1120,14 +1087,10 @@ def _get_k_most_influential( # type: ignore[override] influence scores for a different test example, in sorted order. In particular, `influence_scores[i][j]` is the influence score of example `indices[i][j]` in training dataset `train_dataset` - on example `i` in the test batch represented by `inputs` and - `targets`. + on example `i` in the test batch represented by `inputs`. """ - inputs_batch = (*inputs, targets) input_projections = self._get_intermediate_quantities_tracincp_fast_rand_proj( - DataLoader( - _DatasetFromList([inputs_batch]), shuffle=False, batch_size=None - ), + inputs, self.projection_quantities, test=True, ) @@ -1146,7 +1109,7 @@ def _get_k_most_influential( # type: ignore[override] @log_usage() def self_influence( self, - inputs_dataset: Optional[Union[Tuple[Any, ...], DataLoader]] = None, + inputs: Optional[Union[Tuple[Any, ...], DataLoader]] = None, show_progress: bool = False, outer_loop_by_checkpoints: bool = False, ) -> Tensor: @@ -1157,14 +1120,14 @@ def self_influence( example are stored (other than its self influence score) Computes self influence scores for a single batch or a Pytorch `DataLoader` - that yields batches. Note that if `inputs_dataset` is a single batch, this - will call `model` on that single batch, and if `inputs_dataset` yields + that yields batches. Note that if `inputs` is a single batch, this + will call `model` on that single batch, and if `inputs` yields batches, this will call `model` on each batch that is yielded. Therefore, please ensure that for both cases, the batch(es) that `model` is called with are not too large, so that there will not be an out-of-memory error. Args: - inputs_dataset (tuple or DataLoader): Either a single tuple of any, or a + inputs (tuple or DataLoader): Either a single tuple of any, or a `DataLoader`, where each batch yielded is a tuple of any. In either case, the tuple represents a single batch, where the last element is assumed to be the labels for the batch. That is, @@ -1175,14 +1138,14 @@ def self_influence( `train_dataset` argument to `TracInCP.__init__` for more details on the assumed structure of a batch. show_progress (bool, optional): Computation of self influence scores can - take a long time if `inputs_dataset` represents many examples. If + take a long time if `inputs` represents many examples. If `show_progress` is true, the progress of this computation will be displayed. In more detail, this computation will iterate over all checkpoints (provided as the `checkpoints` initialization argument) - and all batches that `inputs_dataset` represents. Therefore, the + and all batches that `inputs` represents. Therefore, the total number of (checkpoint, batch) combinations that need to be iterated over is - (# of checkpoints x # of batches that `inputs_dataset` represents). + (# of checkpoints x # of batches that `inputs` represents). If `show_progress` is True, the total number of such combinations that have been iterated over is displayed. It will try to use tqdm if available for advanced features (e.g. time estimation). @@ -1195,7 +1158,7 @@ def self_influence( Returns: self_influence_scores (Tensor): This is a 1D tensor containing the self - influence scores of all examples in `inputs_dataset`, regardless of + influence scores of all examples in `inputs`, regardless of whether it represents a single batch or a `DataLoader` that yields batches. """ @@ -1218,53 +1181,37 @@ def self_influence( @log_usage() def influence( # type: ignore[override] self, - inputs: Any, - targets: Tensor, + inputs: Optional[Tuple[Any, ...]] = None, k: int = 5, proponents: bool = True, - unpack_inputs: bool = True, ) -> Union[Tensor, KMostInfluentialResults]: r""" This is the key method of this class, and can be run in 2 different modes, - where the mode that is run depends on the arguments passed to this method - - - influence score mode: This mode is used if `inputs` is not None, and `k` is - None. This mode computes the influence score of every example in - training dataset `train_dataset` on every example in the test - batch represented by `inputs` and `targets`. - - - k-most influential mode: This mode is used if `inputs` is not None, and - `k` is not None, and an int. This mode computes the proponents or - opponents of every example in the test batch represented by `inputs` - and `targets`. In particular, for each test example in the test batch, - this mode computes its proponents (resp. opponents), which are the - indices in the training dataset `train_dataset` of the training - examples with the `k` highest (resp. lowest) influence scores on the - test example. Proponents are computed if `proponents` is True. - Otherwise, opponents are computed. For each test example, this method - also returns the actual influence score of each proponent (resp. - opponent) on the test example. - - Note that unlike `TracInCPFast`, this class should *not* be run in self - influence mode. To compute self influence scores when only considering - gradients in the last fully-connected layer, please use `TracInCPFast` instead. + where the mode that is run depends on the arguments passed to this method: + + - influence score mode: This mode is used if `k` is None. This mode computes + the influence score of every example in training dataset `train_dataset` + on every example in the test batch represented by `inputs`. + - k-most influential mode: This mode is used if `k` is not None, and an int. + This mode computes the proponents or opponents of every example in the + test batch represented by `inputs`. In particular, for each test example in + the test batch, this mode computes its proponents (resp. opponents), + which are the indices in the training dataset `train_dataset` of the + training examples with the `k` highest (resp. lowest) influence scores on the + test example. Proponents are computed if `proponents` is True. Otherwise, + opponents are computed. For each test example, this method also returns the + actual influence score of each proponent (resp. opponent) on the test + example. Args: - inputs (Any, optional): If not provided or `None`, the self influence mode - will be run. Otherwise, `inputs` is the test batch that will be - used when running in either influence score or k-most influential - mode. If the argument `unpack_inputs` is False, the - assumption is that `model(inputs)` produces the predictions - for a batch, and `inputs` can be of any type. Otherwise if the - argument `unpack_inputs` is True, the assumption is that - `model(*inputs)` produces the predictions for a batch, and - `inputs` will need to be a tuple. In other words, `inputs` will be - unpacked as an argument when passing to `model`. - Default: None - targets (Tensor): The labels corresponding to the batch `inputs`. This - method is designed to be applied for a loss function, so `targets` - is required. + inputs (tuple): `inputs` is the test batch and is a tuple of + any, where the last element is assumed to be the labels for the + batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. This is the same + assumption made for each batch yielded by training dataset + `train_dataset` - please see its documentation in `__init__` for + more details on the assumed structure of a batch. k (int, optional): If not provided or `None`, the influence score mode will be run. Otherwise, the k-most influential mode will be run, and `k` is the number of proponents / opponents to return per @@ -1274,55 +1221,40 @@ def influence( # type: ignore[override] or opponents (`proponents=False`), if running in k-most influential mode. Default: True - unpack_inputs (bool, optional): Whether to unpack the `inputs` argument to - when passing it to `model`, if `inputs` is a tuple (no unpacking - done otherwise). - Default: True Returns: - The return value of this method depends on which mode is run. - - influence score mode: if this mode is run (`inputs` is not None, `k` is - None), returns a 2D tensor `influence_scores` of shape - `(input_size, train_dataset_size)`, where `input_size` is - the number of examples in the test batch, and - `train_dataset_size` is the number of examples in - training dataset `train_dataset`. In other words, - `influence_scores[i][j]` is the influence score of the `j`-th - example in `train_dataset` on the `i`-th example in the - test batch. - - k-most influential mode: if this mode is run (`inputs` is not None, - `k` is an int), returns a namedtuple `(indices, influence_scores)`. - `indices` is a 2D tensor of shape `(input_size, k)`, where - `input_size` is the number of examples in the test batch. If - computing proponents (resp. opponents), `indices[i][j]` is the - index in training dataset `train_dataset` of the example - with the `j`-th highest (resp. lowest) influence score (out of the - examples in `train_dataset`) on the `i`-th example in the - test batch. `influence_scores` contains the corresponding influence - scores. In particular, `influence_scores[i][j]` is the influence - score of example `indices[i][j]` in `train_dataset` on - example `i` in the test batch represented by `inputs` and - `targets`. + - influence score mode: if this mode is run (`k` is None), returns a 2D + tensor `influence_scores` of shape `(input_size, train_dataset_size)`, + where `input_size` is the number of examples in the test batch, and + `train_dataset_size` is the number of examples in training dataset + `train_dataset`. In other words, `influence_scores[i][j]` is the + influence score of the `j`-th example in `train_dataset` on the `i`-th + example in the test batch. + - k-most influential mode: if this mode is run (`k` is an int), returns + a namedtuple `(indices, influence_scores)`. `indices` is a 2D tensor of + shape `(input_size, k)`, where `input_size` is the number of examples in + the test batch. If computing proponents (resp. opponents), + `indices[i][j]` is the index in training dataset `train_dataset` of the + example with the `j`-th highest (resp. lowest) influence score (out of + the examples in `train_dataset`) on the `i`-th example in the test + batch. `influence_scores` contains the corresponding influence scores. + In particular, `influence_scores[i][j]` is the influence score of example + `indices[i][j]` in `train_dataset` on example `i` in the test batch + represented by `inputs`. """ - msg = ( - "Since `inputs` is None, this suggests `TracInCPFastRandProj` is being " - "used in self influence mode. However, `TracInCPFastRandProj` should not " - "be used to compute self influence scores. If desiring self influence " - "scores which only consider gradients in the last fully-connected layer, " - "please use `TracInCPFast` instead." + assert inputs is not None, ( + "`inputs` argument is required." + "`TracInCPFastRandProj` does not support computing self influence scores" + "Even if it did, one would use the `self_influence` method." + ) + return _influence_route_to_helpers( + self, + inputs, + k, + proponents, ) - assert inputs is not None, msg - - _inputs = _format_inputs(inputs, unpack_inputs) - - if inputs is None: - return self.self_influence(self.train_dataloader) - elif k is None: - return self._influence(_inputs, targets) - else: - return self._get_k_most_influential(_inputs, targets, k, proponents) def _set_projections_tracincp_fast_rand_proj( self, @@ -1442,7 +1374,7 @@ def _process_src_intermediate_quantities_tracincp_fast_rand_proj( def _get_intermediate_quantities_tracincp_fast_rand_proj( self, - inputs_dataset: Union[Tuple[Any, ...], DataLoader], + inputs: Union[Tuple[Any, ...], DataLoader], projection_quantities: Optional[Tuple[torch.Tensor, torch.Tensor]], test: bool = False, ) -> torch.Tensor: @@ -1455,7 +1387,7 @@ def _get_intermediate_quantities_tracincp_fast_rand_proj( specifically, largest dot-product) data structure. Args: - inputs_dataset (Tuple, or DataLoader): Either a single tuple of any, or a + inputs (Tuple, or DataLoader): Either a single tuple of any, or a `DataLoader`, where each batch yielded is a tuple of any. In either case, the tuple represents a single batch, where the last element is assumed to be the labels for the batch. That is, @@ -1493,8 +1425,8 @@ def _get_intermediate_quantities_tracincp_fast_rand_proj( the variable d in the top of page 15 of the TracIn paper: https://arxiv.org/abs/2002.08484. """ - # if `inputs_dataset` is not a `DataLoader`, turn it into one. - inputs_dataset = _format_inputs_dataset(inputs_dataset) + # if `inputs` is not a `DataLoader`, turn it into one. + inputs = _format_inputs_dataset(inputs) # internally, whether `projection_quantities` is None determines whether # any projection will be applied to reduce the dimension of the "embedding" @@ -1529,7 +1461,7 @@ def _get_intermediate_quantities_tracincp_fast_rand_proj( # after loading a checkpoint, we compute the contribution of that # checkpoint, for *all* batches (instead of a single batch). this enables # increased efficiency. - for batch in inputs_dataset: + for batch in inputs: # compute `input_jacobians` and `layer_inputs`, for a given checkpoint # using a helper function. `input_jacobians` is a 2D tensor, @@ -1595,7 +1527,7 @@ def _get_intermediate_quantities_tracincp_fast_rand_proj( @log_usage() def compute_intermediate_quantities( self, - inputs_dataset: Union[Tuple[Any, ...], DataLoader], + inputs: Union[Tuple[Any, ...], DataLoader], ) -> Tensor: """ Computes "embedding" vectors for all examples in a single batch, or a @@ -1611,7 +1543,7 @@ def compute_intermediate_quantities( the batch size is that for large models, large batches do not fit in memory. Args: - inputs_dataset (Tuple, or DataLoader): Either a single tuple of any, or a + inputs (Tuple, or DataLoader): Either a single tuple of any, or a `DataLoader`, where each batch yielded is a tuple of any. In either case, the tuple represents a single batch, where the last element is assumed to be the labels for the batch. That is, @@ -1626,7 +1558,7 @@ def compute_intermediate_quantities( Returns: intermediate_quantities (Tensor): A tensor of dimension (N, D * C), where N is total number of examples in - `inputs_dataset`, C is the number of checkpoints passed as the + `inputs`, C is the number of checkpoints passed as the `checkpoints` argument of `TracInCPFastRandProj.__init__`, and each row represents the vector for an example. Regarding D: Let I be the dimension of the output of the last fully-connected layer times the @@ -1643,5 +1575,5 @@ def compute_intermediate_quantities( https://arxiv.org/pdf/2002.08484.pdf. """ return self._get_intermediate_quantities_tracincp_fast_rand_proj( - inputs_dataset, self.projection_quantities + inputs, self.projection_quantities ) diff --git a/captum/influence/_utils/common.py b/captum/influence/_utils/common.py index e1a6e27f8f..26895f3da0 100644 --- a/captum/influence/_utils/common.py +++ b/captum/influence/_utils/common.py @@ -189,7 +189,6 @@ def _get_k_most_influential_helper( influence_src_dataloader: DataLoader, influence_batch_fn: Callable, inputs: Tuple[Any, ...], - targets: Optional[Tensor], k: int = 5, proponents: bool = True, show_progress: bool = False, @@ -204,13 +203,12 @@ def _get_k_most_influential_helper( influence_src_dataloader (DataLoader): The DataLoader, representing training data, for which we want to compute proponents / opponents. influence_batch_fn (Callable): A callable that will be called via - `influence_batch_fn(inputs, targets, batch)`, where `batch` is a batch + `influence_batch_fn(inputs, batch)`, where `batch` is a batch in the `influence_src_dataloader` argument. - inputs (tuple[Any, ...]): A batch of examples. Does not represent labels, - which are passed as `targets`. - targets (Tensor, optional): If computing TracIn scores on a loss function, - these are the labels corresponding to the batch `inputs`. - Default: None + inputs (tuple[Any, ...]): This argument represents the test batch, and is a + single tuple of any, where the last element is assumed to be the labels + for the batch. That is, `model(*batch[0:-1])` produces the output for + `model`, and `batch[-1]` are the labels, if any. k (int, optional): The number of proponents or opponents to return per test instance. Default: 5 @@ -272,7 +270,7 @@ def _get_k_most_influential_helper( for batch in influence_src_dataloader: # calculate tracin_scores for the batch - batch_tracin_scores = influence_batch_fn(inputs, targets, batch) + batch_tracin_scores = influence_batch_fn(inputs, batch) batch_tracin_scores *= multiplier # get the top-k indices and tracin_scores for the batch diff --git a/tests/influence/_core/test_dataloader.py b/tests/influence/_core/test_dataloader.py index 9613262573..564ccfa4df 100644 --- a/tests/influence/_core/test_dataloader.py +++ b/tests/influence/_core/test_dataloader.py @@ -10,6 +10,7 @@ from parameterized import parameterized from tests.helpers.basic import assertTensorAlmostEqual, BaseTest from tests.influence._utils.common import ( + _format_batch_into_tuple, build_test_name_func, DataInfluenceConstructor, get_random_model_and_data, @@ -76,7 +77,8 @@ def test_tracin_dataloader( ) train_scores = tracin.influence( - test_samples, test_labels, k=None, unpack_inputs=unpack_inputs + _format_batch_into_tuple(test_samples, test_labels, unpack_inputs), + k=None, ) tracin_dataloader = tracin_constructor( @@ -88,7 +90,8 @@ def test_tracin_dataloader( ) train_scores_dataloader = tracin_dataloader.influence( - test_samples, test_labels, k=None, unpack_inputs=unpack_inputs + _format_batch_into_tuple(test_samples, test_labels, unpack_inputs), + k=None, ) assertTensorAlmostEqual( diff --git a/tests/influence/_core/test_tracin_intermediate_quantities.py b/tests/influence/_core/test_tracin_intermediate_quantities.py index 5d1dde3ab3..8ec0bd4091 100644 --- a/tests/influence/_core/test_tracin_intermediate_quantities.py +++ b/tests/influence/_core/test_tracin_intermediate_quantities.py @@ -12,6 +12,7 @@ from parameterized import parameterized from tests.helpers.basic import assertTensorAlmostEqual, BaseTest from tests.influence._utils.common import ( + _format_batch_into_tuple, build_test_name_func, DataInfluenceConstructor, get_random_model_and_data, @@ -224,25 +225,13 @@ def test_tracin_intermediate_quantities_consistent( ) # compute influence scores without using `compute_intermediate_quantities` + test_batch = _format_batch_into_tuple( + test_features, test_labels, unpack_inputs + ) scores = tracin.influence( - test_features, test_labels, unpack_inputs=unpack_inputs + test_batch, ) - # compute influence scores using `compute_intermediate_quantities` - # we combine `test_features` and `test_labels` into a single tuple - # `test_batch` to pass to the model, with the assumption that - # `model(test_batch[0:-1]` produces the predictions, and `test_batch[-1]` - # are the labels. We do this due to the assumptions made by the - # `compute_intermediate_quantities` method. Therefore, how we - # form `test_batch` depends on whether `unpack_inputs` is True or False - if not unpack_inputs: - # `test_features` is a Tensor - test_batch = (test_features, test_labels) - else: - # `test_features` is a tuple, so we unpack it to place in tuple, - # along with `test_labels` - test_batch = (*test_features, test_labels) # type: ignore[assignment] - # the influence score is the dot product of intermediate quantities intermediate_quantities_scores = torch.matmul( intermediate_quantities_tracin.compute_intermediate_quantities( diff --git a/tests/influence/_core/test_tracin_k_most_influential.py b/tests/influence/_core/test_tracin_k_most_influential.py index 54295b43dd..2e7dbf8184 100644 --- a/tests/influence/_core/test_tracin_k_most_influential.py +++ b/tests/influence/_core/test_tracin_k_most_influential.py @@ -8,6 +8,7 @@ from parameterized import parameterized from tests.helpers.basic import assertTensorAlmostEqual, BaseTest from tests.influence._utils.common import ( + _format_batch_into_tuple, build_test_name_func, DataInfluenceConstructor, get_random_model_and_data, @@ -107,15 +108,14 @@ def test_tracin_k_most_influential( ) train_scores = tracin.influence( - test_samples, test_labels, k=None, unpack_inputs=unpack_inputs + _format_batch_into_tuple(test_samples, test_labels, unpack_inputs), + k=None, ) sort_idx = torch.argsort(train_scores, dim=1, descending=proponents)[:, 0:k] idx, _train_scores = tracin.influence( - test_samples, - test_labels, + _format_batch_into_tuple(test_samples, test_labels, unpack_inputs), k=k, proponents=proponents, - unpack_inputs=unpack_inputs, ) for i in range(len(idx)): # check that idx[i] is correct diff --git a/tests/influence/_core/test_tracin_regression.py b/tests/influence/_core/test_tracin_regression.py index 147566bc22..27b6ec9f5d 100644 --- a/tests/influence/_core/test_tracin_regression.py +++ b/tests/influence/_core/test_tracin_regression.py @@ -183,9 +183,9 @@ def test_tracin_regression( criterion, ) - train_scores = tracin.influence(train_inputs, train_labels) + train_scores = tracin.influence((train_inputs, train_labels)) idx, _ = tracin.influence( - train_inputs, train_labels, k=len(dataset), proponents=True + (train_inputs, train_labels), k=len(dataset), proponents=True ) # check that top influence is one with maximal value # (and hence gradient) @@ -193,9 +193,9 @@ def test_tracin_regression( self.assertEqual(idx[i][0], 15) # check influence scores of test data - test_scores = tracin.influence(test_inputs, test_labels) + test_scores = tracin.influence((test_inputs, test_labels)) idx, _ = tracin.influence( - test_inputs, test_labels, k=len(test_inputs), proponents=True + (test_inputs, test_labels), k=len(test_inputs), proponents=True ) # check that top influence is one with maximal value # (and hence gradient) @@ -226,17 +226,17 @@ def test_tracin_regression( sample_wise_grads_per_batch=True, ) - train_scores = tracin.influence(train_inputs, train_labels) + train_scores = tracin.influence((train_inputs, train_labels)) train_scores_sample_wise_trick = tracin_sample_wise_trick.influence( - train_inputs, train_labels + (train_inputs, train_labels) ) assertTensorAlmostEqual( self, train_scores, train_scores_sample_wise_trick ) - test_scores = tracin.influence(test_inputs, test_labels) + test_scores = tracin.influence((test_inputs, test_labels)) test_scores_sample_wise_trick = tracin_sample_wise_trick.influence( - test_inputs, test_labels + (test_inputs, test_labels) ) assertTensorAlmostEqual( self, test_scores, test_scores_sample_wise_trick @@ -288,7 +288,7 @@ def test_tracin_regression_1D_numerical( criterion, ) - train_scores = tracin.influence(train_inputs, train_labels, k=None) + train_scores = tracin.influence((train_inputs, train_labels), k=None) r""" Derivation for gradient / resulting TracIn score: @@ -382,9 +382,9 @@ def test_tracin_identity_regression( # check influence scores of training data - train_scores = tracin.influence(train_inputs, train_labels) + train_scores = tracin.influence((train_inputs, train_labels)) idx, _ = tracin.influence( - train_inputs, train_labels, k=len(dataset), proponents=True + (train_inputs, train_labels), k=len(dataset), proponents=True ) # check that top influence for an instance is itself @@ -415,9 +415,9 @@ def test_tracin_identity_regression( sample_wise_grads_per_batch=True, ) - train_scores = tracin.influence(train_inputs, train_labels) + train_scores = tracin.influence((train_inputs, train_labels)) train_scores_tracin_sample_wise_trick = ( - tracin_sample_wise_trick.influence(train_inputs, train_labels) + tracin_sample_wise_trick.influence((train_inputs, train_labels)) ) assertTensorAlmostEqual( self, train_scores, train_scores_tracin_sample_wise_trick @@ -496,5 +496,5 @@ def test_loss_fn(input, target): ) # check influence scores of training data. they should all be 0 - train_scores = tracin.influence(train_inputs, train_labels, k=None) + train_scores = tracin.influence((train_inputs, train_labels), k=None) assertTensorAlmostEqual(self, train_scores, torch.zeros(train_scores.shape)) diff --git a/tests/influence/_core/test_tracin_self_influence.py b/tests/influence/_core/test_tracin_self_influence.py index 924f977cc8..e79656e08f 100644 --- a/tests/influence/_core/test_tracin_self_influence.py +++ b/tests/influence/_core/test_tracin_self_influence.py @@ -8,6 +8,7 @@ from parameterized import parameterized from tests.helpers.basic import assertTensorAlmostEqual, BaseTest from tests.influence._utils.common import ( + _format_batch_into_tuple, build_test_name_func, DataInfluenceConstructor, get_random_model_and_data, @@ -108,10 +109,10 @@ def test_tracin_self_influence( criterion, ) train_scores = tracin.influence( - train_dataset.samples, - train_dataset.labels, + _format_batch_into_tuple( + train_dataset.samples, train_dataset.labels, unpack_inputs + ), k=None, - unpack_inputs=unpack_inputs, ) # calculate self_tracin_scores self_tracin_scores = tracin.self_influence( diff --git a/tests/influence/_core/test_tracin_show_progress.py b/tests/influence/_core/test_tracin_show_progress.py index 1c94cb7ce9..82bdf480b8 100644 --- a/tests/influence/_core/test_tracin_show_progress.py +++ b/tests/influence/_core/test_tracin_show_progress.py @@ -178,8 +178,7 @@ def test_tracin_show_progress( elif mode == "influence": tracin.influence( - test_samples, - test_labels, + (test_samples, test_labels), k=None, show_progress=True, ) @@ -196,8 +195,7 @@ def test_tracin_show_progress( elif mode == "k-most": tracin.influence( - test_samples, - test_labels, + (test_samples, test_labels), k=2, proponents=True, show_progress=True, @@ -218,8 +216,7 @@ def test_tracin_show_progress( mock_stderr.truncate(0) tracin.influence( - test_samples, - test_labels, + (test_samples, test_labels), k=2, proponents=False, show_progress=True, diff --git a/tests/influence/_core/test_tracin_validation.py b/tests/influence/_core/test_tracin_validation.py index 5691097af4..5c34c9cc17 100644 --- a/tests/influence/_core/test_tracin_validation.py +++ b/tests/influence/_core/test_tracin_validation.py @@ -63,4 +63,4 @@ def test_tracin_require_inputs_dataset( batch_size=1, ) with self.assertRaisesRegex(AssertionError, "required."): - tracin.influence(None, test_labels, k=None, unpack_inputs=False) + tracin.influence(None, k=None) diff --git a/tests/influence/_core/test_tracin_xor.py b/tests/influence/_core/test_tracin_xor.py index 2a26623ef7..5c5a0bb760 100644 --- a/tests/influence/_core/test_tracin_xor.py +++ b/tests/influence/_core/test_tracin_xor.py @@ -258,7 +258,7 @@ def test_tracin_xor( batch_size, criterion, ) - test_scores = tracin.influence(testset, testlabels) + test_scores = tracin.influence((testset, testlabels)) idx = torch.argsort(test_scores, dim=1, descending=True) # check that top 5 influences have matching binary classification for i in range(len(idx)): @@ -288,9 +288,9 @@ def test_tracin_xor( criterion, sample_wise_grads_per_batch=True, ) - test_scores = tracin.influence(testset, testlabels) + test_scores = tracin.influence((testset, testlabels)) test_scores_sample_wise_trick = tracin_sample_wise_trick.influence( - testset, testlabels + (testset, testlabels) ) assertTensorAlmostEqual( self, test_scores, test_scores_sample_wise_trick diff --git a/tests/influence/_utils/common.py b/tests/influence/_utils/common.py index 26a5146785..f65f8b6b8d 100644 --- a/tests/influence/_utils/common.py +++ b/tests/influence/_utils/common.py @@ -2,7 +2,7 @@ import os import unittest from functools import partial -from typing import Callable, Iterator, List, Optional, Union +from typing import Callable, Iterator, List, Optional, Tuple, Union import torch import torch.nn as nn @@ -14,6 +14,7 @@ ) from parameterized import parameterized from parameterized.parameterized import param +from torch import Tensor from torch.nn import Module from torch.utils.data import DataLoader, Dataset @@ -366,3 +367,12 @@ def build_test_name_func(args_to_skip: Optional[List[str]] = None): """ return partial(generate_test_name, args_to_skip=args_to_skip) + + +def _format_batch_into_tuple( + inputs: Union[Tuple, Tensor], targets: Tensor, unpack_inputs: bool +): + if unpack_inputs: + return (*inputs, targets) + else: + return (inputs, targets) diff --git a/tutorials/TracInCP_Tutorial.ipynb b/tutorials/TracInCP_Tutorial.ipynb index af5fba11a6..733c92df2a 100644 --- a/tutorials/TracInCP_Tutorial.ipynb +++ b/tutorials/TracInCP_Tutorial.ipynb @@ -46,8 +46,7 @@ }, "source": [ "## Overview of different implementations of the TracInCP method\n", - "Currently, Captum offers 3 implementations, all of which implement the same API. More specifically, they define an `influence` method, which can be used in 3 different modes:\n", - "- self influence mode: calculates the self influence scores for all examples in the training dataset.\n", + "Currently, Captum offers 3 implementations, all of which implement the same API. More specifically, they define an `influence` method, which can be used in 2 different modes:\n", "- influence score mode: given a batch of test examples, calculates the influence score of every example in the training dataset on every test example.\n", "- top-k most influential mode: given a batch of test examples, calculates either the proponents or opponents of every test example, as well as their corresponding influence scores.\n", "\n", @@ -249,7 +248,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "code_folding": [], "executionStartTime": 1646008674539, @@ -459,7 +458,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": { "code_folding": [], "customInput": null, @@ -602,7 +601,7 @@ "showInput": false }, "source": [ - "Now, we define `test_examples_batch`, the batch of test examples to identify influential examples for, and also store the correct as well as predicted labels." + "Now, we define `test_examples_features`, the features for a batch of test examples to identify influential examples for, and also store the correct as well as predicted labels." ] }, { @@ -619,8 +618,8 @@ "outputs": [], "source": [ "test_examples_indices = [0,1,2,3]\n", - "test_examples_batch = torch.stack([test_dataset[i][0] for i in test_examples_indices])\n", - "test_examples_predicted_probs, test_examples_predicted_labels = torch.max(F.softmax(net(test_examples_batch), dim=1), dim=1)\n", + "test_examples_features = torch.stack([test_dataset[i][0] for i in test_examples_indices])\n", + "test_examples_predicted_probs, test_examples_predicted_labels = torch.max(F.softmax(net(test_examples_features), dim=1), dim=1)\n", "test_examples_true_labels = torch.Tensor([test_dataset[i][1] for i in test_examples_indices]).long()" ] }, @@ -719,14 +718,16 @@ }, "source": [ "#### Compute the proponents / opponents using `TracInCPFast`\n", - "Now, we call the `influence` method of `tracin_cp_fast` to compute the influential examples of the test examples in `test_examples_batch`. We need to specify whether we want proponents or opponents via the `proponents` boolean argument, and how many influential examples to return per test example via the `k` argument. Note that `k` must be specified. Otherwise, the \"influence score\" mode will be run. This call should take < 2 minutes.\n", + "Now, we call the `influence` method of `tracin_cp_fast` to compute the influential examples of the test examples represented by `test_examples_features` and `test_examples_true_labels`. We need to specify whether we want proponents or opponents via the `proponents` boolean argument, and how many influential examples to return per test example via the `k` argument. Note that `k` must be specified. Otherwise, the \"influence score\" mode will be run. This call should take < 2 minutes.\n", + "\n", + "Note that we pass the test examples as a *single* tuple. This is because for all implementations, when we pass a single batch, `batch` to the `influence` method, we assume that `batch[-1]` has the labels for the batch, and `model(*(batch[0:-1]))` produces the predictions for the batch, so that `batch[0:-1]` contains the features for the batch. This convention is was introduced in a recent API change.\n", "\n", "This call returns a `namedtuple` with ordered elements `(indices, influence_scores)`. `indices` is a 2D tensor of shape `(test_batch_size, k)`, where `test_batch_size` is the number of test examples in `test_examples_batch`. `influence_scores` is of the same shape, but stores the influence scores of the proponents / opponents for each test example in sorted order. For example, if `proponents` is `True`, `influence_scores[i][j]` is the influence score of the training example with the `j`-th most positive influence score on test example `i`." ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": { "code_folding": [], "customInput": null, @@ -742,7 +743,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Computed proponents / opponents over a dataset of 50000 examples in 0.88 minutes\n" + "Computed proponents / opponents over a dataset of 50000 examples in 1.11 minutes\n" ] } ], @@ -750,10 +751,10 @@ "k = 10\n", "start_time = datetime.datetime.now()\n", "proponents_indices, proponents_influence_scores = tracin_cp_fast.influence(\n", - " test_examples_batch, test_examples_true_labels, k=k, proponents=True\n", + " (test_examples_features, test_examples_true_labels), k=k, proponents=True\n", ")\n", "opponents_indices, opponents_influence_scores = tracin_cp_fast.influence(\n", - " test_examples_batch, test_examples_true_labels, k=k, proponents=False\n", + " (test_examples_features, test_examples_true_labels), k=k, proponents=False\n", ")\n", "total_minutes = (datetime.datetime.now() - start_time).total_seconds() / 60.0\n", "print(\n", @@ -1102,7 +1103,7 @@ ], "source": [ "display_proponents_and_opponents(\n", - " test_examples_batch,\n", + " test_examples_features,\n", " proponents_indices,\n", " opponents_indices,\n", " test_examples_true_labels,\n", @@ -1144,7 +1145,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": { "code_folding": [], "customInput": null, @@ -1160,7 +1161,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Performed pre-processing of a dataset of 50000 examples in 5.92 minutes\n" + "Performed pre-processing of a dataset of 50000 examples in 4.98 minutes\n" ] } ], @@ -1201,7 +1202,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": { "code_folding": [], "executionStopTime": 1645988498023, @@ -1214,7 +1215,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Computed proponents / opponents over a dataset of 50000 examples in 0.00 minutes\n" + "Computed proponents / opponents over a dataset of 50000 examples in 0.01 minutes\n" ] } ], @@ -1222,10 +1223,10 @@ "k = 10\n", "start_time = datetime.datetime.now()\n", "proponents_indices, proponents_influence_scores = tracin_cp_fast_rand_proj.influence(\n", - " test_examples_batch, test_examples_true_labels, k=k, proponents=True\n", + " (test_examples_features, test_examples_true_labels), k=k, proponents=True\n", ")\n", "opponents_indices, opponents_influence_scores = tracin_cp_fast_rand_proj.influence(\n", - " test_examples_batch, test_examples_true_labels, k=k, proponents=False\n", + " (test_examples_features, test_examples_true_labels), k=k, proponents=False\n", ")\n", "total_minutes = (datetime.datetime.now() - start_time).total_seconds() / 60.0\n", "print(\n", @@ -1478,7 +1479,7 @@ ], "source": [ "display_proponents_and_opponents(\n", - " test_examples_batch,\n", + " test_examples_features,\n", " proponents_indices,\n", " opponents_indices,\n", " test_examples_true_labels,\n", @@ -1623,7 +1624,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": { "code_folding": [], "customInput": null, @@ -1639,7 +1640,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Generated incorrect labels in 0.36 minutes\n" + "Generated incorrect labels in 0.42 minutes\n" ] } ], @@ -1837,7 +1838,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "metadata": { "code_folding": [], "customInput": null, @@ -1999,7 +2000,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "computed self influence scores for 50000 examples in 0.48 minutes\n" + "computed self influence scores for 50000 examples in 0.50 minutes\n" ] } ],