TypeError: can't pickle _thread.lock objects

[jueqiw]

❓ Questions and Help

What is your question?

Hi, everyone. I run into this problem but I really do not how to solve it. I've been stuck up there for three or four hours.

Code

This is the error:

Traceback (most recent call last):
  File "/home/jq/PycharmProjects/Unet/Code/Lit_train.py", line 50, in <module>
    trainer.fit(model)
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 859, in fit
    self.single_gpu_train(model)
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_parts.py", line 503, in single_gpu_train
    self.run_pretrain_routine(model)
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1015, in run_pretrain_routine
    self.train()
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 347, in train
    self.run_training_epoch()
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 451, in run_training_epoch
    self.run_evaluation(test_mode=self.testing)
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 391, in run_evaluation
    self.log_metrics(log_metrics, {})
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/logging.py", line 74, in log_metrics
    self.logger.save()
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/utilities/distributed.py", line 10, in wrapped_fn
    return fn(*args, **kwargs)
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/loggers/tensorboard.py", line 161, in save
    save_hparams_to_yaml(hparams_file, self.hparams)
  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/core/saving.py", line 151, in save_hparams_to_yaml
    yaml.dump(hparams, fp)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/__init__.py", line 290, in dump
    return dump_all([data], stream, Dumper=Dumper, **kwds)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/__init__.py", line 278, in dump_all
    dumper.represent(data)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 27, in represent
    node = self.represent_data(data)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 48, in represent_data
    node = self.yaml_representers[data_types[0]](self, data)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 207, in represent_dict
    return self.represent_mapping('tag:yaml.org,2002:map', data)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping
    node_value = self.represent_data(item_value)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 52, in represent_data
    node = self.yaml_multi_representers[data_type](self, data)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 343, in represent_object
    'tag:yaml.org,2002:python/object:'+function_name, state)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping
    node_value = self.represent_data(item_value)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 52, in represent_data
    node = self.yaml_multi_representers[data_type](self, data)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 346, in represent_object
    return self.represent_sequence(tag+function_name, args)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 92, in represent_sequence
    node_item = self.represent_data(item)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 52, in represent_data
    node = self.yaml_multi_representers[data_type](self, data)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 343, in represent_object
    'tag:yaml.org,2002:python/object:'+function_name, state)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 118, in represent_mapping
    node_value = self.represent_data(item_value)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 52, in represent_data
    node = self.yaml_multi_representers[data_type](self, data)
  File "/home/jq/.local/lib/python3.6/site-packages/yaml/representer.py", line 317, in represent_object
    reduce = data.__reduce_ex__(2)
TypeError: can't pickle _thread.lock objects

                                                         Exception ignored in: <object repr() failed>
Traceback (most recent call last):
  File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1086, in __del__
  File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1293, in close
  File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1471, in display
  File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1089, in __repr__
  File "/home/jq/.local/lib/python3.6/site-packages/tqdm/std.py", line 1433, in format_dict
TypeError: 'NoneType' object is not iterable

This is my lightning model code:

    def training_step(self, batch, batch_idx):
        inputs, targets = batch["img"][DATA], batch["label"][DATA]
        logits = self(inputs)
        prob = torch.sigmoid(logits)
        dice, iou = get_dice_score(prob, targets)
        if int(batch_idx) != 0 and self.hparams.show_plot and int(batch_idx) % 15 == 0:
            slices = BrainSlices(inputs, targets, logits)
            slices.visualize(int(batch_idx), self.current_epoch,
                             outdir=Path(__file__).resolve().parent / "log" / "plot")
        loss = F.binary_cross_entropy_with_logits(logits, targets)
        tensorboard_logs = {"train_loss": loss, "train_IoU": iou.mean(), "train_dice": dice.mean()}
        return {'loss': loss, "log": tensorboard_logs}

    def validation_step(self, batch, batch_id):
        inputs, targets = batch["img"][DATA], batch["label"][DATA]
        logits = self(inputs)
        prob = torch.sigmoid(logits)
        loss = F.binary_cross_entropy_with_logits(logits, targets)
        dice, iou = get_dice_score(prob, targets)
        tensorboard_logs = {"val_loss": loss, "val_IoU": iou.mean(), "val_dice": dice.mean()}
        return {'val_loss': loss, 'log': tensorboard_logs}

    # Called at the end of the validation epoch with the outputs of all validation steps.
    def validation_epoch_end(self, outputs):
        # torch.stack: Concatenates sequence of tensors along a new dimension.
        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
        tensorboard_logs = {'val_loss_avg': avg_loss}
        return {'val_loss': avg_loss, 'log': tensorboard_logs}

# default used by the Trainer
    checkpoint_callback = ModelCheckpoint(
        filepath="log/checkpoint/{epoch}-{val_loss:.2f}",
        save_top_k=1,
        verbose=True,
        monitor='val_loss',
        mode='min',
        prefix=''
    )

    early_stop_callback = EarlyStopping(
        monitor='val_loss',
        patience=3,
        strict=False,
        verbose=False,
        mode='min'
    )

    model = LitUnet(args)

    trainer = Trainer.from_argparse_args(
        args,
        gpus=1,
        check_val_every_n_epoch=1,
        checkpoint_callback=checkpoint_callback,
        early_stop_callback=early_stop_callback,
        # runs 1 train, val, test  batch and program ends
        fast_dev_run=True,
        default_root_dir='log/checkpoint',
        profiler=True
    )

    trainer.fit(model)

[github-actions]

Hi! thanks for your contribution!, great first issue!

[williamFalcon]

https://pytorch-lightning.readthedocs.io/en/latest/multi_gpu.html?highlight=pickle#pickle-errors

[williamFalcon]

Looks like something you are trying to save to hparams is not picklable?

@Borda should we add a warning for users when this happens? Maybe recursive iterate or something cheap?

  File "/home/jq/.local/lib/python3.6/site-packages/pytorch_lightning/core/saving.py", line 151, in save_hparams_to_yaml
    yaml.dump(hparams, fp)

[jueqiw]

That's really weird, because this is my hparams. I cannot find anyone which is not pickable:

    @staticmethod
    def add_model_specific_args(parent_parser):
        parser = ArgumentParser(parents=[parent_parser], add_help=False)
        parser.add_argument('-e', '--epochs', type=int, default=10000, help='Number of epochs', dest='epochs')
        parser.add_argument('-b', '--batch_size', type=int, default=1, help='Batch size', dest='batch_size')
        parser.add_argument('-l', '--learning_rate', type=float, default=1e-3, help='Learning rate')
        parser.add_argument('-n', '--normalization', type=str, default="Group", help='the way of normalization')
        parser.add_argument('-d', '--down_sample', type=str, default="max", help="the way to down sample")
        parser.add_argument('--loss', type=str, default="BCEWL", help='Loss Function')
        # parser.add_argument('-r', '--run', dest='run', type=int, default=1, help='run times')
        parser.add_argument('-p', '--show_plot', type=bool, default=False, help='whether to plot the figure')
        return parser

[Borda]

@wendy-xiaozong mind share complete example or better minimal example so we can reproduce it also on our side?

[xor-xor]

Had the same issue recently. I guess it boils down to the difference between points (2)
and (3) here, where in my case (3) is more convenient, but it has a problematic side effect of adding TensorBoardLogger and ModelCheckpoint instances to hparams when I use logger and checkpoint_callback kwargs in Trainer.from_argparse_args call.

So as a kind of "middle ground" between (2) and (3), before assigning hparams to self.hparams in __init__ of my LightningModule subclass, I've just added:

delattr(hparams, "logger")
delattr(hparams, "checkpoint_callback")

Maybe not very elegant, but it solved the issue for me.

[Borda]

Had the same issue recently. I guess it boils down to the difference between points (2)
and (3) here, where in my case (3) is more convenient, but it has a problematic side effect of adding TensorBoardLogger and ModelCheckpoint instances to hparams when I use logger and checkpoint_callback kwargs in Trainer.from_argparse_args call.

so you had logger and checkpoint_callback as init arguments of your LightningModule?

[xor-xor]

Well, not as distinct, explicitly defined args (if this is what you mean..?) - but as properties of hparams, which is an instance of argparse.Namespace at this point. Here's a simplified (hopefully, not too simplified 😉 ) example:

import os
import argparse

import pytorch_lightning as pl
from pytorch_lightning.callbacks import ModelCheckpoint
from pytorch_lightning.loggers import TensorBoardLogger


class MyModel(pl.LightningModule):
    def __init__(self, hparams):
        super().__init__()

        # added to mitigate "can't pickle _thread.lock objects" problem
        delattr(hparams, "logger")
        delattr(hparams, "checkpoint_callback")

        self.hparams = hparams
        # ... + futher initialization

    @staticmethod
    def add_model_specific_args(parent_parser):
        parser = argparse.ArgumentParser(parents=[parent_parser], add_help=False)
        parser.add_argument("--train-file-path", type=str, required=True)
        parser.add_argument("--val-file-path", type=str, required=True)
        # ... + other model-specific args
        return parser

    # ... + other model-specific methods, like e.g. prepare_data, train_dataloader etc.


def run():
    parser = argparse.ArgumentParser()
    parser.add_argument("--run-id", type=str, required=True)
    parser.add_argument("--logger-path", type=str, required=True)
    parser.add_argument("--checkpoint-path", type=str, required=True)
    # ... + other program-specific args, like e.g. seed, num_gpus

    parser = MyModel.add_model_specific_args(parser)

    args, _ = parser.parse_known_args()

    logger = TensorBoardLogger(save_dir=args.logger_path, name=args.run_id)
    checkpoint_callback = ModelCheckpoint(
        filepath=os.path.join(args.checkpoint_path, args.run_id, "{epoch}"),
    )

    trainer = pl.Trainer.from_argparse_args(
        args,
        weights_summary=None,
        logger=logger,
        checkpoint_callback=checkpoint_callback,
        # ... + other trainer kwargs, like e.g. num_gpus, num_train_epochs etc.
    )

    model = MyModel(args)
    trainer.fit(model)


if __name__ == "__main__":
    run()

[jueqiw]

@xor-xor Not sure whether we run into the same problem, I discovered my mistake was that I have written this in other files in my code

parser = ArgumentParser(add_help=False)
hparams = parser.parse_args()

And import that files, which caused this error.

Now my code runs perfectly, thank everyone