From 32b2340b6e35fca60c838c55d73dd8c7aaedae03 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 17 Oct 2022 23:42:39 +0200 Subject: [PATCH 1/9] [Better scheduler docs] Improve usage examples of schedulers --- README.md | 6 +----- docs/source/quicktour.mdx | 2 +- src/diffusers/pipeline_flax_utils.py | 2 +- src/diffusers/pipeline_utils.py | 2 +- src/diffusers/pipelines/stable_diffusion/README.md | 8 ++------ tests/test_pipelines.py | 10 +++++----- 6 files changed, 11 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 734cba6efd9e..030626b253be 100644 --- a/README.md +++ b/README.md @@ -131,11 +131,7 @@ it before the pipeline and pass it to `from_pretrained`. ```python from diffusers import LMSDiscreteScheduler -lms = LMSDiscreteScheduler( - beta_start=0.00085, - beta_end=0.012, - beta_schedule="scaled_linear" -) +lms = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") pipe = StableDiffusionPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx index 9574ecac4a6a..0ea30ea2f3a3 100644 --- a/docs/source/quicktour.mdx +++ b/docs/source/quicktour.mdx @@ -122,7 +122,7 @@ you could use it as follows: ```python >>> from diffusers import LMSDiscreteScheduler ->>> scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") +>>> scheduler = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") >>> generator = StableDiffusionPipeline.from_pretrained( ... "CompVis/stable-diffusion-v1-4", scheduler=scheduler, use_auth_token=AUTH_TOKEN diff --git a/src/diffusers/pipeline_flax_utils.py b/src/diffusers/pipeline_flax_utils.py index d55338b50343..cb204d179cb6 100644 --- a/src/diffusers/pipeline_flax_utils.py +++ b/src/diffusers/pipeline_flax_utils.py @@ -271,7 +271,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P >>> # Download pipeline, but overwrite scheduler >>> from diffusers import LMSDiscreteScheduler - >>> scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") + >>> scheduler = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") >>> pipeline = FlaxDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", scheduler=scheduler) ``` """ diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py index d3e6113dac8e..f6ee8b697752 100644 --- a/src/diffusers/pipeline_utils.py +++ b/src/diffusers/pipeline_utils.py @@ -343,7 +343,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P >>> # Download pipeline, but overwrite scheduler >>> from diffusers import LMSDiscreteScheduler - >>> scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") + >>> scheduler = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") >>> pipeline = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", scheduler=scheduler) ``` """ diff --git a/src/diffusers/pipelines/stable_diffusion/README.md b/src/diffusers/pipelines/stable_diffusion/README.md index 47c38acbdb35..891f62f2d57c 100644 --- a/src/diffusers/pipelines/stable_diffusion/README.md +++ b/src/diffusers/pipelines/stable_diffusion/README.md @@ -72,7 +72,7 @@ image.save("astronaut_rides_horse.png") # make sure you're logged in with `huggingface-cli login` from diffusers import StableDiffusionPipeline, DDIMScheduler -scheduler = DDIMScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", clip_sample=False, set_alpha_to_one=False) +scheduler = DDIMScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") pipe = StableDiffusionPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", @@ -91,11 +91,7 @@ image.save("astronaut_rides_horse.png") # make sure you're logged in with `huggingface-cli login` from diffusers import StableDiffusionPipeline, LMSDiscreteScheduler -lms = LMSDiscreteScheduler( - beta_start=0.00085, - beta_end=0.012, - beta_schedule="scaled_linear" -) +lms = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") pipe = StableDiffusionPipeline.from_pretrained( "CompVis/stable-diffusion-v1-4", diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 69a45c4247aa..6d4f64943500 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -512,7 +512,7 @@ def test_stable_diffusion_no_safety_checker(self): def test_stable_diffusion_k_lms(self): device = "cpu" # ensure determinism for the device-dependent torch.Generator unet = self.dummy_cond_unet - scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") + scheduler = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") vae = self.dummy_vae bert = self.dummy_text_encoder tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") @@ -557,7 +557,7 @@ def test_stable_diffusion_k_lms(self): def test_stable_diffusion_attention_chunk(self): device = "cpu" # ensure determinism for the device-dependent torch.Generator unet = self.dummy_cond_unet - scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") + scheduler = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") vae = self.dummy_vae bert = self.dummy_text_encoder tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") @@ -842,7 +842,7 @@ def test_stable_diffusion_img2img_multiple_init_images(self): def test_stable_diffusion_img2img_k_lms(self): device = "cpu" # ensure determinism for the device-dependent torch.Generator unet = self.dummy_cond_unet - scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") + scheduler = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") vae = self.dummy_vae bert = self.dummy_text_encoder @@ -1890,7 +1890,7 @@ def test_stable_diffusion_img2img_pipeline_k_lms(self): init_image = init_image.resize((768, 512)) expected_image = np.array(expected_image, dtype=np.float32) / 255.0 - lms = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") + lms = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") model_id = "CompVis/stable-diffusion-v1-4" pipe = StableDiffusionImg2ImgPipeline.from_pretrained( @@ -1979,7 +1979,7 @@ def test_stable_diffusion_inpaint_pipeline_k_lms(self): ) expected_image = np.array(expected_image, dtype=np.float32) / 255.0 - lms = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") + lms = LMSDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") model_id = "CompVis/stable-diffusion-v1-4" pipe = StableDiffusionInpaintPipeline.from_pretrained( From 1a514b7468884cba226113f6b94f418e7467d899 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 17 Oct 2022 23:51:18 +0200 Subject: [PATCH 2/9] finish --- examples/dreambooth/train_dreambooth.py | 4 +--- examples/text_to_image/train_text_to_image.py | 5 +---- examples/textual_inversion/textual_inversion.py | 8 +------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/examples/dreambooth/train_dreambooth.py b/examples/dreambooth/train_dreambooth.py index ca39aeff236b..2005edead110 100644 --- a/examples/dreambooth/train_dreambooth.py +++ b/examples/dreambooth/train_dreambooth.py @@ -414,9 +414,7 @@ def main(): eps=args.adam_epsilon, ) - noise_scheduler = DDPMScheduler( - beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000 - ) + noise_scheduler = DDPMScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") train_dataset = DreamBoothDataset( instance_data_root=args.instance_data_dir, diff --git a/examples/text_to_image/train_text_to_image.py b/examples/text_to_image/train_text_to_image.py index c3c3c75438d0..9b5f46ad5e35 100644 --- a/examples/text_to_image/train_text_to_image.py +++ b/examples/text_to_image/train_text_to_image.py @@ -373,10 +373,7 @@ def main(): eps=args.adam_epsilon, ) - # TODO (patil-suraj): load scheduler using args - noise_scheduler = DDPMScheduler( - beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, tensor_format="pt" - ) + noise_scheduler = DDPMScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") # Get the datasets: you can either provide your own training and evaluation files (see below) # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). diff --git a/examples/textual_inversion/textual_inversion.py b/examples/textual_inversion/textual_inversion.py index 18469af3d4ae..aabf1aa48326 100644 --- a/examples/textual_inversion/textual_inversion.py +++ b/examples/textual_inversion/textual_inversion.py @@ -419,13 +419,7 @@ def main(): eps=args.adam_epsilon, ) - # TODO (patil-suraj): load scheduler using args - noise_scheduler = DDPMScheduler( - beta_start=0.00085, - beta_end=0.012, - beta_schedule="scaled_linear", - num_train_timesteps=1000, - ) + noise_scheduler = DDPMScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler") train_dataset = TextualInversionDataset( data_root=args.train_data_dir, From 21f4bb6da1e601ad7555f165ae23a65d4ca954e7 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 31 Oct 2022 10:47:21 +0000 Subject: [PATCH 3/9] fix warnings and add test --- src/diffusers/configuration_utils.py | 12 +++++ tests/test_config.py | 66 ++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py index 6df103823086..55867c1a3e2f 100644 --- a/src/diffusers/configuration_utils.py +++ b/src/diffusers/configuration_utils.py @@ -16,6 +16,7 @@ """ ConfigMixin base class and utilities.""" import dataclasses import functools +import importlib import inspect import json import os @@ -304,6 +305,17 @@ def extract_init_dict(cls, config_dict, **kwargs): # use value from config dict init_dict[key] = config_dict.pop(key) + # remove attributes from other class that cannot be expected + orig_cls_name = config_dict.pop("_class_name", cls.__name__) + if orig_cls_name != cls.__name__: + diffusers_library = importlib.import_module(__name__.split(".")[0]) + orig_cls = getattr(diffusers_library, orig_cls_name) + unexpected_keys_from_orig = ( + set(dict(inspect.signature(orig_cls.__init__).parameters).keys()) - expected_keys + ) + config_dict = {k: v for k, v in config_dict.items() if k not in unexpected_keys_from_orig} + + # remove private attributes config_dict = {k: v for k, v in config_dict.items() if not k.startswith("_")} if len(config_dict) > 0: diff --git a/tests/test_config.py b/tests/test_config.py index ab5b1e86bca8..70782f50388b 100755 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -13,10 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os import tempfile import unittest +import diffusers +from diffusers import DDIMScheduler, logging from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils.testing_utils import CaptureLogger class SampleObject(ConfigMixin): @@ -34,6 +39,21 @@ def __init__( pass +class SampleObject_2(ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + f=[1, 3], + ): + pass + + class ConfigTester(unittest.TestCase): def test_load_not_from_mixin(self): with self.assertRaises(ValueError): @@ -97,3 +117,49 @@ def test_save_load(self): assert config.pop("c") == (2, 5) # instantiated as tuple assert new_config.pop("c") == [2, 5] # saved & loaded as list because of json assert config == new_config + + def test_save_load_from_different_config(self): + obj = SampleObject() + + # mock add obj class to `diffusers` + setattr(diffusers, "SampleObject", SampleObject) + logger = logging.get_logger("diffusers.configuration_utils") + + with tempfile.TemporaryDirectory() as tmpdirname: + obj.save_config(tmpdirname) + with CaptureLogger(logger) as cap_logger_1: + new_obj_1 = SampleObject_2.from_config(tmpdirname) + + with open(os.path.join(tmpdirname, SampleObject.config_name), "r") as f: + data = json.load(f) + data["unexpected"] = True + + with open(os.path.join(tmpdirname, SampleObject.config_name), "w") as f: + json.dump(data, f) + + with CaptureLogger(logger) as cap_logger_2: + new_obj_2 = SampleObject.from_config(tmpdirname) + + with CaptureLogger(logger) as cap_logger_3: + new_obj_3 = SampleObject_2.from_config(tmpdirname) + + assert new_obj_1.__class__ == SampleObject_2 + assert new_obj_2.__class__ == SampleObject + assert new_obj_3.__class__ == SampleObject_2 + + assert cap_logger_1.out == "" + assert ( + cap_logger_2.out + == "The config attributes {'unexpected': True} were passed to SampleObject, but are not expected and will" + " be ignored. Please verify your config.json configuration file.\n" + ) + assert cap_logger_2.out.replace("SampleObject", "SampleObject_2") == cap_logger_3.out + + def test_load_ddim_from_pndm(self): + logger = logging.get_logger("diffusers.configuration_utils") + + with CaptureLogger(logger) as cap_logger: + ddim = DDIMScheduler.from_config("runwayml/stable-diffusion-v1-5", subfolder="scheduler") + + assert ddim.__class__ == DDIMScheduler + assert cap_logger.out == "" From 3af5b757da7fa51187b2dfa7fb0c7741f33353a6 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 31 Oct 2022 11:46:29 +0000 Subject: [PATCH 4/9] finish --- src/diffusers/configuration_utils.py | 57 ++++++++++++------ .../pipeline_onnx_stable_diffusion.py | 29 +++++++++ .../pipeline_onnx_stable_diffusion_img2img.py | 13 ++++ .../pipeline_onnx_stable_diffusion_inpaint.py | 13 ++++ .../pipeline_stable_diffusion.py | 13 ++++ .../pipeline_stable_diffusion_img2img.py | 13 ++++ .../pipeline_stable_diffusion_inpaint.py | 13 ++++ ...ipeline_stable_diffusion_inpaint_legacy.py | 13 ++++ src/diffusers/schedulers/scheduling_ddim.py | 2 + src/diffusers/schedulers/scheduling_ddpm.py | 2 + .../schedulers/scheduling_lms_discrete.py | 2 + src/diffusers/schedulers/scheduling_pndm.py | 2 + tests/test_config.py | 60 ++++++++++++++++--- 13 files changed, 208 insertions(+), 24 deletions(-) diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py index 55867c1a3e2f..e556ac9d3f7d 100644 --- a/src/diffusers/configuration_utils.py +++ b/src/diffusers/configuration_utils.py @@ -48,10 +48,13 @@ class ConfigMixin: - **config_name** (`str`) -- A filename under which the config should stored when calling [`~ConfigMixin.save_config`] (should be overridden by parent class). - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be - overridden by parent class). + - **_compatible_classes** (`List[str]`) -- A list of classes that are compatible with parent classes, so that + `from_pretrained` can be used from class that is different to class that was used to save the config (should + be overridden by parent class). """ config_name = None ignore_for_config = [] + _compatible_classes = [] def register_to_config(self, **kwargs): if self.config_name is None: @@ -281,9 +284,14 @@ def get_config_dict( return config_dict + @staticmethod + def _get_init_keys(cls): + return set(dict(inspect.signature(cls.__init__).parameters).keys()) + @classmethod def extract_init_dict(cls, config_dict, **kwargs): - expected_keys = set(dict(inspect.signature(cls.__init__).parameters).keys()) + # 1. Retrieve expected config attributes from __init__ signature + expected_keys = cls._get_init_keys(cls) expected_keys.remove("self") # remove general kwargs if present in dict if "kwargs" in expected_keys: @@ -293,31 +301,44 @@ def extract_init_dict(cls, config_dict, **kwargs): for arg in cls._flax_internal_args: expected_keys.remove(arg) + # 2. Remove attributes that cannot be expected from expected config attributes # remove keys to be ignored if len(cls.ignore_for_config) > 0: expected_keys = expected_keys - set(cls.ignore_for_config) - init_dict = {} - for key in expected_keys: - if key in kwargs: - # overwrite key - init_dict[key] = kwargs.pop(key) - elif key in config_dict: - # use value from config dict - init_dict[key] = config_dict.pop(key) - # remove attributes from other class that cannot be expected + # load diffusers library to import compatible and original scheduler + diffusers_library = importlib.import_module(__name__.split(".")[0]) + + # remove attributes from compatible classes that orig cannot expect + compatible_classes = [getattr(diffusers_library, c) for c in cls._compatible_classes] + expected_keys_comp_cls = set() + for c in compatible_classes: + expected_keys_c = cls._get_init_keys(c) + expected_keys_comp_cls = expected_keys_comp_cls.union(expected_keys_c) + expected_keys_comp_cls = expected_keys_comp_cls - cls._get_init_keys(cls) + config_dict = {k: v for k, v in config_dict.items() if k not in expected_keys_comp_cls} + + # remove attributes from orig class that cannot be expected orig_cls_name = config_dict.pop("_class_name", cls.__name__) if orig_cls_name != cls.__name__: - diffusers_library = importlib.import_module(__name__.split(".")[0]) orig_cls = getattr(diffusers_library, orig_cls_name) - unexpected_keys_from_orig = ( - set(dict(inspect.signature(orig_cls.__init__).parameters).keys()) - expected_keys - ) + unexpected_keys_from_orig = cls._get_init_keys(orig_cls) - expected_keys config_dict = {k: v for k, v in config_dict.items() if k not in unexpected_keys_from_orig} # remove private attributes config_dict = {k: v for k, v in config_dict.items() if not k.startswith("_")} + # 3. Create keyword arguments that will be passed to __init__ from expected keyword arguments + init_dict = {} + for key in expected_keys: + if key in kwargs: + # overwrite key + init_dict[key] = kwargs.pop(key) + elif key in config_dict: + # use value from config dict + init_dict[key] = config_dict.pop(key) + + # 4. Give nice warning if unexpected values have been passed if len(config_dict) > 0: logger.warning( f"The config attributes {config_dict} were passed to {cls.__name__}, " @@ -325,14 +346,16 @@ def extract_init_dict(cls, config_dict, **kwargs): f"{cls.config_name} configuration file." ) - unused_kwargs = {**config_dict, **kwargs} - + # 5. Give nice info if config attributes are initiliazed to default because they have not been passed passed_keys = set(init_dict.keys()) if len(expected_keys - passed_keys) > 0: logger.info( f"{expected_keys - passed_keys} was not found in config. Values will be initialized to default values." ) + # 6. Define unused keyword arguments + unused_kwargs = {**config_dict, **kwargs} + return init_dict, unused_kwargs @classmethod diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py index a2cb99be421f..ce8f37608463 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py @@ -5,6 +5,7 @@ from transformers import CLIPFeatureExtractor, CLIPTokenizer +from ...configuration_utils import FrozenDict from ...onnx_utils import OnnxRuntimeModel from ...pipeline_utils import DiffusionPipeline from ...schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler @@ -36,6 +37,34 @@ def __init__( feature_extractor: CLIPFeatureExtractor, ): super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + self.register_modules( vae_encoder=vae_encoder, vae_decoder=vae_decoder, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py index b2105458b574..304e52c62d4c 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py @@ -90,6 +90,19 @@ def __init__( new_config["steps_offset"] = 1 scheduler._internal_dict = FrozenDict(new_config) + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + if safety_checker is None: logger.warning( f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py index 37374aa4b1bc..70948be2deca 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py @@ -104,6 +104,19 @@ def __init__( new_config["steps_offset"] = 1 scheduler._internal_dict = FrozenDict(new_config) + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + if safety_checker is None: logger.warning( f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 2a33074a2473..15c03a3c5c9c 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -72,6 +72,19 @@ def __init__( new_config["steps_offset"] = 1 scheduler._internal_dict = FrozenDict(new_config) + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + if safety_checker is None: logger.warn( f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index db8134834564..aa152f48144e 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -83,6 +83,19 @@ def __init__( new_config["steps_offset"] = 1 scheduler._internal_dict = FrozenDict(new_config) + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + if safety_checker is None: logger.warn( f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index d49f55aa921d..907bde125aca 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -90,6 +90,19 @@ def __init__( new_config["steps_offset"] = 1 scheduler._internal_dict = FrozenDict(new_config) + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + if safety_checker is None: logger.warn( f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py index 71a2be62128c..f04ea6b53926 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py @@ -96,6 +96,19 @@ def __init__( new_config["steps_offset"] = 1 scheduler._internal_dict = FrozenDict(new_config) + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + if safety_checker is None: logger.warn( f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" diff --git a/src/diffusers/schedulers/scheduling_ddim.py b/src/diffusers/schedulers/scheduling_ddim.py index 58e40b57e923..3db6b749c703 100644 --- a/src/diffusers/schedulers/scheduling_ddim.py +++ b/src/diffusers/schedulers/scheduling_ddim.py @@ -109,6 +109,8 @@ class DDIMScheduler(SchedulerMixin, ConfigMixin): """ + _compatible_classes = ["PNDMScheduler", "DDPMScheduler", "LMSDiscreteScheduler"] + @register_to_config def __init__( self, diff --git a/src/diffusers/schedulers/scheduling_ddpm.py b/src/diffusers/schedulers/scheduling_ddpm.py index d51d58ac8f45..ce713a3150d1 100644 --- a/src/diffusers/schedulers/scheduling_ddpm.py +++ b/src/diffusers/schedulers/scheduling_ddpm.py @@ -102,6 +102,8 @@ class DDPMScheduler(SchedulerMixin, ConfigMixin): """ + _compatible_classes = ["DDIMScheduler", "PNDMScheduler", "LMSDiscreteScheduler"] + @register_to_config def __init__( self, diff --git a/src/diffusers/schedulers/scheduling_lms_discrete.py b/src/diffusers/schedulers/scheduling_lms_discrete.py index 6157f4b4dc65..fc0f6daf1992 100644 --- a/src/diffusers/schedulers/scheduling_lms_discrete.py +++ b/src/diffusers/schedulers/scheduling_lms_discrete.py @@ -67,6 +67,8 @@ class LMSDiscreteScheduler(SchedulerMixin, ConfigMixin): """ + _compatible_classes = ["DDIMScheduler", "DDPMScheduler", "PNDMScheduler"] + @register_to_config def __init__( self, diff --git a/src/diffusers/schedulers/scheduling_pndm.py b/src/diffusers/schedulers/scheduling_pndm.py index 03ea5c15d1b0..a2181384ec24 100644 --- a/src/diffusers/schedulers/scheduling_pndm.py +++ b/src/diffusers/schedulers/scheduling_pndm.py @@ -88,6 +88,8 @@ class PNDMScheduler(SchedulerMixin, ConfigMixin): """ + _compatible_classes = ["DDIMScheduler", "DDPMScheduler", "LMSDiscreteScheduler"] + @register_to_config def __init__( self, diff --git a/tests/test_config.py b/tests/test_config.py index 70782f50388b..b7bfc0b947d5 100755 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,7 +19,7 @@ import unittest import diffusers -from diffusers import DDIMScheduler, logging +from diffusers import DDIMScheduler, PNDMScheduler, logging from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.utils.testing_utils import CaptureLogger @@ -39,7 +39,7 @@ def __init__( pass -class SampleObject_2(ConfigMixin): +class SampleObject2(ConfigMixin): config_name = "config.json" @register_to_config @@ -128,8 +128,9 @@ def test_save_load_from_different_config(self): with tempfile.TemporaryDirectory() as tmpdirname: obj.save_config(tmpdirname) with CaptureLogger(logger) as cap_logger_1: - new_obj_1 = SampleObject_2.from_config(tmpdirname) + new_obj_1 = SampleObject2.from_config(tmpdirname) + # now save a config parameter that is not expected with open(os.path.join(tmpdirname, SampleObject.config_name), "r") as f: data = json.load(f) data["unexpected"] = True @@ -141,11 +142,11 @@ def test_save_load_from_different_config(self): new_obj_2 = SampleObject.from_config(tmpdirname) with CaptureLogger(logger) as cap_logger_3: - new_obj_3 = SampleObject_2.from_config(tmpdirname) + new_obj_3 = SampleObject2.from_config(tmpdirname) - assert new_obj_1.__class__ == SampleObject_2 + assert new_obj_1.__class__ == SampleObject2 assert new_obj_2.__class__ == SampleObject - assert new_obj_3.__class__ == SampleObject_2 + assert new_obj_3.__class__ == SampleObject2 assert cap_logger_1.out == "" assert ( @@ -153,7 +154,41 @@ def test_save_load_from_different_config(self): == "The config attributes {'unexpected': True} were passed to SampleObject, but are not expected and will" " be ignored. Please verify your config.json configuration file.\n" ) - assert cap_logger_2.out.replace("SampleObject", "SampleObject_2") == cap_logger_3.out + assert cap_logger_2.out.replace("SampleObject", "SampleObject2") == cap_logger_3.out + + def test_save_load_compatible_schedulers(self): + SampleObject2._compatible_classes = ["SampleObject"] + SampleObject._compatible_classes = ["SampleObject2"] + + obj = SampleObject() + + # mock add obj class to `diffusers` + setattr(diffusers, "SampleObject", SampleObject) + setattr(diffusers, "SampleObject2", SampleObject2) + logger = logging.get_logger("diffusers.configuration_utils") + + with tempfile.TemporaryDirectory() as tmpdirname: + obj.save_config(tmpdirname) + + # now save a config parameter that is expected by another class, but not origin class + with open(os.path.join(tmpdirname, SampleObject.config_name), "r") as f: + data = json.load(f) + data["f"] = [0, 0] + data["unexpected"] = True + + with open(os.path.join(tmpdirname, SampleObject.config_name), "w") as f: + json.dump(data, f) + + with CaptureLogger(logger) as cap_logger: + new_obj = SampleObject.from_config(tmpdirname) + + assert new_obj.__class__ == SampleObject + + assert ( + cap_logger.out + == "The config attributes {'unexpected': True} were passed to SampleObject, but are not expected and will" + " be ignored. Please verify your config.json configuration file.\n" + ) def test_load_ddim_from_pndm(self): logger = logging.get_logger("diffusers.configuration_utils") @@ -162,4 +197,15 @@ def test_load_ddim_from_pndm(self): ddim = DDIMScheduler.from_config("runwayml/stable-diffusion-v1-5", subfolder="scheduler") assert ddim.__class__ == DDIMScheduler + # no warning should be thrown + assert cap_logger.out == "" + + def test_load_pndm(self): + logger = logging.get_logger("diffusers.configuration_utils") + + with CaptureLogger(logger) as cap_logger: + pndm = PNDMScheduler.from_config("runwayml/stable-diffusion-v1-5", subfolder="scheduler") + + assert pndm.__class__ == PNDMScheduler + # no warning should be thrown assert cap_logger.out == "" From 38296c6e285d940117b069071eb2de41b5302198 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 31 Oct 2022 11:59:36 +0000 Subject: [PATCH 5/9] more replacements --- examples/text_to_image/train_text_to_image.py | 4 +--- examples/textual_inversion/textual_inversion.py | 4 +--- tests/pipelines/stable_diffusion/test_stable_diffusion.py | 8 +------- .../stable_diffusion/test_stable_diffusion_img2img.py | 3 +-- .../stable_diffusion/test_stable_diffusion_inpaint.py | 2 +- .../test_stable_diffusion_inpaint_legacy.py | 3 +-- 6 files changed, 6 insertions(+), 18 deletions(-) diff --git a/examples/text_to_image/train_text_to_image.py b/examples/text_to_image/train_text_to_image.py index 7bb8cd6c0048..cf7dac89330c 100644 --- a/examples/text_to_image/train_text_to_image.py +++ b/examples/text_to_image/train_text_to_image.py @@ -605,9 +605,7 @@ def collate_fn(examples): vae=vae, unet=unet, tokenizer=tokenizer, - scheduler=PNDMScheduler( - beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", skip_prk_steps=True - ), + scheduler=PNDMScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler"), safety_checker=StableDiffusionSafetyChecker.from_pretrained("CompVis/stable-diffusion-safety-checker"), feature_extractor=CLIPFeatureExtractor.from_pretrained("openai/clip-vit-base-patch32"), ) diff --git a/examples/textual_inversion/textual_inversion.py b/examples/textual_inversion/textual_inversion.py index aabf1aa48326..bb3517608c3a 100644 --- a/examples/textual_inversion/textual_inversion.py +++ b/examples/textual_inversion/textual_inversion.py @@ -552,9 +552,7 @@ def main(): vae=vae, unet=unet, tokenizer=tokenizer, - scheduler=PNDMScheduler( - beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", skip_prk_steps=True - ), + scheduler=PNDMScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler"), safety_checker=StableDiffusionSafetyChecker.from_pretrained("CompVis/stable-diffusion-safety-checker"), feature_extractor=CLIPFeatureExtractor.from_pretrained("openai/clip-vit-base-patch32"), ) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index 6b23aca33bc6..1959d5f50a0b 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -552,13 +552,7 @@ def test_stable_diffusion_fast_ddim(self): sd_pipe = sd_pipe.to(torch_device) sd_pipe.set_progress_bar_config(disable=None) - scheduler = DDIMScheduler( - beta_start=0.00085, - beta_end=0.012, - beta_schedule="scaled_linear", - clip_sample=False, - set_alpha_to_one=False, - ) + scheduler = DDIMScheduler.from_config("CompVis/stable-diffusion-v1-1", subfolder="scheduler") sd_pipe.scheduler = scheduler prompt = "A painting of a squirrel eating a burger" diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index 7cbb9bc51e81..78d001e3c758 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -523,9 +523,8 @@ def test_stable_diffusion_img2img_pipeline_k_lms(self): init_image = init_image.resize((768, 512)) expected_image = np.array(expected_image, dtype=np.float32) / 255.0 - lms = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") - model_id = "CompVis/stable-diffusion-v1-4" + lms = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler") pipe = StableDiffusionImg2ImgPipeline.from_pretrained( model_id, scheduler=lms, diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py index e63738ae8a22..0a373ada68bc 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py @@ -366,8 +366,8 @@ def test_stable_diffusion_inpaint_pipeline_pndm(self): ) expected_image = np.array(expected_image, dtype=np.float32) / 255.0 - pndm = PNDMScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", skip_prk_steps=True) model_id = "runwayml/stable-diffusion-inpainting" + pndm = PNDMScheduler.from_config(model_id, subfolder="scheduler") pipe = StableDiffusionInpaintPipeline.from_pretrained( model_id, safety_checker=None, scheduler=pndm, device_map="auto" ) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py index 90dbf3b63604..d25342a35aea 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py @@ -407,9 +407,8 @@ def test_stable_diffusion_inpaint_legacy_pipeline_k_lms(self): ) expected_image = np.array(expected_image, dtype=np.float32) / 255.0 - lms = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear") - model_id = "CompVis/stable-diffusion-v1-4" + lms = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler") pipe = StableDiffusionInpaintPipeline.from_pretrained( model_id, scheduler=lms, From c02d166ab6bbb1ffef2e80beb2c0ead7e6987280 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 31 Oct 2022 12:16:38 +0000 Subject: [PATCH 6/9] adapt fast tests hf token --- .github/workflows/pr_tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 81c75fecec6e..c9e304555297 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -10,6 +10,7 @@ concurrency: cancel-in-progress: true env: + HF_HOME: /mnt/cache OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 PYTEST_TIMEOUT: 60 @@ -41,6 +42,8 @@ jobs: python utils/print_env.py - name: Run all fast tests on CPU + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_cpu tests/ @@ -90,6 +93,8 @@ jobs: - name: Run all fast tests on MPS shell: arch -arch arm64 bash {0} + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} run: | ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps tests/ From 905adc150575cfbd7a9b1a66e2dee33ffe738e99 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 31 Oct 2022 12:21:16 +0000 Subject: [PATCH 7/9] correct more --- .github/workflows/pr_tests.yml | 1 - src/diffusers/configuration_utils.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index c9e304555297..2e6ff82eeaa8 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -10,7 +10,6 @@ concurrency: cancel-in-progress: true env: - HF_HOME: /mnt/cache OMP_NUM_THREADS: 8 MKL_NUM_THREADS: 8 PYTEST_TIMEOUT: 60 diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py index e556ac9d3f7d..460b0514c1f3 100644 --- a/src/diffusers/configuration_utils.py +++ b/src/diffusers/configuration_utils.py @@ -48,6 +48,7 @@ class ConfigMixin: - **config_name** (`str`) -- A filename under which the config should stored when calling [`~ConfigMixin.save_config`] (should be overridden by parent class). - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be + overridden by parent class). - **_compatible_classes** (`List[str]`) -- A list of classes that are compatible with parent classes, so that `from_pretrained` can be used from class that is different to class that was used to save the config (should be overridden by parent class). @@ -310,7 +311,9 @@ def extract_init_dict(cls, config_dict, **kwargs): diffusers_library = importlib.import_module(__name__.split(".")[0]) # remove attributes from compatible classes that orig cannot expect - compatible_classes = [getattr(diffusers_library, c) for c in cls._compatible_classes] + compatible_classes = [getattr(diffusers_library, c, None) for c in cls._compatible_classes] + # filder out None potentially undefined dummy classes + compatible_classes = [c for c in compatible_classes if c is not None] expected_keys_comp_cls = set() for c in compatible_classes: expected_keys_c = cls._get_init_keys(c) From 5738cdaed409d70b8ce913252c9689bf35adfe59 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 31 Oct 2022 16:23:00 +0100 Subject: [PATCH 8/9] Apply suggestions from code review Co-authored-by: Pedro Cuenca --- examples/textual_inversion/textual_inversion.py | 2 +- src/diffusers/configuration_utils.py | 6 +++--- .../stable_diffusion/pipeline_onnx_stable_diffusion.py | 2 +- .../pipeline_onnx_stable_diffusion_img2img.py | 2 +- .../pipeline_onnx_stable_diffusion_inpaint.py | 2 +- .../pipelines/stable_diffusion/pipeline_stable_diffusion.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 2 +- .../pipeline_stable_diffusion_inpaint_legacy.py | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/textual_inversion/textual_inversion.py b/examples/textual_inversion/textual_inversion.py index bb3517608c3a..fc9380edcd73 100644 --- a/examples/textual_inversion/textual_inversion.py +++ b/examples/textual_inversion/textual_inversion.py @@ -552,7 +552,7 @@ def main(): vae=vae, unet=unet, tokenizer=tokenizer, - scheduler=PNDMScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler"), + scheduler=PNDMScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler"), safety_checker=StableDiffusionSafetyChecker.from_pretrained("CompVis/stable-diffusion-safety-checker"), feature_extractor=CLIPFeatureExtractor.from_pretrained("openai/clip-vit-base-patch32"), ) diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py index 460b0514c1f3..9e10d039f4e8 100644 --- a/src/diffusers/configuration_utils.py +++ b/src/diffusers/configuration_utils.py @@ -49,8 +49,8 @@ class ConfigMixin: [`~ConfigMixin.save_config`] (should be overridden by parent class). - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be overridden by parent class). - - **_compatible_classes** (`List[str]`) -- A list of classes that are compatible with parent classes, so that - `from_pretrained` can be used from class that is different to class that was used to save the config (should + - **_compatible_classes** (`List[str]`) -- A list of classes that are compatible with the parent class, so that + `from_config` can be used from a class different than the one used to save the config (should be overridden by parent class). """ config_name = None @@ -312,7 +312,7 @@ def extract_init_dict(cls, config_dict, **kwargs): # remove attributes from compatible classes that orig cannot expect compatible_classes = [getattr(diffusers_library, c, None) for c in cls._compatible_classes] - # filder out None potentially undefined dummy classes + # filter out None potentially undefined dummy classes compatible_classes = [c for c in compatible_classes if c is not None] expected_keys_comp_cls = set() for c in compatible_classes: diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py index ce8f37608463..22f5bf6c432d 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py @@ -56,7 +56,7 @@ def __init__( deprecation_message = ( f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." " `clip_sample` should be set to False in the configuration file. Please make sure to update the" - " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py index 304e52c62d4c..5e6b2e6f2fc3 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py @@ -94,7 +94,7 @@ def __init__( deprecation_message = ( f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." " `clip_sample` should be set to False in the configuration file. Please make sure to update the" - " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py index 70948be2deca..2ce9831a1676 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py @@ -108,7 +108,7 @@ def __init__( deprecation_message = ( f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." " `clip_sample` should be set to False in the configuration file. Please make sure to update the" - " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py index 15c03a3c5c9c..4a99831dcc79 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -76,7 +76,7 @@ def __init__( deprecation_message = ( f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." " `clip_sample` should be set to False in the configuration file. Please make sure to update the" - " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index aa152f48144e..ec9c2abde27b 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -87,7 +87,7 @@ def __init__( deprecation_message = ( f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." " `clip_sample` should be set to False in the configuration file. Please make sure to update the" - " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 907bde125aca..43ad5a29b859 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -94,7 +94,7 @@ def __init__( deprecation_message = ( f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." " `clip_sample` should be set to False in the configuration file. Please make sure to update the" - " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" ) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py index f04ea6b53926..11c2f8bd19a4 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint_legacy.py @@ -100,7 +100,7 @@ def __init__( deprecation_message = ( f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." " `clip_sample` should be set to False in the configuration file. Please make sure to update the" - " config accordingly as not setting `clip_sample` in the config might led to incorrect results in" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" ) From efb48478ebcc8be2108bc5e527827a96f6590675 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 31 Oct 2022 16:08:18 +0000 Subject: [PATCH 9/9] Integrate compatibility with euler --- src/diffusers/configuration_utils.py | 4 +- src/diffusers/schedulers/scheduling_ddim.py | 8 +- src/diffusers/schedulers/scheduling_ddpm.py | 8 +- .../scheduling_euler_ancestral_discrete.py | 8 ++ .../schedulers/scheduling_euler_discrete.py | 8 ++ .../schedulers/scheduling_lms_discrete.py | 8 +- src/diffusers/schedulers/scheduling_pndm.py | 8 +- tests/test_config.py | 74 ++++++++++++++++++- 8 files changed, 119 insertions(+), 7 deletions(-) diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py index 9e10d039f4e8..df18458c571d 100644 --- a/src/diffusers/configuration_utils.py +++ b/src/diffusers/configuration_utils.py @@ -50,8 +50,8 @@ class ConfigMixin: - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be overridden by parent class). - **_compatible_classes** (`List[str]`) -- A list of classes that are compatible with the parent class, so that - `from_config` can be used from a class different than the one used to save the config (should - be overridden by parent class). + `from_config` can be used from a class different than the one used to save the config (should be overridden + by parent class). """ config_name = None ignore_for_config = [] diff --git a/src/diffusers/schedulers/scheduling_ddim.py b/src/diffusers/schedulers/scheduling_ddim.py index 3db6b749c703..f95c18d9fae2 100644 --- a/src/diffusers/schedulers/scheduling_ddim.py +++ b/src/diffusers/schedulers/scheduling_ddim.py @@ -109,7 +109,13 @@ class DDIMScheduler(SchedulerMixin, ConfigMixin): """ - _compatible_classes = ["PNDMScheduler", "DDPMScheduler", "LMSDiscreteScheduler"] + _compatible_classes = [ + "PNDMScheduler", + "DDPMScheduler", + "LMSDiscreteScheduler", + "EulerDiscreteScheduler", + "EulerAncestralDiscreteScheduler", + ] @register_to_config def __init__( diff --git a/src/diffusers/schedulers/scheduling_ddpm.py b/src/diffusers/schedulers/scheduling_ddpm.py index ce713a3150d1..114a86b4320e 100644 --- a/src/diffusers/schedulers/scheduling_ddpm.py +++ b/src/diffusers/schedulers/scheduling_ddpm.py @@ -102,7 +102,13 @@ class DDPMScheduler(SchedulerMixin, ConfigMixin): """ - _compatible_classes = ["DDIMScheduler", "PNDMScheduler", "LMSDiscreteScheduler"] + _compatible_classes = [ + "DDIMScheduler", + "PNDMScheduler", + "LMSDiscreteScheduler", + "EulerDiscreteScheduler", + "EulerAncestralDiscreteScheduler", + ] @register_to_config def __init__( diff --git a/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py b/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py index b153dfbe2f02..3fe52a498044 100644 --- a/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py +++ b/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py @@ -67,6 +67,14 @@ class EulerAncestralDiscreteScheduler(SchedulerMixin, ConfigMixin): """ + _compatible_classes = [ + "DDIMScheduler", + "DDPMScheduler", + "LMSDiscreteScheduler", + "PNDMScheduler", + "EulerDiscreteScheduler", + ] + @register_to_config def __init__( self, diff --git a/src/diffusers/schedulers/scheduling_euler_discrete.py b/src/diffusers/schedulers/scheduling_euler_discrete.py index a71fbda13df6..93aeb8cc3865 100644 --- a/src/diffusers/schedulers/scheduling_euler_discrete.py +++ b/src/diffusers/schedulers/scheduling_euler_discrete.py @@ -68,6 +68,14 @@ class EulerDiscreteScheduler(SchedulerMixin, ConfigMixin): """ + _compatible_classes = [ + "DDIMScheduler", + "DDPMScheduler", + "LMSDiscreteScheduler", + "PNDMScheduler", + "EulerAncestralDiscreteScheduler", + ] + @register_to_config def __init__( self, diff --git a/src/diffusers/schedulers/scheduling_lms_discrete.py b/src/diffusers/schedulers/scheduling_lms_discrete.py index fc0f6daf1992..43e577b40936 100644 --- a/src/diffusers/schedulers/scheduling_lms_discrete.py +++ b/src/diffusers/schedulers/scheduling_lms_discrete.py @@ -67,7 +67,13 @@ class LMSDiscreteScheduler(SchedulerMixin, ConfigMixin): """ - _compatible_classes = ["DDIMScheduler", "DDPMScheduler", "PNDMScheduler"] + _compatible_classes = [ + "DDIMScheduler", + "DDPMScheduler", + "PNDMScheduler", + "EulerDiscreteScheduler", + "EulerAncestralDiscreteScheduler", + ] @register_to_config def __init__( diff --git a/src/diffusers/schedulers/scheduling_pndm.py b/src/diffusers/schedulers/scheduling_pndm.py index a2181384ec24..0082ede787b8 100644 --- a/src/diffusers/schedulers/scheduling_pndm.py +++ b/src/diffusers/schedulers/scheduling_pndm.py @@ -88,7 +88,13 @@ class PNDMScheduler(SchedulerMixin, ConfigMixin): """ - _compatible_classes = ["DDIMScheduler", "DDPMScheduler", "LMSDiscreteScheduler"] + _compatible_classes = [ + "DDIMScheduler", + "DDPMScheduler", + "LMSDiscreteScheduler", + "EulerDiscreteScheduler", + "EulerAncestralDiscreteScheduler", + ] @register_to_config def __init__( diff --git a/tests/test_config.py b/tests/test_config.py index b7bfc0b947d5..1c6178d7208a 100755 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,7 +19,7 @@ import unittest import diffusers -from diffusers import DDIMScheduler, PNDMScheduler, logging +from diffusers import DDIMScheduler, EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, PNDMScheduler, logging from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.utils.testing_utils import CaptureLogger @@ -54,6 +54,22 @@ def __init__( pass +class SampleObject3(ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + e=[1, 3], + f=[1, 3], + ): + pass + + class ConfigTester(unittest.TestCase): def test_load_not_from_mixin(self): with self.assertRaises(ValueError): @@ -190,6 +206,40 @@ def test_save_load_compatible_schedulers(self): " be ignored. Please verify your config.json configuration file.\n" ) + def test_save_load_from_different_config_comp_schedulers(self): + SampleObject3._compatible_classes = ["SampleObject", "SampleObject2"] + SampleObject2._compatible_classes = ["SampleObject", "SampleObject3"] + SampleObject._compatible_classes = ["SampleObject2", "SampleObject3"] + + obj = SampleObject() + + # mock add obj class to `diffusers` + setattr(diffusers, "SampleObject", SampleObject) + setattr(diffusers, "SampleObject2", SampleObject2) + setattr(diffusers, "SampleObject3", SampleObject3) + logger = logging.get_logger("diffusers.configuration_utils") + logger.setLevel(diffusers.logging.INFO) + + with tempfile.TemporaryDirectory() as tmpdirname: + obj.save_config(tmpdirname) + + with CaptureLogger(logger) as cap_logger_1: + new_obj_1 = SampleObject.from_config(tmpdirname) + + with CaptureLogger(logger) as cap_logger_2: + new_obj_2 = SampleObject2.from_config(tmpdirname) + + with CaptureLogger(logger) as cap_logger_3: + new_obj_3 = SampleObject3.from_config(tmpdirname) + + assert new_obj_1.__class__ == SampleObject + assert new_obj_2.__class__ == SampleObject2 + assert new_obj_3.__class__ == SampleObject3 + + assert cap_logger_1.out == "" + assert cap_logger_2.out == "{'f'} was not found in config. Values will be initialized to default values.\n" + assert cap_logger_3.out == "{'f'} was not found in config. Values will be initialized to default values.\n" + def test_load_ddim_from_pndm(self): logger = logging.get_logger("diffusers.configuration_utils") @@ -200,6 +250,28 @@ def test_load_ddim_from_pndm(self): # no warning should be thrown assert cap_logger.out == "" + def test_load_ddim_from_euler(self): + logger = logging.get_logger("diffusers.configuration_utils") + + with CaptureLogger(logger) as cap_logger: + euler = EulerDiscreteScheduler.from_config("runwayml/stable-diffusion-v1-5", subfolder="scheduler") + + assert euler.__class__ == EulerDiscreteScheduler + # no warning should be thrown + assert cap_logger.out == "" + + def test_load_ddim_from_euler_ancestral(self): + logger = logging.get_logger("diffusers.configuration_utils") + + with CaptureLogger(logger) as cap_logger: + euler = EulerAncestralDiscreteScheduler.from_config( + "runwayml/stable-diffusion-v1-5", subfolder="scheduler" + ) + + assert euler.__class__ == EulerAncestralDiscreteScheduler + # no warning should be thrown + assert cap_logger.out == "" + def test_load_pndm(self): logger = logging.get_logger("diffusers.configuration_utils")