From 2e6e4328eab31e59903aa1904a0272a761703eba Mon Sep 17 00:00:00 2001 From: yuwenzho Date: Mon, 4 Dec 2023 18:19:42 +0800 Subject: [PATCH 1/7] fix bugs in ort examples Signed-off-by: yuwenzho --- .../unet/quantization/ptq_static/README.md | 2 +- .../unet/quantization/ptq_static/main.py | 2 +- .../quantization/ptq_static/prepare_model.py | 28 ++++++++++++++++++- .../quantization/ptq_static/prepare_model.py | 2 +- .../quantization/ptq_dynamic/prepare_model.py | 19 +++++++++++++ .../quantization/ptq_static/prepare_model.py | 21 +++++++++++++- .../adaptor/ox_utils/operators/lstm.py | 8 ++++-- 7 files changed, 74 insertions(+), 8 deletions(-) diff --git a/examples/onnxrt/image_recognition/unet/quantization/ptq_static/README.md b/examples/onnxrt/image_recognition/unet/quantization/ptq_static/README.md index 72843058f39..d7f119e043c 100644 --- a/examples/onnxrt/image_recognition/unet/quantization/ptq_static/README.md +++ b/examples/onnxrt/image_recognition/unet/quantization/ptq_static/README.md @@ -16,7 +16,7 @@ pip install -r requirements.txt ## 2. Prepare Model ```bash -python prepare_model.py --input_model='CompVis/stable-diffusion-v1-4' --output_model='.' +python prepare_model.py --input_model='CompVis/stable-diffusion-v1-4' --output_model='unet-export.onnx' ``` # Run diff --git a/examples/onnxrt/image_recognition/unet/quantization/ptq_static/main.py b/examples/onnxrt/image_recognition/unet/quantization/ptq_static/main.py index a6eabf5b38a..a949faeef48 100644 --- a/examples/onnxrt/image_recognition/unet/quantization/ptq_static/main.py +++ b/examples/onnxrt/image_recognition/unet/quantization/ptq_static/main.py @@ -32,7 +32,7 @@ class Dataloader: def __init__(self, batch_size): self.batch_size = batch_size shape = [[batch_size, 4, 64, 64], [batch_size], [batch_size, 77, 768]] - dtype = ['float32', 'int64', 'float32'] + dtype = ['float32', 'float32', 'float32'] self.dataset = [] for idx in range(0, len(shape)): tensor = np.random.uniform(size=shape[idx]) diff --git a/examples/onnxrt/image_recognition/unet/quantization/ptq_static/prepare_model.py b/examples/onnxrt/image_recognition/unet/quantization/ptq_static/prepare_model.py index 9e6ac2eb688..18e79ff6cd3 100644 --- a/examples/onnxrt/image_recognition/unet/quantization/ptq_static/prepare_model.py +++ b/examples/onnxrt/image_recognition/unet/quantization/ptq_static/prepare_model.py @@ -1,5 +1,6 @@ import argparse import os +import shutil import subprocess @@ -9,10 +10,28 @@ def parse_arguments(): parser.add_argument("--output_model", type=str, required=True) return parser.parse_args() +def move_and_rename_model(source_folder, destination_folder): + if not os.path.exists(source_folder): + raise RuntimeError("{} path is not exists".format(source_folder)) + for file_name in os.listdir(source_folder): + source_file = os.path.join(source_folder, file_name) + destination_file = os.path.join(destination_folder, file_name) + + if os.path.isdir(source_file): + continue + + shutil.move(source_file, destination_file) + + if file_name == "model.onnx": + new_file_name = "unet-export.onnx" + new_file_path = os.path.join(destination_folder, new_file_name) + os.rename(destination_file, new_file_path) def prepare_model(input_model, output_model): # Use [tf2onnx tool](https://github.com/onnx/tensorflow-onnx) to convert tflite to onnx model. print("\nexport model...") + + export_file = "prepare_unet" subprocess.run( [ "git", @@ -34,11 +53,18 @@ def prepare_model(input_model, output_model): "--model_path", input_model, "--output_path", - output_model, + export_file, ], stdout=subprocess.PIPE, text=True, ) + + move_and_rename_model(os.path.join(export_file, "unet"), os.path.dirname(output_model)) + try: + shutil.rmtree(export_file, ignore_errors=True) + except OSError as e: + raise e + assert os.path.exists(output_model), f"Export failed! {output_model} doesn't exist!" diff --git a/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_static/prepare_model.py b/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_static/prepare_model.py index ec927be7c7f..6d8d774887b 100644 --- a/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_static/prepare_model.py +++ b/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_static/prepare_model.py @@ -88,6 +88,6 @@ def export_onnx_model(args, model): if args.input_model == 'Intel/bart-large-mrpc': import os - os.system('python -m transformers.onnx --model=Intel/bart-large-mrpc --feature=sequence-classification bart-large-mrpc/') + os.system('python -m transformers.onnx --model=Intel/bart-large-mrpc --feature=sequence-classification --export_with_transformers bart-large-mrpc/') else: export_onnx_model(args, model) \ No newline at end of file diff --git a/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py b/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py index 00b29ceb2a7..de66bb36dc5 100644 --- a/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py +++ b/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py @@ -25,6 +25,21 @@ def parse_arguments(): help='Maximum length of the sentence pairs') return parser.parse_args() +def comment_out_line(filepath, code): + modified_lines = [] + + with open(filepath, 'r') as file: + lines = file.readlines() + file.seek(0) + for line in lines: + if re.match(code, line.strip()): + import pdb;pdb.set_trace() + line = "#" + line + modified_lines.append(line) + + with open(filepath, 'w') as file: + file.writelines(modified_lines) + def prepare_model(input_model, output_model, task_name): print("\nexport model...") subprocess.run( @@ -42,6 +57,10 @@ def prepare_model(input_model, output_model, task_name): text=True, ) + # remove transformers min version check + comment_out_line("my_transformers/examples/pytorch/text-classification/run_glue.py", + r"check_min_version\(.*\)") + subprocess.run( [ "python", diff --git a/examples/onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py b/examples/onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py index 00b29ceb2a7..326467149c7 100644 --- a/examples/onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py +++ b/examples/onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py @@ -1,5 +1,5 @@ import argparse -import os +import re import subprocess import torch @@ -25,6 +25,21 @@ def parse_arguments(): help='Maximum length of the sentence pairs') return parser.parse_args() +def comment_out_line(filepath, code): + modified_lines = [] + + with open(filepath, 'r') as file: + lines = file.readlines() + file.seek(0) + for line in lines: + if re.match(code, line.strip()): + import pdb;pdb.set_trace() + line = "#" + line + modified_lines.append(line) + + with open(filepath, 'w') as file: + file.writelines(modified_lines) + def prepare_model(input_model, output_model, task_name): print("\nexport model...") subprocess.run( @@ -42,6 +57,10 @@ def prepare_model(input_model, output_model, task_name): text=True, ) + # remove transformers min version check + comment_out_line("my_transformers/examples/pytorch/text-classification/run_glue.py", + r"check_min_version\(.*\)") + subprocess.run( [ "python", diff --git a/neural_compressor/adaptor/ox_utils/operators/lstm.py b/neural_compressor/adaptor/ox_utils/operators/lstm.py index 9e35139f209..cb0004d5b3e 100644 --- a/neural_compressor/adaptor/ox_utils/operators/lstm.py +++ b/neural_compressor/adaptor/ox_utils/operators/lstm.py @@ -128,10 +128,12 @@ def convert(self, convert_format): kwargs = {} for attribute in node.attribute: - kwargs.update(attribute_to_kwarg(attribute)) - kwargs["domain"] = ms_domain + if attribute.name == "layout": + continue + kwarg = attribute_to_kwarg(attribute) + kwargs.update(kwarg) quant_lstm_name = node.name + "_quant" - quant_lstm_node = onnx.helper.make_node("DynamicQuantizeLSTM", inputs, node.output, quant_lstm_name, **kwargs) + quant_lstm_node = onnx.helper.make_node("DynamicQuantizeLSTM", inputs, node.output, quant_lstm_name, domain="com.microsoft", **kwargs) self.quantizer.remove_nodes.append(node) self.quantizer.new_nodes.append(quant_lstm_node) From 0a606cf47ca79c047de519c9fae08389b15823ac Mon Sep 17 00:00:00 2001 From: yuwenzho Date: Mon, 4 Dec 2023 18:29:02 +0800 Subject: [PATCH 2/7] fix prepare model bug Signed-off-by: yuwenzho --- .../quantization/ptq_dynamic/prepare_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_dynamic/prepare_model.py b/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_dynamic/prepare_model.py index ba584d1bbc6..d020e8aecb6 100644 --- a/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_dynamic/prepare_model.py +++ b/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_dynamic/prepare_model.py @@ -89,6 +89,6 @@ def export_onnx_model(args, model): if args.input_model == 'Intel/bart-large-mrpc': import os - os.system('python -m transformers.onnx --model=Intel/bart-large-mrpc --feature=sequence-classification bart-large-mrpc/') + os.system('python -m transformers.onnx --model=Intel/bart-large-mrpc --feature=sequence-classification --export_with_transformers bart-large-mrpc/') else: export_onnx_model(args, model) \ No newline at end of file From 66e383dfdbef13909136db9021eb882d6fa77d19 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 10:30:25 +0000 Subject: [PATCH 3/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- neural_compressor/adaptor/ox_utils/operators/lstm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/neural_compressor/adaptor/ox_utils/operators/lstm.py b/neural_compressor/adaptor/ox_utils/operators/lstm.py index cb0004d5b3e..74f2b988ca0 100644 --- a/neural_compressor/adaptor/ox_utils/operators/lstm.py +++ b/neural_compressor/adaptor/ox_utils/operators/lstm.py @@ -134,6 +134,8 @@ def convert(self, convert_format): kwargs.update(kwarg) quant_lstm_name = node.name + "_quant" - quant_lstm_node = onnx.helper.make_node("DynamicQuantizeLSTM", inputs, node.output, quant_lstm_name, domain="com.microsoft", **kwargs) + quant_lstm_node = onnx.helper.make_node( + "DynamicQuantizeLSTM", inputs, node.output, quant_lstm_name, domain="com.microsoft", **kwargs + ) self.quantizer.remove_nodes.append(node) self.quantizer.new_nodes.append(quant_lstm_node) From d7f57df847479a45dc1d1a57ee5a6fed12ab9aa1 Mon Sep 17 00:00:00 2001 From: yuwenzho Date: Tue, 5 Dec 2023 14:32:49 +0800 Subject: [PATCH 4/7] update README.md Signed-off-by: yuwenzho --- .../language_modeling/quantization/ptq_dynamic/README.md | 2 +- .../language_modeling/quantization/ptq_static/README.md | 2 +- .../nlp/onnx_model_zoo/gpt2/quantization/ptq_dynamic/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_dynamic/README.md b/examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_dynamic/README.md index ccf185569a6..30dbe733f51 100644 --- a/examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_dynamic/README.md +++ b/examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_dynamic/README.md @@ -21,7 +21,7 @@ Supported model identifier from [huggingface.co](https://huggingface.co/): | gpt2 | | distilgpt2 | -Require transformers==3.2.0. +Require python <=3.8 and transformers==3.2.0. ```shell python prepare_model.py --input_model=gpt2 --output_model=gpt2.onnx # or other supported model identifier diff --git a/examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_static/README.md b/examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_static/README.md index b9b49b96268..075fd8cd663 100644 --- a/examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_static/README.md +++ b/examples/onnxrt/nlp/huggingface_model/language_modeling/quantization/ptq_static/README.md @@ -21,7 +21,7 @@ Supported model identifier from [huggingface.co](https://huggingface.co/): | gpt2 | | distilgpt2 | -Require transformers==3.2.0. +Require python <=3.8 and transformers==3.2.0. ```shell python prepare_model.py --input_model=gpt2 --output_model=gpt2.onnx # or other supported model identifier diff --git a/examples/onnxrt/nlp/onnx_model_zoo/gpt2/quantization/ptq_dynamic/README.md b/examples/onnxrt/nlp/onnx_model_zoo/gpt2/quantization/ptq_dynamic/README.md index d3356daa6eb..3976bac085c 100644 --- a/examples/onnxrt/nlp/onnx_model_zoo/gpt2/quantization/ptq_dynamic/README.md +++ b/examples/onnxrt/nlp/onnx_model_zoo/gpt2/quantization/ptq_dynamic/README.md @@ -15,7 +15,7 @@ pip install -r requirements.txt ## 2. Prepare Model Use `prepare_model.py` script for ONNX model conversion. -Require transformers==3.2.0. +Require python <=3.8 and transformers==3.2.0. ```shell python prepare_model.py From a74e1d005cadb0c6faf375573a6d45304e5dd3d2 Mon Sep 17 00:00:00 2001 From: yuwenzho Date: Tue, 5 Dec 2023 14:35:08 +0800 Subject: [PATCH 5/7] update prepare_model.py Signed-off-by: yuwenzho --- .../nlp/roberta/quantization/ptq_dynamic/prepare_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py b/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py index de66bb36dc5..326467149c7 100644 --- a/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py +++ b/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py @@ -1,5 +1,5 @@ import argparse -import os +import re import subprocess import torch From f8996f3bb3d830d95be0403e19d397534ff3391a Mon Sep 17 00:00:00 2001 From: yuwenzho Date: Tue, 5 Dec 2023 15:07:23 +0800 Subject: [PATCH 6/7] update prepare_model.py Signed-off-by: yuwenzho --- .../onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py | 1 - .../onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py | 1 - 2 files changed, 2 deletions(-) diff --git a/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py b/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py index 326467149c7..32b4289fdcc 100644 --- a/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py +++ b/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/prepare_model.py @@ -33,7 +33,6 @@ def comment_out_line(filepath, code): file.seek(0) for line in lines: if re.match(code, line.strip()): - import pdb;pdb.set_trace() line = "#" + line modified_lines.append(line) diff --git a/examples/onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py b/examples/onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py index 326467149c7..32b4289fdcc 100644 --- a/examples/onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py +++ b/examples/onnxrt/nlp/roberta/quantization/ptq_static/prepare_model.py @@ -33,7 +33,6 @@ def comment_out_line(filepath, code): file.seek(0) for line in lines: if re.match(code, line.strip()): - import pdb;pdb.set_trace() line = "#" + line modified_lines.append(line) From dfb142a4e063b06ccc0cfaa16c1d5f3bd461c443 Mon Sep 17 00:00:00 2001 From: yuwenzho Date: Tue, 5 Dec 2023 17:08:03 +0800 Subject: [PATCH 7/7] update onnxrt config Signed-off-by: yuwenzho --- examples/.config/model_params_onnxrt.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/.config/model_params_onnxrt.json b/examples/.config/model_params_onnxrt.json index 8695baa3dc1..fe66f6f836f 100644 --- a/examples/.config/model_params_onnxrt.json +++ b/examples/.config/model_params_onnxrt.json @@ -486,7 +486,7 @@ "unet": { "model_src_dir": "image_recognition/unet/quantization/ptq_static", "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", - "input_model": "/tf_dataset2/models/onnx/unet/model.onnx", + "input_model": "/tf_dataset2/models/onnx/unet/unet-export.onnx", "main_script": "main.py", "batch_size": 1 },