add two export examples to validate accuracy and performance of export API. (#563)

Signed-off-by: Xin He <[email protected]>
Co-authored-by: chensuyue <[email protected]>

Author: xin3he

docs/source/export.md

@@ -67,6 +67,9 @@ int8_onnx_config = Torch2ONNXConfig(
 )
 q_model.export('int8-model.onnx', int8_onnx_config)
 ```
+> **Note**: Two export examples covering computer vision and natural language processing tasks exist in examples. Users can leverage them to verify the accuracy and performance of the exported ONNX model.
+ - [Image recognition](/examples/pytorch/image_recognition/torchvision_models/export/fx/)
+ - [Text classification](/examples/pytorch/nlp/huggingface_models/text-classification/export/fx/)
 
 # Appendix
 

examples/.config/model_params_pt2onnx.json

@@ -0,0 +1,36 @@
+{
+  "pt2onnx": {
+    "resnet18": {
+      "model_src_dir": "image_recognition/torchvision_models/export/fx",
+      "source_model_dataset": "/tf_dataset/pytorch/ImageNet/raw",
+      "target_model_dataset": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ImagenetRaw_small_5000",
+      "input_model": "resnet18",
+      "main_script": "main.py",
+      "batch_size": 100
+    },
+    "resnet50": {
+      "model_src_dir": "image_recognition/torchvision_models/export/fx",
+      "source_model_dataset": "/tf_dataset/pytorch/ImageNet/raw",
+      "target_model_dataset": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ImagenetRaw_small_5000",
+      "input_model": "resnet50",
+      "main_script": "main.py",
+      "batch_size": 100
+    },
+    "bert_base_MRPC": {
+      "model_src_dir": "nlp/huggingface_models/text-classification/export/fx",
+      "source_model_dataset": "mrpc",
+      "target_model_dataset": "mrpc",
+      "input_model": "/tf_dataset/pytorch/glue_data/base_weights/bert_MRPC_output",
+      "main_script": "run_glue.py",
+      "batch_size": 64
+    },
+    "bert_large_MRPC": {
+      "model_src_dir": "nlp/huggingface_models/text-classification/export/fx",
+      "source_model_dataset": "mrpc",
+      "target_model_dataset": "mrpc",
+      "input_model": "/tf_dataset/pytorch/glue_data/weights/bert_MRPC_output",
+      "main_script": "run_glue.py",
+      "batch_size": 64
+    }
+  }
+}

examples/pytorch/image_recognition/torchvision_models/export/fx/README.md

@@ -0,0 +1,49 @@
+Step-by-Step
+============
+
+This document describes the step-by-step instructions for reproducing PyTorch tuning results with Intel® Neural Compressor.
+
+# Prerequisite
+
+## 1. Environment
+
+PyTorch 1.8 or higher version is needed with pytorch_fx backend.
+
+```shell
+cd examples/pytorch/image_recognition/torchvision_models/quantization/ptq/cpu/fx
+pip install -r requirements.txt
+```
+> Note: Validated PyTorch [Version](/docs/source/installation_guide.md#validated-software-environment).
+
+## 2. Prepare Dataset
+
+Download [ImageNet](http://www.image-net.org/) Raw image to dir: /path/to/imagenet.  The dir include below folder:
+
+```bash
+ls /path/to/pytorch-imagenet
+train  val
+ls /path/to/onnx-imagenet-validation
+ILSVRC2012_img_val val.txt
+```
+
+# Run
+### 1. To get the exported model: 
+
+Run run_export.sh to get ONNX model from PyTorch model.
+```bash
+# export fp32 model
+bash run_export.sh --input_model=resnet50 --dtype=fp32 --dataset_location=/path/to/pytorch-imagenet --output_model=resnet50-fp32.onnx
+# export int8 model
+bash run_export.sh --input_model=resnet50 --dtype=int8 --quant_format=[QDQ|QLinear] --dataset_location=/path/to/pytorch-imagenet --output_model=resnet50-int8.onnx
+```
+
+### 2. To get the benchmark of exported and tuned models, includes Batch_size and Throughput: 
+Run run_benchmark.sh to benchmark the accuracy and performance of ONNX models and PyTorch model.
+```bash
+# benchmark ONNX model
+bash run_benchmark.sh --input_model=[resnet50-fp32.onnx|resnet50-int8.onnx] --dataset_location=/path/to/onnx-imagenet-validation --mode=[accuracy|performance] --batch_size=[16]
+# benchmark PyTorch model
+bash run_benchmark.sh --input_model=[resnet50|/path/to/saved_results] --dataset_location=/path/to/pytorch-imagenet --mode=[accuracy|performance] --int8=[true|false] --batch_size=[16]
+```
+
+> Note: All torchvision model names can be passed as long as they are included in `torchvision.models`, below are some examples.