Update PyTorch examples quantization API

.azure-pipelines/model-test.yml

@@ -45,7 +45,7 @@ parameters:
       - ssd_mobilenet_v1_ckpt
       # - ssd_resnet50_v1_ckpt
       - inception_v1
-      - resnet50_fashion
+      # - resnet50_fashion
       - darknet19
       - densenet-121
       - resnet-101
@@ -156,7 +156,8 @@ stages:
               cd ${OUT_SCRIPT_PATH}
               mkdir generated
               mkdir last_generated
-              python -u collect_log_all.py --logs_dir $(OUT_SCRIPT_PATH) --output_dir generated
+              pip install requests
+              python -u collect_log_all.py --logs_dir $(OUT_SCRIPT_PATH) --output_dir generated --build_id=$(Build.BuildId)
             displayName: "Collect all logs"
           - task: DownloadPipelineArtifact@2
             continueOnError: true

.azure-pipelines/scripts/codeScan/pylint/pylint.sh

@@ -10,13 +10,13 @@ pip install -r /neural-compressor/requirements.txt
 pip install torch==1.12.0
 
 python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto \
---ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor \
+--ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor \
 > $log_dir/pylint.json
 
 exit_code=$?
 
 $BOLD_YELLOW && echo " -----------------  Current pylint cmd start --------------------------" && $RESET
-echo "python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor > $log_dir/pylint.json"
+echo "python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor > $log_dir/pylint.json"
 $BOLD_YELLOW && echo " -----------------  Current pylint cmd end --------------------------" && $RESET
 
 $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------" && $RESET

.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt → .azure-pipelines/scripts/codeScan/pyspelling/inc_dict.txt

@@ -55,6 +55,7 @@ amazonlinux
 Amodei
 AmpConf
 AMX
+amx
 analytics
 Analytics
 Anastasiia
@@ -149,6 +150,7 @@ berts
 bertsquad
 BertTokenizer
 bfloat
+blockwise
 BFP
 BGR
 Bianchi
@@ -326,6 +328,7 @@ convolutional
 Convolutional
 ConvPerStage
 ConvReLU
+cooldown
 copt
 coreml
 CoreML
@@ -546,6 +549,7 @@ ensp
 entrypoint
 enum
 env
+environ
 eq
 erf
 Erf
@@ -696,6 +700,7 @@ Goyal
 gpg
 GPG
 gpt
+GPTJ
 gpu
 gpus
 GPUs
@@ -738,6 +743,7 @@ horovodrun
 hostfile
 Hounsfield
 howpublished
+hyp
 HqEgzS
 href
 html
@@ -787,6 +793,7 @@ IML
 impl
 ImportError
 IMS
+ibean
 inceptionresnetv
 InceptionResNetV
 inceptionv
@@ -831,6 +838,7 @@ ipc
 ipex
 IPEX
 ipynb
+ipynbrun
 ipython
 ir
 irv
@@ -843,6 +851,7 @@ IssueQueryThreads
 iter
 IteratorGetNext
 iters
+intrinsics
 Jäger
 jemalloc
 Jens
@@ -1173,6 +1182,7 @@ ngatang
 NGPUS
 ngram
 NHWC
+ni
 NIC
 nifti
 niftis
@@ -1234,8 +1244,11 @@ nvidia
 NVIDIA
 NVIDIA's
 nvme
+nw
 Nx
+NxM
 nyu
+oc
 ok
 ol
 Omer
@@ -1245,6 +1258,7 @@ oneapi
 oneAPI
 onednn
 oneDNN
+oneshot
 onlinedocs
 onnx
 ONNX
@@ -1783,6 +1797,7 @@ TestSettings
 tf
 TF
 TFBertForSequenceClassification
+tfhub
 tflite
 tfp
 tfrecord
@@ -1878,6 +1893,7 @@ UI
 UID
 uint
 uk
+ultralytics
 un
 uncomment
 uncompress
@@ -1888,6 +1904,7 @@ unidecode
 uniq
 unittest
 unref
+unscale
 unsqueeze
 unstack
 upenn
@@ -2114,6 +2131,7 @@ tensorrt
 hardwares
 BenchmarkConf
 PruningConf
+Pruning's
 DistillationConf
 grey
 ModelZoo
@@ -2379,3 +2397,66 @@ grappler
 amsgrad
 qoperator
 apis
+CPz
+PostTrainingQuantConfig
+dgpu
+Nsh
+UmK
+fe
+vmware
+keepachangelog
+vscode
+IntelNeuralCompressor
+SettingsPython
+VSCode
+argparse
+autoEnabling
+clickAuto
+clickEnable
+clickSetting
+connectSSH
+enableHistory
+historyDetail
+itemName
+leftIcon
+outPut
+settingPath
+topRight
+visualstudio
+amodio
+dbaeumer
+dropdown
+eslint
+registerCommand
+tsl
+viewlet
+PythonLauncher
+BigDL
+BigDLNanoSupport
+Nano
+bigdl
+inferenceoptimizer
+nano
+SageMaker
+bb
+beba
+ccdb
+ceba
+deeb
+ebbce
+efe
+npmjs
+AWSSageMakerSupport
+sagemaker
+xpu
+dgpu
+BenchmarkConfig
+QuantizationAwareTrainingConfig
+Startup
+doesn
+startup
+Ajanthan
+WeightPruningConfig
+Namhoon
+Thalaiyasingam
+Torr

.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml

@@ -4,8 +4,8 @@ matrix:
     d: en_US.ISO8859-15
   dictionary:
     wordlists:
-    - ${DICT_DIR}/lpot_dict.txt
-    output: ${DICT_DIR}/lpot_dict.dic
+    - ${DICT_DIR}/inc_dict.txt
+    output: ${DICT_DIR}/inc_dict.dic
   sources:
   - ${REPO_DIR}/docs/source/*.md
   - ${REPO_DIR}/*.md

.azure-pipelines/scripts/models/collect_log_all.py

@@ -1,9 +1,11 @@
 import argparse
 import os
+import requests
 
 parser = argparse.ArgumentParser(allow_abbrev=False)
 parser.add_argument("--logs_dir", type=str, default=".")
 parser.add_argument("--output_dir", type=str, default=".")
+parser.add_argument("--build_id", type=str, default="0")
 args = parser.parse_args()
 print(args)
 
@@ -12,20 +14,21 @@ def main():
     file_dir = args.logs_dir
     summary_content = ['OS;Platform;Framework;Version;Precision;Model;Mode;Type;BS;Value;Url\n']
     tuning_info_content = ['OS;Platform;Framework;Version;Model;Strategy;Tune_time\n']
+    url_dict = parse_download_url()
     # get full path of all files
     for root, dirs, files in os.walk(file_dir):
         for name in files:
             file_name = os.path.join(root, name)
             print(file_name)
             if '_summary.log' in name:
                 for line in open(file_name, "r"):
-                    # print(line)
                     if 'linux' in line:
+                        line = line.replace("<url>", parse_summary_log(line, url_dict))
                         summary_content.append(line)
             if '_tuning_info.log' in name:
                 for line in open(file_name, "r"):
-                    # print(line)
                     if 'linux' in line:
+                        line = line.replace("<url>", parse_tuning_log(line, url_dict))
                         tuning_info_content.append(line)
     f = open(args.output_dir + '/summary.log', "a")
     for summary in summary_content:
@@ -35,5 +38,39 @@ def main():
         f2.writelines(str(tuning_info))
 
 
+def parse_tuning_log(line, url_dict):
+    """Parsing {Framework}-{Model}-tune.log to get tuning result"""
+    result = line.split(";")
+    OS, Platform, Framework, Version, Model, Strategy, Tune_time, Tuning_trials, URL, __ = result
+    file_name = f"{Framework}-{Model}-tune.log"
+    download_url = url_dict.get(f"{Framework}_{Model}")
+    download_url = f"{download_url}{file_name}"
+    return download_url
+
+
+def parse_summary_log(line, url_dict):
+    """Parse {Framework}-{Model}-tune.log to get benchmarking accuracy result"""
+    result = line.split(";")
+    OS, Platform, Framework, Version, Precision, Model, Mode, Type, BS, Value, Url = result
+    file_name = f"{Framework}-{Model}-tune.log"
+    download_url = url_dict.get(f"{Framework}_{Model}")
+    download_url = f"{download_url}{file_name}"
+    return download_url
+
+
+def parse_download_url():
+    """Get azure artifact information"""
+    azure_artifact_api_url = f'https://dev.azure.com/lpot-inc/neural-compressor/_apis/build/builds/{args.build_id}/artifacts?api-version=5.1'
+    azure_artifacts_data = dict(requests.get(azure_artifact_api_url).json().items())
+    artifact_count = azure_artifacts_data.get("count")
+    artifact_value = azure_artifacts_data.get("value")
+    url_dict = {}
+    for item in artifact_value:
+        artifact_download_url = item.get("resource").get("downloadUrl")
+        artifact_download_url = f"{artifact_download_url[:-3]}file&subPath=%2F"
+        url_dict[item.get("name")] = artifact_download_url
+    return url_dict
+
+
 if __name__ == '__main__':
     main()

.azure-pipelines/scripts/models/collect_log_model.py

@@ -133,9 +133,9 @@ def collect_log():
                 parse_tuning_line(line, tmp)
         print(tmp)
 
-        results.append('{};{};{};{};FP32;{};Inference;Accuracy;1;{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, args.model, tmp['fp32_acc'], URL))
-        results.append('{};{};{};{};INT8;{};Inference;Accuracy;1;{};{}\n'.format(OS, PLATFORM, args.framework,  args.fwk_ver, args.model, tmp['int8_acc'], URL))
-        tuning_infos.append(';'.join([OS, PLATFORM, args.framework,  args.fwk_ver, args.model, tmp['strategy'], str(tmp['tune_time']), str(tmp['tuning_trials']), URL, f"{round(tmp['max_mem_size'] / tmp['total_mem_size'] * 100, 4)}%"])+'\n')
+        results.append('{};{};{};{};FP32;{};Inference;Accuracy;1;{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, args.model, tmp['fp32_acc'], "<url>"))
+        results.append('{};{};{};{};INT8;{};Inference;Accuracy;1;{};{}\n'.format(OS, PLATFORM, args.framework,  args.fwk_ver, args.model, tmp['int8_acc'], "<url>"))
+        tuning_infos.append(';'.join([OS, PLATFORM, args.framework,  args.fwk_ver, args.model, tmp['strategy'], str(tmp['tune_time']), str(tmp['tuning_trials']), "<url>", f"{round(tmp['max_mem_size'] / tmp['total_mem_size'] * 100, 4)}%"])+'\n')
     # get model benchmark results
     for precision in ['int8', 'fp32']:
         throughput = 0.0

.azure-pipelines/scripts/models/generate_report.sh

@@ -198,7 +198,7 @@ function generate_html_core {
                         printf("<td style=\"background-color:#90EE90\">%.2f</td>", target);
                     }else if(target < 1) {
                         printf("<td style=\"background-color:#FFD2D2\">%.2f</td>", target);
-                        job_status = "fail"
+                        perf_status = "fail"
                     }else{
                         printf("<td>%.2f</td>", target);
                     }
@@ -233,11 +233,11 @@ function generate_html_core {
                     printf("<td style=\"%s\" colspan=2>%.2f %</td>", status_png, target*100);
                 } else {
                     target = new_result / previous_result;
-                    if(target <= 1.104 && target >= 0.895) {
+                    if(target <= 1.054 && target >= 0.945) {
                         status_png = "background-color:#90EE90";
                     } else {
                         status_png = "background-color:#FFD2D2";
-                        job_status = "fail"
+                        perf_status = "fail"
                     }
                     printf("<td style=\"%s\" colspan=2>%.2f</td>", status_png, target);
                 }
@@ -265,15 +265,15 @@ function generate_html_core {
                     status_png = "background-color:#90EE90";
                 } else {
                     status_png = "background-color:#FFD2D2";
-                    job_status = "fail"
+                    ratio_status = "fail"
                 }
                 printf("<td style=\"%s\">%.2f</td>", status_png, target);
             } else {
                 if (new_result == nan && previous_result == nan) {
                     printf("<td class=\"col-cell col-cell3\"></td>");
                 } else {
                     if (new_result == nan) {
-                        job_status = "fail"
+                        ratio_status = "fail"
                         status_png = "background-color:#FFD2D2";
                         printf("<td style=\"%s\"></td>", status_png);
                     } else {
@@ -285,6 +285,8 @@ function generate_html_core {
 
         BEGIN {
             job_status = "pass"
+            perf_status = "pass"
+            ratio_status = "pass"
             // issue list
             jira_mobilenet = "https://jira01.devtools.intel.com/browse/PADDLEQ-384";
             jira_resnext = "https://jira01.devtools.intel.com/browse/PADDLEQ-387";
@@ -378,8 +380,11 @@ function generate_html_core {
 
             printf("</tr>\n");
 
+            status = (perf_status == "fail" && ratio_status == "fail") ? "fail" : "pass"
+            status = (job_status == "fail") ? "fail" : status
+
         } END{
-            printf("\n%s", job_status);
+            printf("\n%s", status);
         }
     ' >> ${output_dir}/report.html
     job_state=$(tail -1 ${WORKSPACE}/report.html)