diff --git a/.circleci/unittest/windows/scripts/set_cuda_envs.sh b/.circleci/unittest/windows/scripts/set_cuda_envs.sh
new file mode 100644
index 00000000000..3e3b510e64b
--- /dev/null
+++ b/.circleci/unittest/windows/scripts/set_cuda_envs.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+if [ "${CU_VERSION:-}" == "cpu" ] ; then
+    exit 0
+fi
+
+if [[ ${#CU_VERSION} -eq 5 ]]; then
+    CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}"
+fi
+
+# It's a log to see if CU_VERSION exists, if not, we use environment CUDA_VERSION directly
+# in unittest_windows_gpu, there's no CU_VERSION, but CUDA_VERSION.
+echo "Using CUDA $CUDA_VERSION, CU_VERSION is $CU_VERSION now"
+
+version=$CUDA_VERSION
+
+# set cuda envs
+export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/bin:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${version}/libnvvp:$PATH"
+export CUDA_PATH_V${version/./_}="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}"
+export CUDA_PATH="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${version}"
+
+if  [ ! -d "$CUDA_PATH" ]
+then
+    echo "$CUDA_PATH" does not exist
+    exit 1
+fi
+
+# check cuda driver version
+for path in '/c/Program Files/NVIDIA Corporation/NVSMI/nvidia-smi.exe' /c/Windows/System32/nvidia-smi.exe; do
+    if [[ -x "$path" ]]; then
+        "$path" || echo "true";
+        break
+    fi
+done
+which nvcc
diff --git a/packaging/build_cmake.sh b/packaging/build_cmake.sh
index f99922bb4d5..d42d8770f59 100755
--- a/packaging/build_cmake.sh
+++ b/packaging/build_cmake.sh
@@ -45,6 +45,10 @@ else
     yum install -y libpng-devel libjpeg-turbo-devel
 fi
 
+if [[ "$OSTYPE" == "msys" ]]; then
+    source .circleci/unittest/windows/scripts/set_cuda_envs.sh
+fi
+
 mkdir cpp_build
 pushd cpp_build
 
diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat
index 57cfd3e8c0c..a93f8d2a9b1 100644
--- a/packaging/windows/internal/cuda_install.bat
+++ b/packaging/windows/internal/cuda_install.bat
@@ -187,7 +187,13 @@ start /wait setup.exe -s %ARGS%
 popd
 
 echo Installing VS integration...
-xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\IDE\VC\VCTargets\BuildCustomizations"
+rem It's for VS 2019
+if "%CUDA_VER_MAJOR%" == "10" (
+    xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160\BuildCustomizations"
+)
+if "%CUDA_VER_MAJOR%" == "11" (
+    xcopy /Y "%SRC_DIR%\temp_build\cuda\visual_studio_integration\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160\BuildCustomizations"
+)
 
 echo Installing NvToolsExt...
 7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt"