diff --git a/torchaudio/csrc/CMakeLists.txt b/torchaudio/csrc/CMakeLists.txt
index b930773eec..f21149d380 100644
--- a/torchaudio/csrc/CMakeLists.txt
+++ b/torchaudio/csrc/CMakeLists.txt
@@ -163,12 +163,13 @@ if(USE_FFMPEG)
 endif()
 
 ################################################################################
+# TODO: Rename this to _torchaudio_sox.so
 # _torchaudio.so
 ################################################################################
 if (BUILD_TORCHAUDIO_PYTHON_EXTENSION)
   set(
     EXTENSION_SOURCES
-    pybind/pybind.cpp
+    sox/pybind/pybind.cpp
     )
   #----------------------------------------------------------------------------#
   # START OF CUSTOMIZATION LOGICS
@@ -177,10 +178,10 @@ if (BUILD_TORCHAUDIO_PYTHON_EXTENSION)
     list(
       APPEND
       EXTENSION_SOURCES
-      pybind/sox/effects.cpp
-      pybind/sox/effects_chain.cpp
-      pybind/sox/io.cpp
-      pybind/sox/utils.cpp
+      sox/pybind/effects.cpp
+      sox/pybind/effects_chain.cpp
+      sox/pybind/io.cpp
+      sox/pybind/utils.cpp
       )
   endif()
   #----------------------------------------------------------------------------#
diff --git a/torchaudio/csrc/pybind/sox/effects.cpp b/torchaudio/csrc/sox/pybind/effects.cpp
similarity index 97%
rename from torchaudio/csrc/pybind/sox/effects.cpp
rename to torchaudio/csrc/sox/pybind/effects.cpp
index f5a6bd6ba0..43212c83fc 100644
--- a/torchaudio/csrc/pybind/sox/effects.cpp
+++ b/torchaudio/csrc/sox/pybind/effects.cpp
@@ -1,6 +1,6 @@
-#include <torchaudio/csrc/pybind/sox/effects.h>
-#include <torchaudio/csrc/pybind/sox/effects_chain.h>
-#include <torchaudio/csrc/pybind/sox/utils.h>
+#include <torchaudio/csrc/sox/pybind/effects.h>
+#include <torchaudio/csrc/sox/pybind/effects_chain.h>
+#include <torchaudio/csrc/sox/pybind/utils.h>
 
 using namespace torchaudio::sox_utils;
 
diff --git a/torchaudio/csrc/pybind/sox/effects.h b/torchaudio/csrc/sox/pybind/effects.h
similarity index 100%
rename from torchaudio/csrc/pybind/sox/effects.h
rename to torchaudio/csrc/sox/pybind/effects.h
diff --git a/torchaudio/csrc/pybind/sox/effects_chain.cpp b/torchaudio/csrc/sox/pybind/effects_chain.cpp
similarity index 98%
rename from torchaudio/csrc/pybind/sox/effects_chain.cpp
rename to torchaudio/csrc/sox/pybind/effects_chain.cpp
index 9770028815..05e7351d28 100644
--- a/torchaudio/csrc/pybind/sox/effects_chain.cpp
+++ b/torchaudio/csrc/sox/pybind/effects_chain.cpp
@@ -1,6 +1,6 @@
 #include <sox.h>
-#include <torchaudio/csrc/pybind/sox/effects_chain.h>
-#include <torchaudio/csrc/pybind/sox/utils.h>
+#include <torchaudio/csrc/sox/pybind/effects_chain.h>
+#include <torchaudio/csrc/sox/pybind/utils.h>
 
 using namespace torchaudio::sox_utils;
 
diff --git a/torchaudio/csrc/pybind/sox/effects_chain.h b/torchaudio/csrc/sox/pybind/effects_chain.h
similarity index 100%
rename from torchaudio/csrc/pybind/sox/effects_chain.h
rename to torchaudio/csrc/sox/pybind/effects_chain.h
diff --git a/torchaudio/csrc/pybind/sox/io.cpp b/torchaudio/csrc/sox/pybind/io.cpp
similarity index 97%
rename from torchaudio/csrc/pybind/sox/io.cpp
rename to torchaudio/csrc/sox/pybind/io.cpp
index cb59520235..2ddc069170 100644
--- a/torchaudio/csrc/pybind/sox/io.cpp
+++ b/torchaudio/csrc/sox/pybind/io.cpp
@@ -1,8 +1,8 @@
-#include <torchaudio/csrc/pybind/sox/effects.h>
-#include <torchaudio/csrc/pybind/sox/effects_chain.h>
-#include <torchaudio/csrc/pybind/sox/io.h>
-#include <torchaudio/csrc/pybind/sox/utils.h>
 #include <torchaudio/csrc/sox/io.h>
+#include <torchaudio/csrc/sox/pybind/effects.h>
+#include <torchaudio/csrc/sox/pybind/effects_chain.h>
+#include <torchaudio/csrc/sox/pybind/io.h>
+#include <torchaudio/csrc/sox/pybind/utils.h>
 #include <torchaudio/csrc/sox/types.h>
 
 #include <utility>
diff --git a/torchaudio/csrc/pybind/sox/io.h b/torchaudio/csrc/sox/pybind/io.h
similarity index 100%
rename from torchaudio/csrc/pybind/sox/io.h
rename to torchaudio/csrc/sox/pybind/io.h
diff --git a/torchaudio/csrc/pybind/pybind.cpp b/torchaudio/csrc/sox/pybind/pybind.cpp
similarity index 87%
rename from torchaudio/csrc/pybind/pybind.cpp
rename to torchaudio/csrc/sox/pybind/pybind.cpp
index 583ee41515..448d3fc6b4 100644
--- a/torchaudio/csrc/pybind/pybind.cpp
+++ b/torchaudio/csrc/sox/pybind/pybind.cpp
@@ -1,8 +1,8 @@
 #include <torch/extension.h>
 
 #ifdef INCLUDE_SOX
-#include <torchaudio/csrc/pybind/sox/effects.h>
-#include <torchaudio/csrc/pybind/sox/io.h>
+#include <torchaudio/csrc/sox/pybind/effects.h>
+#include <torchaudio/csrc/sox/pybind/io.h>
 #endif
 
 PYBIND11_MODULE(_torchaudio, m) {
diff --git a/torchaudio/csrc/pybind/sox/utils.cpp b/torchaudio/csrc/sox/pybind/utils.cpp
similarity index 95%
rename from torchaudio/csrc/pybind/sox/utils.cpp
rename to torchaudio/csrc/sox/pybind/utils.cpp
index 39669b0250..47d3b512c6 100644
--- a/torchaudio/csrc/pybind/sox/utils.cpp
+++ b/torchaudio/csrc/sox/pybind/utils.cpp
@@ -1,4 +1,4 @@
-#include <torchaudio/csrc/pybind/sox/utils.h>
+#include <torchaudio/csrc/sox/pybind/utils.h>
 
 namespace torchaudio::sox_utils {
 
diff --git a/torchaudio/csrc/pybind/sox/utils.h b/torchaudio/csrc/sox/pybind/utils.h
similarity index 100%
rename from torchaudio/csrc/pybind/sox/utils.h
rename to torchaudio/csrc/sox/pybind/utils.h