--- a/setup.py
+++ b/setup.py
@@ -470,14 +470,6 @@
     check_submodules()
     check_pydep("yaml", "pyyaml")
     build_python = not BUILD_LIBTORCH_WHL
-    build_caffe2(
-        version=version,
-        cmake_python_library=cmake_python_library,
-        build_python=build_python,
-        rerun_cmake=RERUN_CMAKE,
-        cmake_only=CMAKE_ONLY,
-        cmake=cmake,
-    )
 
     if CMAKE_ONLY:
         report(