diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py
index 6f577d292..59c602933 100644
--- a/esphome/components/esp32/__init__.py
+++ b/esphome/components/esp32/__init__.py
@@ -883,6 +883,12 @@ async def to_code(config):
         CORE.relative_internal_path(".espressif")
     )
 
+    add_extra_script(
+        "pre",
+        "pre_build.py",
+        Path(__file__).parent / "pre_build.py.script",
+    )
+
     add_extra_script(
         "post",
         "post_build.py",
diff --git a/esphome/components/esp32/pre_build.py.script b/esphome/components/esp32/pre_build.py.script
new file mode 100644
index 000000000..af12275a0
--- /dev/null
+++ b/esphome/components/esp32/pre_build.py.script
@@ -0,0 +1,9 @@
+Import("env")  # noqa: F821
+
+# Remove custom_sdkconfig from the board config as it causes
+# pioarduino to enable some strange hybrid build mode that breaks IDF
+board = env.BoardConfig()
+if "espidf.custom_sdkconfig" in board:
+    del board._manifest["espidf"]["custom_sdkconfig"]
+    if not board._manifest["espidf"]:
+        del board._manifest["espidf"]