ESP-IDF support and generic target platforms (#2303)

* Socket refactor and SSL

* esp-idf temp

* Fixes

* Echo component and noise

* Add noise API transport support

* Updates

* ESP-IDF

* Complete

* Fixes

* Fixes

* Versions update

* New i2c APIs

* Complete i2c refactor

* SPI migration

* Revert ESP Preferences migration, too complex for now

* OTA support

* Remove echo again

* Remove ssl again

* GPIOFlags updates

* Rename esphal and ICACHE_RAM_ATTR

* Make ESP32 arduino compilable again

* Fix GPIO flags

* Complete pin registry refactor and fixes

* Fixes to make test1 compile

* Remove sdkconfig file

* Ignore sdkconfig file

* Fixes in reviewing

* Make test2 compile

* Make test4 compile

* Make test5 compile

* Run clang-format

* Fix lint errors

* Use esp-idf APIs instead of btStart

* Another round of fixes

* Start implementing ESP8266

* Make test3 compile

* Guard esp8266 code

* Lint

* Reformat

* Fixes

* Fixes v2

* more fixes

* ESP-IDF tidy target

* Convert ARDUINO_ARCH_ESPxx

* Update WiFiSignalSensor

* Update time ifdefs

* OTA needs millis from hal

* RestartSwitch needs delay from hal

* ESP-IDF Uart

* Fix OTA blank password

* Allow setting sdkconfig

* Fix idf partitions and allow setting sdkconfig from yaml

* Re-add read/write compat APIs and fix esp8266 uart

* Fix esp8266 store log strings in flash

* Fix ESP32 arduino preferences not initialized

* Update ifdefs

* Change how sdkconfig change is detected

* Add checks to ci-custom and fix them

* Run clang-format

* Add esp-idf clang-tidy target and fix errors

* Fixes from clang-tidy idf round 2

* Fixes from compiling tests with esp-idf

* Run clang-format

* Switch test5.yaml to esp-idf

* Implement ESP8266 Preferences

* Lint

* Re-do PIO package version selection a bit

* Fix arduinoespressif32 package version

* Fix unit tests

* Lint

* Lint fixes

* Fix readv/writev not defined

* Fix graphing component

* Re-add all old options from core/config.py

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

platformio.ini

@@ -5,7 +5,7 @@
 
 [platformio]
 default_envs = esp8266, esp32
-src_dir = .
+src_dir = esphome
 include_dir =
 
 [runtime]
@@ -47,7 +47,11 @@ src_filter =
 
 [common:esp8266]
 extends = common
-platform = platformio/espressif8266@3.1.0
+; when changing this also copy it to esphome-docker-base images
+platform = platformio/espressif8266 @ 3.2.0
+platform_packages =
+    platformio/framework-arduinoespressif8266 @ ~3.30002.0
+
 framework = arduino
 board = nodemcuv2
 lib_deps =
@@ -55,29 +59,88 @@ lib_deps =
     ESP8266WiFi                           ; wifi (Arduino built-in)
     Update                                ; ota (Arduino built-in)
     ottowinter/ESPAsyncTCP-esphome@1.2.3  ; async_tcp
+build_flags =
+    ${common.build_flags}
+    -DUSE_ESP8266
 
 [common:esp32]
 extends = common
-platform = platformio/espressif32@3.2.0
+; when changing this also copy it to esphome-docker-base images
+platform = platformio/espressif32 @ 3.3.2
+platform_packages =
+    platformio/framework-arduinoespressif32 @ ~3.10006.0
+
 framework = arduino
 board = nodemcu-32s
 lib_deps =
     ${common.lib_deps}
     Hash                            ; ota (Arduino built-in)
     esphome/AsyncTCP-esphome@1.2.2  ; async_tcp
+build_flags =
+    ${common.build_flags}
+    -DUSE_ESP32
+src_filter = ${common.src_filter}
+
+[common:espidf]
+; when changing this also copy it to esphome-docker-base images
+platform = platformio/espressif32 @ 3.3.2
+platform_packages =
+    platformio/framework-espidf @ ~3.40300.0
+
+framework = espidf
+board = nodemcu-32s
+lib_deps =
+    esphome/noise-c@0.1.1  ; used by api
+    espressif/esp32-camera@1.0.0  ; used by esp32_camera
+    NeoPixelBus@2.6.7
+build_flags =
+    ${common.build_flags}
+    ${common:esp32.build_flags}
+    -DUSE_ESP_IDF
+    -DUSE_ESP32_FRAMEWORK_ESP_IDF
+src_filter = ${common:esp32.src_filter}
 
 [env:esp8266]
 extends = common:esp8266
-build_flags = ${common:esp8266.build_flags} ${runtime.build_flags}
+build_flags =
+    ${common:esp8266.build_flags}
+    ${runtime.build_flags}
+    -DUSE_ARDUINO
+    -DUSE_ESP8266_FRAMEWORK_ARDUINO
 
 [env:esp8266-tidy]
 extends = common:esp8266
-build_flags = ${common:esp8266.build_flags} ${clangtidy.build_flags}
+build_flags =
+    ${common:esp8266.build_flags}
+    ${clangtidy.build_flags}
+    -DUSE_ARDUINO
+    -DUSE_ESP8266_FRAMEWORK_ARDUINO
 
 [env:esp32]
 extends = common:esp32
-build_flags = ${common:esp32.build_flags} ${runtime.build_flags}
+build_flags =
+    ${common:esp32.build_flags}
+    ${runtime.build_flags}
+    -DUSE_ARDUINO
+    -DUSE_ESP32_FRAMEWORK_ARDUINO
 
 [env:esp32-tidy]
 extends = common:esp32
-build_flags = ${common:esp32.build_flags} ${clangtidy.build_flags}
+build_flags =
+    ${common:esp32.build_flags}
+    ${clangtidy.build_flags}
+    -DUSE_ARDUINO
+    -DUSE_ESP32_FRAMEWORK_ARDUINO
+
+[env:esp32-idf]
+extends = common:espidf
+build_flags =
+    ${common:espidf.build_flags}
+    ${runtime.build_flags}
+
+
+[env:esp32-idf-tidy]
+extends = common:espidf
+build_flags =
+    ${common:espidf.build_flags}
+    ${clangtidy.build_flags}