Merge remote-tracking branch 'upstream/dev' into loop_runtime_stats

.github/workflows/ci.yml

@@ -377,15 +377,7 @@ jobs:
         id: list-components
         run: |
           . venv/bin/activate
-
-          # Check if we should test all components (via label)
-          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'test-all-components') }}" == "true" ]]; then
-            echo "Label 'test-all-components' found - testing ALL components"
-            components=$(script/list-components.py)
-          else
-            components=$(script/list-components.py --changed --branch ${{ steps.target-branch.outputs.branch }})
-          fi
-
+          components=$(script/list-components.py --changed --branch ${{ steps.target-branch.outputs.branch }})
           output_components=$(echo "$components" | jq -R -s -c 'split("\n")[:-1] | map(select(length > 0))')
           count=$(echo "$output_components" | jq length)
 
@@ -423,15 +415,11 @@ jobs:
       - name: test_build_components -e config -c ${{ matrix.file }}
         run: |
           . venv/bin/activate
-          # Use 4 parallel jobs for config validation
-          ./script/test_build_components -e config -c ${{ matrix.file }} -j 4 -f
+          ./script/test_build_components -e config -c ${{ matrix.file }}
       - name: test_build_components -e compile -c ${{ matrix.file }}
         run: |
           . venv/bin/activate
-          mkdir -p build_cache
-          export PLATFORMIO_BUILD_CACHE_DIR=$PWD/build_cache
-          # Use 2 parallel jobs for compilation (resource intensive)
-          ./script/test_build_components -e compile -c ${{ matrix.file }} -j 2 -f -b $PWD/build_cache
+          ./script/test_build_components -e compile -c ${{ matrix.file }}
 
   test-build-components-splitter:
     name: Split components for testing into 20 groups maximum
@@ -483,28 +471,17 @@ jobs:
       - name: Validate config
         run: |
           . venv/bin/activate
-          # Process all components in parallel for config validation
-          components="${{ matrix.components }}"
-          # Convert space-separated list to multiple -c flags
-          component_args=""
-          for component in $components; do
-            component_args="$component_args -c $component"
+          for component in ${{ matrix.components }}; do
+            ./script/test_build_components -e config -c $component
           done
-          # Use 8 parallel jobs for lightweight config validation
-          ./script/test_build_components -e config $component_args -j 8 -f
       - name: Compile config
         run: |
           . venv/bin/activate
-          mkdir -p build_cache
+          mkdir build_cache
           export PLATFORMIO_BUILD_CACHE_DIR=$PWD/build_cache
-          # Process all components in parallel for compilation
-          components="${{ matrix.components }}"
-          component_args=""
-          for component in $components; do
-            component_args="$component_args -c $component"
+          for component in ${{ matrix.components }}; do
+            ./script/test_build_components -e compile -c $component
           done
-          # Use 4 parallel jobs for resource-intensive compilation
-          ./script/test_build_components -e compile $component_args -j 4 -f -b $PWD/build_cache
 
   ci-status:
     name: CI Status

esphome/components/nextion/nextion_upload_arduino.cpp

@@ -337,26 +337,23 @@ bool Nextion::upload_tft(uint32_t baud_rate, bool exit_reparse) {
 
 bool Nextion::upload_end_(bool successful) {
   ESP_LOGD(TAG, "TFT upload done: %s", YESNO(successful));
+  this->is_updating_ = false;
+  this->ignore_is_setup_ = false;
+
+  uint32_t baud_rate = this->parent_->get_baud_rate();
+  if (baud_rate != this->original_baud_rate_) {
+    ESP_LOGD(TAG, "Baud back: %" PRIu32 "->%" PRIu32, baud_rate, this->original_baud_rate_);
+    this->parent_->set_baud_rate(this->original_baud_rate_);
+    this->parent_->load_settings();
+  }
 
   if (successful) {
     ESP_LOGD(TAG, "Restart");
     delay(1500);  // NOLINT
     App.safe_reboot();
-    delay(1500);  // NOLINT
   } else {
     ESP_LOGE(TAG, "TFT upload failed");
-
-    this->is_updating_ = false;
-    this->ignore_is_setup_ = false;
-
-    uint32_t baud_rate = this->parent_->get_baud_rate();
-    if (baud_rate != this->original_baud_rate_) {
-      ESP_LOGD(TAG, "Baud back: %" PRIu32 "->%" PRIu32, baud_rate, this->original_baud_rate_);
-      this->parent_->set_baud_rate(this->original_baud_rate_);
-      this->parent_->load_settings();
-    }
   }
-
   return successful;
 }
 

esphome/components/nextion/nextion_upload_idf.cpp

@@ -337,6 +337,15 @@ bool Nextion::upload_tft(uint32_t baud_rate, bool exit_reparse) {
 
 bool Nextion::upload_end_(bool successful) {
   ESP_LOGD(TAG, "TFT upload done: %s", YESNO(successful));
+  this->is_updating_ = false;
+  this->ignore_is_setup_ = false;
+
+  uint32_t baud_rate = this->parent_->get_baud_rate();
+  if (baud_rate != this->original_baud_rate_) {
+    ESP_LOGD(TAG, "Baud back: %" PRIu32 "->%" PRIu32, baud_rate, this->original_baud_rate_);
+    this->parent_->set_baud_rate(this->original_baud_rate_);
+    this->parent_->load_settings();
+  }
 
   if (successful) {
     ESP_LOGD(TAG, "Restart");
@@ -344,18 +353,7 @@ bool Nextion::upload_end_(bool successful) {
     App.safe_reboot();
   } else {
     ESP_LOGE(TAG, "TFT upload failed");
-
-    this->is_updating_ = false;
-    this->ignore_is_setup_ = false;
-
-    uint32_t baud_rate = this->parent_->get_baud_rate();
-    if (baud_rate != this->original_baud_rate_) {
-      ESP_LOGD(TAG, "Baud back: %" PRIu32 "->%" PRIu32, baud_rate, this->original_baud_rate_);
-      this->parent_->set_baud_rate(this->original_baud_rate_);
-      this->parent_->load_settings();
-    }
   }
-
   return successful;
 }
 

esphome/components/runtime_stats/__init__.py

@@ -0,0 +1,26 @@
+"""
+Runtime statistics component for ESPHome.
+"""
+
+import esphome.codegen as cg
+import esphome.config_validation as cv
+
+DEPENDENCIES = []
+
+CONF_ENABLED = "enabled"
+CONF_LOG_INTERVAL = "log_interval"
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.Optional(CONF_ENABLED, default=True): cv.boolean,
+        cv.Optional(
+            CONF_LOG_INTERVAL, default=60000
+        ): cv.positive_time_period_milliseconds,
+    }
+)
+
+
+async def to_code(config):
+    """Generate code for the runtime statistics component."""
+    cg.add(cg.App.set_runtime_stats_enabled(config[CONF_ENABLED]))
+    cg.add(cg.App.set_runtime_stats_log_interval(config[CONF_LOG_INTERVAL]))

esphome/core/application.cpp

@@ -117,9 +117,7 @@ void Application::loop() {
   // Use the last component's end time instead of calling millis() again
   auto elapsed = last_op_end_time - this->last_loop_;
   if (elapsed >= this->loop_interval_ || HighFrequencyLoopRequester::is_high_frequency()) {
-    // Even if we overran the loop interval, we still need to select()
-    // to know if any sockets have data ready
-    this->yield_with_select_(0);
+    yield();
   } else {
     uint32_t delay_time = this->loop_interval_ - elapsed;
     uint32_t next_schedule = this->scheduler.next_schedule_in().value_or(delay_time);
@@ -128,7 +126,7 @@ void Application::loop() {
     next_schedule = std::max(next_schedule, delay_time / 2);
     delay_time = std::min(next_schedule, delay_time);
 
-    this->yield_with_select_(delay_time);
+    this->delay_with_select_(delay_time);
   }
   this->last_loop_ = last_op_end_time;
 
@@ -217,7 +215,7 @@ void Application::teardown_components(uint32_t timeout_ms) {
 
     // Give some time for I/O operations if components are still pending
     if (!pending_components.empty()) {
-      this->yield_with_select_(1);
+      this->delay_with_select_(1);
     }
 
     // Update time for next iteration
@@ -295,6 +293,8 @@ bool Application::is_socket_ready(int fd) const {
   // This function is thread-safe for reading the result of select()
   // However, it should only be called after select() has been executed in the main loop
   // The read_fds_ is only modified by select() in the main loop
+  if (HighFrequencyLoopRequester::is_high_frequency())
+    return true;  // fd sets via select are not updated in high frequency looping - so force true fallback behavior
   if (fd < 0 || fd >= FD_SETSIZE)
     return false;
 
@@ -302,9 +302,7 @@ bool Application::is_socket_ready(int fd) const {
 }
 #endif
 
-void Application::yield_with_select_(uint32_t delay_ms) {
-  // Delay while monitoring sockets. When delay_ms is 0, always yield() to ensure other tasks run
-  // since select() with 0 timeout only polls without yielding.
+void Application::delay_with_select_(uint32_t delay_ms) {
 #ifdef USE_SOCKET_SELECT_SUPPORT
   if (!this->socket_fds_.empty()) {
     // Update fd_set if socket list has changed
@@ -342,10 +340,6 @@ void Application::yield_with_select_(uint32_t delay_ms) {
       ESP_LOGW(TAG, "select() failed with errno %d", errno);
       delay(delay_ms);
     }
-    // When delay_ms is 0, we need to yield since select(0) doesn't yield
-    if (delay_ms == 0) {
-      yield();
-    }
   } else {
     // No sockets registered, use regular delay
     delay(delay_ms);

esphome/core/application.h

@@ -7,6 +7,7 @@
 #include "esphome/core/hal.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/preferences.h"
+#include "esphome/core/runtime_stats.h"
 #include "esphome/core/scheduler.h"
 
 #ifdef USE_SOCKET_SELECT_SUPPORT
@@ -314,6 +315,18 @@ class Application {
 
   uint32_t get_loop_interval() const { return this->loop_interval_; }
 
+  /** Enable or disable runtime statistics collection.
+   *
+   * @param enable Whether to enable runtime statistics collection.
+   */
+  void set_runtime_stats_enabled(bool enable) { runtime_stats.set_enabled(enable); }
+
+  /** Set the interval at which runtime statistics are logged.
+   *
+   * @param interval The interval in milliseconds between logging of runtime statistics.
+   */
+  void set_runtime_stats_log_interval(uint32_t interval) { runtime_stats.set_log_interval(interval); }
+
   void schedule_dump_config() { this->dump_config_at_ = 0; }
 
   void feed_wdt(uint32_t time = 0);
@@ -575,7 +588,7 @@ class Application {
   void feed_wdt_arch_();
 
   /// Perform a delay while also monitoring socket file descriptors for readiness
-  void yield_with_select_(uint32_t delay_ms);
+  void delay_with_select_(uint32_t delay_ms);
 
   std::vector<Component *> components_{};
   std::vector<Component *> looping_components_{};

esphome/core/component.cpp

@@ -246,6 +246,9 @@ uint32_t WarnIfComponentBlockingGuard::finish() {
   uint32_t curr_time = millis();
 
   uint32_t blocking_time = curr_time - this->started_;
+
+  // Record component runtime stats
+  runtime_stats.record_component_time(this->component_, blocking_time, curr_time);
   bool should_warn;
   if (this->component_ != nullptr) {
     should_warn = this->component_->should_warn_of_blocking(blocking_time);

esphome/core/component.h

@@ -6,6 +6,7 @@
 #include <string>
 
 #include "esphome/core/optional.h"
+#include "esphome/core/runtime_stats.h"
 
 namespace esphome {
 

esphome/core/runtime_stats.cpp

@@ -0,0 +1,28 @@
+#include "esphome/core/runtime_stats.h"
+#include "esphome/core/component.h"
+
+namespace esphome {
+
+RuntimeStatsCollector runtime_stats;
+
+void RuntimeStatsCollector::record_component_time(Component *component, uint32_t duration_ms, uint32_t current_time) {
+  if (!this->enabled_ || component == nullptr)
+    return;
+
+  const char *component_source = component->get_component_source();
+  this->component_stats_[component_source].record_time(duration_ms);
+
+  // If next_log_time_ is 0, initialize it
+  if (this->next_log_time_ == 0) {
+    this->next_log_time_ = current_time + this->log_interval_;
+    return;
+  }
+
+  if (current_time >= this->next_log_time_) {
+    this->log_stats_();
+    this->reset_stats_();
+    this->next_log_time_ = current_time + this->log_interval_;
+  }
+}
+
+}  // namespace esphome

esphome/core/runtime_stats.h

@@ -0,0 +1,161 @@
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+#include <cstdint>
+#include <algorithm>
+#include "esphome/core/helpers.h"
+#include "esphome/core/log.h"
+
+namespace esphome {
+
+static const char *const RUNTIME_TAG = "runtime";
+
+class Component;  // Forward declaration
+
+class ComponentRuntimeStats {
+ public:
+  ComponentRuntimeStats()
+      : period_count_(0),
+        total_count_(0),
+        period_time_ms_(0),
+        total_time_ms_(0),
+        period_max_time_ms_(0),
+        total_max_time_ms_(0) {}
+
+  void record_time(uint32_t duration_ms) {
+    // Update period counters
+    this->period_count_++;
+    this->period_time_ms_ += duration_ms;
+    if (duration_ms > this->period_max_time_ms_)
+      this->period_max_time_ms_ = duration_ms;
+
+    // Update total counters
+    this->total_count_++;
+    this->total_time_ms_ += duration_ms;
+    if (duration_ms > this->total_max_time_ms_)
+      this->total_max_time_ms_ = duration_ms;
+  }
+
+  void reset_period_stats() {
+    this->period_count_ = 0;
+    this->period_time_ms_ = 0;
+    this->period_max_time_ms_ = 0;
+  }
+
+  // Period stats (reset each logging interval)
+  uint32_t get_period_count() const { return this->period_count_; }
+  uint32_t get_period_time_ms() const { return this->period_time_ms_; }
+  uint32_t get_period_max_time_ms() const { return this->period_max_time_ms_; }
+  float get_period_avg_time_ms() const {
+    return this->period_count_ > 0 ? this->period_time_ms_ / static_cast<float>(this->period_count_) : 0.0f;
+  }
+
+  // Total stats (persistent until reboot)
+  uint32_t get_total_count() const { return this->total_count_; }
+  uint32_t get_total_time_ms() const { return this->total_time_ms_; }
+  uint32_t get_total_max_time_ms() const { return this->total_max_time_ms_; }
+  float get_total_avg_time_ms() const {
+    return this->total_count_ > 0 ? this->total_time_ms_ / static_cast<float>(this->total_count_) : 0.0f;
+  }
+
+ protected:
+  // Period stats (reset each logging interval)
+  uint32_t period_count_;
+  uint32_t period_time_ms_;
+  uint32_t period_max_time_ms_;
+
+  // Total stats (persistent until reboot)
+  uint32_t total_count_;
+  uint32_t total_time_ms_;
+  uint32_t total_max_time_ms_;
+};
+
+// For sorting components by run time
+struct ComponentStatPair {
+  std::string name;
+  const ComponentRuntimeStats *stats;
+
+  bool operator>(const ComponentStatPair &other) const {
+    // Sort by period time as that's what we're displaying in the logs
+    return stats->get_period_time_ms() > other.stats->get_period_time_ms();
+  }
+};
+
+class RuntimeStatsCollector {
+ public:
+  RuntimeStatsCollector() : log_interval_(60000), next_log_time_(0), enabled_(true) {}
+
+  void set_log_interval(uint32_t log_interval) { this->log_interval_ = log_interval; }
+  uint32_t get_log_interval() const { return this->log_interval_; }
+
+  void set_enabled(bool enabled) { this->enabled_ = enabled; }
+  bool is_enabled() const { return this->enabled_; }
+
+  void record_component_time(Component *component, uint32_t duration_ms, uint32_t current_time);
+
+ protected:
+  void log_stats_() {
+    ESP_LOGI(RUNTIME_TAG, "Component Runtime Statistics");
+    ESP_LOGI(RUNTIME_TAG, "Period stats (last %" PRIu32 "ms):", this->log_interval_);
+
+    // First collect stats we want to display
+    std::vector<ComponentStatPair> stats_to_display;
+
+    for (const auto &it : this->component_stats_) {
+      const ComponentRuntimeStats &stats = it.second;
+      if (stats.get_period_count() > 0) {
+        ComponentStatPair pair = {it.first, &stats};
+        stats_to_display.push_back(pair);
+      }
+    }
+
+    // Sort by period runtime (descending)
+    std::sort(stats_to_display.begin(), stats_to_display.end(), std::greater<ComponentStatPair>());
+
+    // Log top components by period runtime
+    for (const auto &it : stats_to_display) {
+      const std::string &source = it.name;
+      const ComponentRuntimeStats *stats = it.stats;
+
+      ESP_LOGI(RUNTIME_TAG, "  %s: count=%" PRIu32 ", avg=%.2fms, max=%" PRIu32 "ms, total=%" PRIu32 "ms",
+               source.c_str(), stats->get_period_count(), stats->get_period_avg_time_ms(),
+               stats->get_period_max_time_ms(), stats->get_period_time_ms());
+    }
+
+    // Log total stats since boot
+    ESP_LOGI(RUNTIME_TAG, "Total stats (since boot):");
+
+    // Re-sort by total runtime for all-time stats
+    std::sort(stats_to_display.begin(), stats_to_display.end(),
+              [](const ComponentStatPair &a, const ComponentStatPair &b) {
+                return a.stats->get_total_time_ms() > b.stats->get_total_time_ms();
+              });
+
+    for (const auto &it : stats_to_display) {
+      const std::string &source = it.name;
+      const ComponentRuntimeStats *stats = it.stats;
+
+      ESP_LOGI(RUNTIME_TAG, "  %s: count=%" PRIu32 ", avg=%.2fms, max=%" PRIu32 "ms, total=%" PRIu32 "ms",
+               source.c_str(), stats->get_total_count(), stats->get_total_avg_time_ms(), stats->get_total_max_time_ms(),
+               stats->get_total_time_ms());
+    }
+  }
+
+  void reset_stats_() {
+    for (auto &it : this->component_stats_) {
+      it.second.reset_period_stats();
+    }
+  }
+
+  std::map<std::string, ComponentRuntimeStats> component_stats_;
+  uint32_t log_interval_;
+  uint32_t next_log_time_;
+  bool enabled_;
+};
+
+// Global instance for runtime stats collection
+extern RuntimeStatsCollector runtime_stats;
+
+}  // namespace esphome

esphome/git.py

@@ -10,7 +10,6 @@ import urllib.parse
 
 import esphome.config_validation as cv
 from esphome.core import CORE, TimePeriodSeconds
-from esphome.git_lock import git_operation_lock
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -60,72 +59,66 @@ def clone_or_update(
         )
 
     repo_dir = _compute_destination_path(key, domain)
+    if not repo_dir.is_dir():
+        _LOGGER.info("Cloning %s", key)
+        _LOGGER.debug("Location: %s", repo_dir)
+        cmd = ["git", "clone", "--depth=1"]
+        cmd += ["--", url, str(repo_dir)]
+        run_git_command(cmd)
 
-    # Use lock to prevent concurrent access to the same repository
-    with git_operation_lock(key):
-        if not repo_dir.is_dir():
-            _LOGGER.info("Cloning %s", key)
+        if ref is not None:
+            # We need to fetch the PR branch first, otherwise git will complain
+            # about missing objects
+            _LOGGER.info("Fetching %s", ref)
+            run_git_command(["git", "fetch", "--", "origin", ref], str(repo_dir))
+            run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir))
+
+        if submodules is not None:
+            _LOGGER.info(
+                "Initialising submodules (%s) for %s", ", ".join(submodules), key
+            )
+            run_git_command(
+                ["git", "submodule", "update", "--init"] + submodules, str(repo_dir)
+            )
+
+    else:
+        # Check refresh needed
+        file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD")
+        # On first clone, FETCH_HEAD does not exists
+        if not file_timestamp.exists():
+            file_timestamp = Path(repo_dir / ".git" / "HEAD")
+        age = datetime.now() - datetime.fromtimestamp(file_timestamp.stat().st_mtime)
+        if refresh is None or age.total_seconds() > refresh.total_seconds:
+            old_sha = run_git_command(["git", "rev-parse", "HEAD"], str(repo_dir))
+            _LOGGER.info("Updating %s", key)
             _LOGGER.debug("Location: %s", repo_dir)
-            cmd = ["git", "clone", "--depth=1"]
-            cmd += ["--", url, str(repo_dir)]
-            run_git_command(cmd)
-
+            # Stash local changes (if any)
+            run_git_command(
+                ["git", "stash", "push", "--include-untracked"], str(repo_dir)
+            )
+            # Fetch remote ref
+            cmd = ["git", "fetch", "--", "origin"]
             if ref is not None:
-                # We need to fetch the PR branch first, otherwise git will complain
-                # about missing objects
-                _LOGGER.info("Fetching %s", ref)
-                run_git_command(["git", "fetch", "--", "origin", ref], str(repo_dir))
-                run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir))
+                cmd.append(ref)
+            run_git_command(cmd, str(repo_dir))
+            # Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch)
+            run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir))
 
             if submodules is not None:
                 _LOGGER.info(
-                    "Initialising submodules (%s) for %s", ", ".join(submodules), key
+                    "Updating submodules (%s) for %s", ", ".join(submodules), key
                 )
                 run_git_command(
                     ["git", "submodule", "update", "--init"] + submodules, str(repo_dir)
                 )
 
-        else:
-            # Check refresh needed
-            file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD")
-            # On first clone, FETCH_HEAD does not exists
-            if not file_timestamp.exists():
-                file_timestamp = Path(repo_dir / ".git" / "HEAD")
-            age = datetime.now() - datetime.fromtimestamp(
-                file_timestamp.stat().st_mtime
-            )
-            if refresh is None or age.total_seconds() > refresh.total_seconds:
-                old_sha = run_git_command(["git", "rev-parse", "HEAD"], str(repo_dir))
-                _LOGGER.info("Updating %s", key)
-                _LOGGER.debug("Location: %s", repo_dir)
-                # Stash local changes (if any)
-                run_git_command(
-                    ["git", "stash", "push", "--include-untracked"], str(repo_dir)
-                )
-                # Fetch remote ref
-                cmd = ["git", "fetch", "--", "origin"]
-                if ref is not None:
-                    cmd.append(ref)
-                run_git_command(cmd, str(repo_dir))
-                # Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch)
-                run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir))
+            def revert():
+                _LOGGER.info("Reverting changes to %s -> %s", key, old_sha)
+                run_git_command(["git", "reset", "--hard", old_sha], str(repo_dir))
 
-                if submodules is not None:
-                    _LOGGER.info(
-                        "Updating submodules (%s) for %s", ", ".join(submodules), key
-                    )
-                    run_git_command(
-                        ["git", "submodule", "update", "--init"] + submodules,
-                        str(repo_dir),
-                    )
+            return repo_dir, revert
 
-                def revert():
-                    _LOGGER.info("Reverting changes to %s -> %s", key, old_sha)
-                    run_git_command(["git", "reset", "--hard", old_sha], str(repo_dir))
-
-                return repo_dir, revert
-
-        return repo_dir, None
+    return repo_dir, None
 
 
 GIT_DOMAINS = {

esphome/git_lock.py

@@ -1,141 +0,0 @@
-"""File locking for git operations to prevent race conditions."""
-
-from contextlib import contextmanager
-import hashlib
-import logging
-from pathlib import Path
-import sys
-import tempfile
-import time
-
-# Platform-specific imports
-if sys.platform == "win32":
-    import msvcrt
-else:
-    import fcntl
-
-_LOGGER = logging.getLogger(__name__)
-
-# Global lock directory
-LOCK_DIR = Path(tempfile.gettempdir()) / "esphome_git_locks"
-LOCK_DIR.mkdir(exist_ok=True)
-
-
-def _acquire_lock_unix(lock_file, timeout, identifier):
-    """Acquire lock on Unix systems using fcntl."""
-    start_time = time.time()
-    last_log_time = start_time
-    while True:
-        try:
-            fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
-            return True
-        except OSError:
-            elapsed = time.time() - start_time
-            if elapsed > timeout:
-                raise TimeoutError(
-                    f"Could not acquire lock for {identifier} within {timeout}s"
-                )
-
-            # Log progress every 10 seconds
-            if time.time() - last_log_time > 10:
-                _LOGGER.info(
-                    f"Still waiting for lock {identifier} ({elapsed:.1f}s elapsed)..."
-                )
-                last_log_time = time.time()
-
-            time.sleep(0.1)
-
-
-def _release_lock_unix(lock_file):
-    """Release lock on Unix systems."""
-    try:
-        fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
-    except Exception:
-        pass
-
-
-def _acquire_lock_windows(lock_file, timeout, identifier):
-    """Acquire lock on Windows systems using msvcrt."""
-    start_time = time.time()
-    while True:
-        try:
-            msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1)
-            return True
-        except OSError:
-            if time.time() - start_time > timeout:
-                raise TimeoutError(
-                    f"Could not acquire lock for {identifier} within {timeout}s"
-                )
-            time.sleep(0.1)
-
-
-def _release_lock_windows(lock_file):
-    """Release lock on Windows systems."""
-    try:
-        msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
-    except Exception:
-        pass
-
-
-@contextmanager
-def git_operation_lock(identifier: str, timeout: float = 30.0):
-    """
-    Acquire a file lock for a git operation.
-
-    :param identifier: Unique identifier for the operation (e.g., repo URL or path)
-    :param timeout: Maximum time to wait for the lock in seconds
-    """
-    # Create a safe filename from the identifier
-    lock_name = hashlib.sha256(identifier.encode()).hexdigest()[:16]
-    lock_path = LOCK_DIR / f"{lock_name}.lock"
-
-    # Ensure lock file exists
-    lock_path.touch(exist_ok=True)
-
-    lock_file = None
-    acquired = False
-
-    try:
-        # Open in binary mode for Windows compatibility
-        lock_file = open(lock_path, "r+b")
-
-        # Platform-specific lock acquisition
-        if sys.platform == "win32":
-            acquired = _acquire_lock_windows(lock_file, timeout, identifier)
-        else:
-            acquired = _acquire_lock_unix(lock_file, timeout, identifier)
-
-        if acquired:
-            _LOGGER.debug(f"Acquired lock for {identifier}")
-
-        yield
-
-    finally:
-        if lock_file:
-            if acquired:
-                # Platform-specific lock release
-                if sys.platform == "win32":
-                    _release_lock_windows(lock_file)
-                else:
-                    _release_lock_unix(lock_file)
-                _LOGGER.debug(f"Released lock for {identifier}")
-            lock_file.close()
-
-
-@contextmanager
-def platformio_init_lock(timeout: float = 30.0):
-    """Lock for PlatformIO initialization to prevent race conditions."""
-    with git_operation_lock("platformio_init", timeout=timeout):
-        yield
-
-
-@contextmanager
-def platformio_install_lock(package_name: str, timeout: float = 300.0):
-    """Lock for PlatformIO package installation to prevent race conditions."""
-    _LOGGER.info(
-        f"Waiting for PlatformIO package installation lock ({package_name})..."
-    )
-    with git_operation_lock(f"platformio_install_{package_name}", timeout=timeout):
-        _LOGGER.info(f"Acquired PlatformIO package installation lock ({package_name})")
-        yield
-        _LOGGER.info(f"Released PlatformIO package installation lock ({package_name})")

esphome/platformio_api.py

@@ -86,28 +86,9 @@ def run_platformio_cli(*args, **kwargs) -> str | int:
     if os.environ.get("ESPHOME_USE_SUBPROCESS") is not None:
         return run_external_process(*cmd, **kwargs)
 
-    # Import with minimal locking to prevent initialization race conditions
-    from esphome.git_lock import platformio_init_lock
-
-    with platformio_init_lock():
-        import platformio.__main__
+    import platformio.__main__
 
     patch_structhash()
-
-    # For first-time PlatformIO runs, use a lock to prevent directory creation conflicts
-    home_pio = Path.home() / ".platformio"
-    if not home_pio.exists() and len(args) > 0 and args[0] == "run":
-        from esphome.git_lock import platformio_install_lock
-
-        _LOGGER.info("First PlatformIO run detected, using initialization lock...")
-        with platformio_install_lock("first_run", timeout=120.0):
-            # Create the directory if it still doesn't exist
-            home_pio.mkdir(exist_ok=True)
-            result = run_external_command(platformio.__main__.main, *cmd, **kwargs)
-            _LOGGER.info("First PlatformIO run completed")
-            return result
-
-    # Normal execution without locking
     return run_external_command(platformio.__main__.main, *cmd, **kwargs)
 
 

requirements_test.txt

@@ -6,7 +6,7 @@ pre-commit
 
 # Unit tests
 pytest==8.4.0
-pytest-cov==6.2.1
+pytest-cov==6.1.1
 pytest-mock==3.14.1
 pytest-asyncio==0.26.0
 pytest-xdist==3.7.0

script/test_build_components

@@ -3,37 +3,25 @@
 set -e
 
 help() {
-  echo "Usage: $0 [-e <config|compile|clean>] [-c <string>] [-t <string>] [-j <number>] [-p <string>] [-f]" 1>&2
+  echo "Usage: $0 [-e <config|compile|clean>] [-c <string>] [-t <string>]" 1>&2
   echo 1>&2
   echo "  - e - Parameter for esphome command. Default compile. Common alternative is config." 1>&2
   echo "  - c - Component folder name to test. Default *. E.g. '-c logger'." 1>&2
   echo "  - t - Target name to test. Put '-t list' to display all possibilities. E.g. '-t esp32-s2-idf-51'." 1>&2
-  echo "  - j - Number of parallel jobs. Default is number of CPU cores." 1>&2
-  echo "  - p - Platform filter. E.g. '-p esp32' to test only ESP32 platforms." 1>&2
-  echo "  - f - Fail fast. Exit on first failure." 1>&2
-  echo "  - b - Build cache directory. E.g. '-b /tmp/esphome_cache'." 1>&2
   exit 1
 }
 
 # Parse parameter:
+# - `e` - Parameter for `esphome` command. Default `compile`. Common alternative is `config`.
+# - `c` - Component folder name to test. Default `*`.
 esphome_command="compile"
 target_component="*"
-num_jobs=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
-platform_filter=""
-fail_fast=false
-build_cache_dir=""
-
-while getopts e:c:t:j:p:b:fh flag
+while getopts e:c:t: flag
 do
     case $flag in
         e) esphome_command=${OPTARG};;
         c) target_component=${OPTARG};;
         t) requested_target_platform=${OPTARG};;
-        j) num_jobs=${OPTARG};;
-        p) platform_filter=${OPTARG};;
-        f) fail_fast=true;;
-        b) build_cache_dir=${OPTARG};;
-        h) help;;
         \?) help;;
     esac
 done
@@ -41,66 +29,16 @@ done
 cd "$(dirname "$0")/.."
 
 if ! [ -d "./tests/test_build_components/build" ]; then
-  mkdir -p ./tests/test_build_components/build
+  mkdir ./tests/test_build_components/build
 fi
 
-# Export build cache directory if specified
-if [ -n "$build_cache_dir" ]; then
-  export PLATFORMIO_BUILD_CACHE_DIR="$build_cache_dir"
-  mkdir -p "$build_cache_dir"
-  echo "Using build cache directory: $build_cache_dir"
-fi
-
-# Track PIDs for parallel execution
-pids=()
-failed_builds=()
-build_count=0
-total_builds=0
-
-# Function to wait for jobs and handle failures
-wait_for_jobs() {
-  local max_jobs=$1
-  while [ ${#pids[@]} -ge $max_jobs ]; do
-    for i in "${!pids[@]}"; do
-      if ! kill -0 "${pids[$i]}" 2>/dev/null; then
-        wait "${pids[$i]}"
-        exit_code=$?
-        if [ $exit_code -ne 0 ]; then
-          failed_builds+=("${build_info[$i]}")
-          if [ "$fail_fast" = true ]; then
-            echo "Build failed, exiting due to fail-fast mode"
-            # Kill remaining jobs
-            for pid in "${pids[@]}"; do
-              kill -TERM "$pid" 2>/dev/null || true
-            done
-            exit 1
-          fi
-        fi
-        unset pids[$i]
-        unset build_info[$i]
-        # Reindex arrays
-        pids=("${pids[@]}")
-        build_info=("${build_info[@]}")
-        break
-      fi
-    done
-    sleep 0.1
-  done
-}
-
 start_esphome() {
   if [ -n "$requested_target_platform" ] && [ "$requested_target_platform" != "$target_platform_with_version" ]; then
     echo "Skipping $target_platform_with_version"
     return
   fi
-
-  # Apply platform filter if specified
-  if [ -n "$platform_filter" ] && [[ ! "$target_platform_with_version" =~ ^$platform_filter ]]; then
-    echo "Skipping $target_platform_with_version (filtered)"
-    return
-  fi
-
   # create dynamic yaml file in `build` folder.
+  # `./tests/test_build_components/build/[target_component].[test_name].[target_platform_with_version].yaml`
   component_test_file="./tests/test_build_components/build/$target_component.$test_name.$target_platform_with_version.yaml"
 
   cp $target_platform_file $component_test_file
@@ -111,79 +49,17 @@ start_esphome() {
     sed -i "s!\$component_test_file!../../.$f!g" $component_test_file
   fi
 
-  # Start esphome process in background
-  build_count=$((build_count + 1))
-  echo "> [$build_count/$total_builds] [$target_component] [$test_name] [$target_platform_with_version]"
-
-  (
-    # Add compile process limit for ESPHome internal parallelization
-    export ESPHOME_COMPILE_PROCESS_LIMIT=2
-    
-    # For compilation, add a small random delay to reduce thundering herd effect
-    # This helps stagger the package installation requests
-    if [ "$esphome_command" = "compile" ]; then
-      sleep $((RANDOM % 5))
-    fi
-    
-    python3 -m esphome -s component_name $target_component -s component_dir ../../components/$target_component -s test_name $test_name -s target_platform $target_platform $esphome_command $component_test_file
-  ) &
-
-  local pid=$!
-  pids+=($pid)
-  build_info+=("$target_component/$test_name/$target_platform_with_version")
-
-  # Wait if we've reached the job limit
-  wait_for_jobs $num_jobs
+  # Start esphome process
+  echo "> [$target_component] [$test_name] [$target_platform_with_version]"
+  set -x
+  # TODO: Validate escape of Command line substitution value
+  python3 -m esphome -s component_name $target_component -s component_dir ../../components/$target_component -s test_name $test_name -s target_platform $target_platform $esphome_command $component_test_file
+  { set +x; } 2>/dev/null
 }
 
-# First pass: count total builds
-echo "Calculating total number of builds..."
-for f in ./tests/components/$target_component/*.*.yaml; do
-  [ -f "$f" ] || continue
-  IFS='/' read -r -a folder_name <<< "$f"
-  IFS='.' read -r -a file_name <<< "${folder_name[4]}"
-  target_platform="${file_name[1]}"
-  file_name_parts=${#file_name[@]}
-
-  if [ "$target_platform" = "all" ] || [ $file_name_parts = 2 ]; then
-    for target_platform_file in ./tests/test_build_components/build_components_base.*.yaml; do
-      IFS='/' read -r -a folder_name <<< "$target_platform_file"
-      IFS='.' read -r -a file_name <<< "${folder_name[3]}"
-      target_platform="${file_name[1]}"
-      target_platform_with_version=${target_platform_file:52}
-      target_platform_with_version=${target_platform_with_version%.*}
-
-      if [ -n "$platform_filter" ] && [[ ! "$target_platform_with_version" =~ ^$platform_filter ]]; then
-        continue
-      fi
-      if [ -n "$requested_target_platform" ] && [ "$requested_target_platform" != "$target_platform_with_version" ]; then
-        continue
-      fi
-      total_builds=$((total_builds + 1))
-    done
-  else
-    target_platform_file="./tests/test_build_components/build_components_base.$target_platform.yaml"
-    if [ -f "$target_platform_file" ]; then
-      for target_platform_file in ./tests/test_build_components/build_components_base.$target_platform*.yaml; do
-        target_platform_with_version=${target_platform_file:52}
-        target_platform_with_version=${target_platform_with_version%.*}
-
-        if [ -n "$platform_filter" ] && [[ ! "$target_platform_with_version" =~ ^$platform_filter ]]; then
-          continue
-        fi
-        if [ -n "$requested_target_platform" ] && [ "$requested_target_platform" != "$target_platform_with_version" ]; then
-          continue
-        fi
-        total_builds=$((total_builds + 1))
-      done
-    fi
-  fi
-done
-
-echo "Total builds to execute: $total_builds with $num_jobs parallel jobs"
-echo
-
-# Second pass: execute builds
+# Find all test yaml files.
+# - `./tests/components/[target_component]/[test_name].[target_platform].yaml`
+# - `./tests/components/[target_component]/[test_name].all.yaml`
 for f in ./tests/components/$target_component/*.*.yaml; do
   [ -f "$f" ] || continue
   IFS='/' read -r -a folder_name <<< "$f"
@@ -196,21 +72,22 @@ for f in ./tests/components/$target_component/*.*.yaml; do
 
   if [ "$target_platform" = "all" ] || [ $file_name_parts = 2 ]; then
     # Test has *not* defined a specific target platform. Need to run tests for all possible target platforms.
-
+    
     for target_platform_file in ./tests/test_build_components/build_components_base.*.yaml; do
       IFS='/' read -r -a folder_name <<< "$target_platform_file"
       IFS='.' read -r -a file_name <<< "${folder_name[3]}"
       target_platform="${file_name[1]}"
-      target_platform_with_version=${target_platform_file:52}
-      target_platform_with_version=${target_platform_with_version%.*}
 
       start_esphome
     done
 
   else
     # Test has defined a specific target platform.
-
+  
     # Validate we have a base test yaml for selected platform.
+    # The target_platform is sourced from the following location.
+    # 1. `./tests/test_build_components/build_components_base.[target_platform].yaml`
+    # 2. `./tests/test_build_components/build_components_base.[target_platform]-ard.yaml`
     target_platform_file="./tests/test_build_components/build_components_base.$target_platform.yaml"
     if ! [ -f "$target_platform_file" ]; then
       echo "No base test file [./tests/test_build_components/build_components_base.$target_platform.yaml] for component test [$f] found."
@@ -227,23 +104,3 @@ for f in ./tests/components/$target_component/*.*.yaml; do
     done
   fi
 done
-
-# Wait for all remaining jobs
-wait_for_jobs 1
-
-echo
-echo "============================================"
-echo "Build Summary:"
-echo "Total builds: $total_builds"
-echo "Failed builds: ${#failed_builds[@]}"
-
-if [ ${#failed_builds[@]} -gt 0 ]; then
-  echo
-  echo "Failed builds:"
-  for build in "${failed_builds[@]}"; do
-    echo "  - $build"
-  done
-  exit 1
-else
-  echo "All builds completed successfully!"
-fi