From e7a3cccb4d76e44e9c0bd25248feff1f5bb36717 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 7 Dec 2025 13:30:06 -0600 Subject: [PATCH] [text_sensor] Reduce filter memory usage using const char* (#12334) --- esphome/components/text_sensor/filter.cpp | 25 ++- esphome/components/text_sensor/filter.h | 12 +- .../fixtures/text_sensor_raw_state.yaml | 127 ++++++++++++ .../integration/test_text_sensor_raw_state.py | 190 ++++++++++++++++-- 4 files changed, 326 insertions(+), 28 deletions(-) diff --git a/esphome/components/text_sensor/filter.cpp b/esphome/components/text_sensor/filter.cpp index 40a37febe..4cace372a 100644 --- a/esphome/components/text_sensor/filter.cpp +++ b/esphome/components/text_sensor/filter.cpp @@ -56,10 +56,16 @@ optional ToLowerFilter::new_value(std::string value) { } // Append -optional AppendFilter::new_value(std::string value) { return value + this->suffix_; } +optional AppendFilter::new_value(std::string value) { + value.append(this->suffix_); + return value; +} // Prepend -optional PrependFilter::new_value(std::string value) { return this->prefix_ + value; } +optional PrependFilter::new_value(std::string value) { + value.insert(0, this->prefix_); + return value; +} // Substitute SubstituteFilter::SubstituteFilter(const std::initializer_list &substitutions) @@ -67,12 +73,15 @@ SubstituteFilter::SubstituteFilter(const std::initializer_list &su optional SubstituteFilter::new_value(std::string value) { for (const auto &sub : this->substitutions_) { + // Compute lengths once per substitution (strlen is fast, called infrequently) + const size_t from_len = strlen(sub.from); + const size_t to_len = strlen(sub.to); std::size_t pos = 0; - while ((pos = value.find(sub.from, pos)) != std::string::npos) { - value.replace(pos, sub.from.size(), sub.to); + while ((pos = value.find(sub.from, pos, from_len)) != std::string::npos) { + value.replace(pos, from_len, sub.to, to_len); // Advance past the replacement to avoid infinite loop when // the replacement contains the search pattern (e.g., f -> foo) - pos += sub.to.size(); + pos += to_len; } } return value; @@ -83,8 +92,10 @@ MapFilter::MapFilter(const std::initializer_list &mappings) : mapp optional MapFilter::new_value(std::string value) { for (const auto &mapping : this->mappings_) { - if (mapping.from == value) - return mapping.to; + if (value == mapping.from) { + value.assign(mapping.to); + return value; + } } return value; // Pass through if no match } diff --git a/esphome/components/text_sensor/filter.h b/esphome/components/text_sensor/filter.h index 85acac5c8..0f66b753b 100644 --- a/esphome/components/text_sensor/filter.h +++ b/esphome/components/text_sensor/filter.h @@ -92,26 +92,26 @@ class ToLowerFilter : public Filter { /// A simple filter that adds a string to the end of another string class AppendFilter : public Filter { public: - AppendFilter(std::string suffix) : suffix_(std::move(suffix)) {} + explicit AppendFilter(const char *suffix) : suffix_(suffix) {} optional new_value(std::string value) override; protected: - std::string suffix_; + const char *suffix_; }; /// A simple filter that adds a string to the start of another string class PrependFilter : public Filter { public: - PrependFilter(std::string prefix) : prefix_(std::move(prefix)) {} + explicit PrependFilter(const char *prefix) : prefix_(prefix) {} optional new_value(std::string value) override; protected: - std::string prefix_; + const char *prefix_; }; struct Substitution { - std::string from; - std::string to; + const char *from; + const char *to; }; /// A simple filter that replaces a substring with another substring diff --git a/tests/integration/fixtures/text_sensor_raw_state.yaml b/tests/integration/fixtures/text_sensor_raw_state.yaml index 03aece0a0..54ab2e8dc 100644 --- a/tests/integration/fixtures/text_sensor_raw_state.yaml +++ b/tests/integration/fixtures/text_sensor_raw_state.yaml @@ -20,6 +20,42 @@ text_sensor: filters: - to_upper + # StringRef-based filters (append, prepend, substitute, map) + - platform: template + name: "Append Sensor" + id: append_sensor + filters: + - append: " suffix" + + - platform: template + name: "Prepend Sensor" + id: prepend_sensor + filters: + - prepend: "prefix " + + - platform: template + name: "Substitute Sensor" + id: substitute_sensor + filters: + - substitute: + - foo -> bar + - hello -> world + + - platform: template + name: "Map Sensor" + id: map_sensor + filters: + - map: + - ON -> Active + - OFF -> Inactive + + - platform: template + name: "Chained Sensor" + id: chained_sensor + filters: + - prepend: "[" + - append: "]" + # Button to publish values and log raw_state vs state button: - platform: template @@ -52,3 +88,94 @@ button: args: - id(with_filter_sensor).state.c_str() - id(with_filter_sensor).get_raw_state().c_str() + + - platform: template + name: "Test Append Button" + id: test_append_button + on_press: + - text_sensor.template.publish: + id: append_sensor + state: "test" + - delay: 50ms + - logger.log: + format: "APPEND: state='%s'" + args: + - id(append_sensor).state.c_str() + + - platform: template + name: "Test Prepend Button" + id: test_prepend_button + on_press: + - text_sensor.template.publish: + id: prepend_sensor + state: "test" + - delay: 50ms + - logger.log: + format: "PREPEND: state='%s'" + args: + - id(prepend_sensor).state.c_str() + + - platform: template + name: "Test Substitute Button" + id: test_substitute_button + on_press: + - text_sensor.template.publish: + id: substitute_sensor + state: "foo says hello" + - delay: 50ms + - logger.log: + format: "SUBSTITUTE: state='%s'" + args: + - id(substitute_sensor).state.c_str() + + - platform: template + name: "Test Map ON Button" + id: test_map_on_button + on_press: + - text_sensor.template.publish: + id: map_sensor + state: "ON" + - delay: 50ms + - logger.log: + format: "MAP_ON: state='%s'" + args: + - id(map_sensor).state.c_str() + + - platform: template + name: "Test Map OFF Button" + id: test_map_off_button + on_press: + - text_sensor.template.publish: + id: map_sensor + state: "OFF" + - delay: 50ms + - logger.log: + format: "MAP_OFF: state='%s'" + args: + - id(map_sensor).state.c_str() + + - platform: template + name: "Test Map Unknown Button" + id: test_map_unknown_button + on_press: + - text_sensor.template.publish: + id: map_sensor + state: "UNKNOWN" + - delay: 50ms + - logger.log: + format: "MAP_UNKNOWN: state='%s'" + args: + - id(map_sensor).state.c_str() + + - platform: template + name: "Test Chained Button" + id: test_chained_button + on_press: + - text_sensor.template.publish: + id: chained_sensor + state: "value" + - delay: 50ms + - logger.log: + format: "CHAINED: state='%s'" + args: + - id(chained_sensor).state.c_str() diff --git a/tests/integration/test_text_sensor_raw_state.py b/tests/integration/test_text_sensor_raw_state.py index a53ec8c96..482ebbe9c 100644 --- a/tests/integration/test_text_sensor_raw_state.py +++ b/tests/integration/test_text_sensor_raw_state.py @@ -1,8 +1,10 @@ -"""Integration test for TextSensor get_raw_state() functionality. +"""Integration test for TextSensor get_raw_state() and StringRef-based filters. -This tests the optimization in PR #12205 where raw_state is only stored -when filters are configured. When no filters exist, get_raw_state() should -return state directly. +This tests: +1. The optimization in PR #12205 where raw_state is only stored when filters + are configured. When no filters exist, get_raw_state() should return state. +2. StringRef-based filters (append, prepend, substitute, map) which store + static string data in flash instead of heap-allocating std::string. """ from __future__ import annotations @@ -21,16 +23,25 @@ async def test_text_sensor_raw_state( run_compiled: RunCompiledFunction, api_client_connected: APIClientConnectedFactory, ) -> None: - """Test that get_raw_state() works correctly with and without filters. + """Test text sensor filters and raw_state behavior. - Without filters: get_raw_state() should return the same value as state - With filters: get_raw_state() should return the original (unfiltered) value + Tests: + 1. get_raw_state() without filters returns same as state + 2. get_raw_state() with filters returns original (unfiltered) value + 3. StringRef-based filters: append, prepend, substitute, map, chained """ loop = asyncio.get_running_loop() # Futures to track log messages no_filter_future: asyncio.Future[tuple[str, str]] = loop.create_future() with_filter_future: asyncio.Future[tuple[str, str]] = loop.create_future() + append_future: asyncio.Future[str] = loop.create_future() + prepend_future: asyncio.Future[str] = loop.create_future() + substitute_future: asyncio.Future[str] = loop.create_future() + map_on_future: asyncio.Future[str] = loop.create_future() + map_off_future: asyncio.Future[str] = loop.create_future() + map_unknown_future: asyncio.Future[str] = loop.create_future() + chained_future: asyncio.Future[str] = loop.create_future() # Patterns to match log output # NO_FILTER: state='hello world' raw_state='hello world' @@ -39,18 +50,47 @@ async def test_text_sensor_raw_state( with_filter_pattern = re.compile( r"WITH_FILTER: state='([^']*)' raw_state='([^']*)'" ) + # StringRef-based filter patterns + append_pattern = re.compile(r"APPEND: state='([^']*)'") + prepend_pattern = re.compile(r"PREPEND: state='([^']*)'") + substitute_pattern = re.compile(r"SUBSTITUTE: state='([^']*)'") + map_on_pattern = re.compile(r"MAP_ON: state='([^']*)'") + map_off_pattern = re.compile(r"MAP_OFF: state='([^']*)'") + map_unknown_pattern = re.compile(r"MAP_UNKNOWN: state='([^']*)'") + chained_pattern = re.compile(r"CHAINED: state='([^']*)'") def check_output(line: str) -> None: """Check log output for expected messages.""" - if not no_filter_future.done(): - match = no_filter_pattern.search(line) - if match: - no_filter_future.set_result((match.group(1), match.group(2))) + if not no_filter_future.done() and (match := no_filter_pattern.search(line)): + no_filter_future.set_result((match.group(1), match.group(2))) - if not with_filter_future.done(): - match = with_filter_pattern.search(line) - if match: - with_filter_future.set_result((match.group(1), match.group(2))) + if not with_filter_future.done() and ( + match := with_filter_pattern.search(line) + ): + with_filter_future.set_result((match.group(1), match.group(2))) + + if not append_future.done() and (match := append_pattern.search(line)): + append_future.set_result(match.group(1)) + + if not prepend_future.done() and (match := prepend_pattern.search(line)): + prepend_future.set_result(match.group(1)) + + if not substitute_future.done() and (match := substitute_pattern.search(line)): + substitute_future.set_result(match.group(1)) + + if not map_on_future.done() and (match := map_on_pattern.search(line)): + map_on_future.set_result(match.group(1)) + + if not map_off_future.done() and (match := map_off_pattern.search(line)): + map_off_future.set_result(match.group(1)) + + if not map_unknown_future.done() and ( + match := map_unknown_pattern.search(line) + ): + map_unknown_future.set_result(match.group(1)) + + if not chained_future.done() and (match := chained_pattern.search(line)): + chained_future.set_result(match.group(1)) async with ( run_compiled(yaml_config, line_callback=check_output), @@ -112,3 +152,123 @@ async def test_text_sensor_raw_state( f"With filters, state and raw_state should differ. " f"state='{state}', raw_state='{raw_state}'" ) + + # Test 3: Append filter (StringRef-based) + # "test" + " suffix" = "test suffix" + append_button = next( + (e for e in entities if "test_append_button" in e.object_id.lower()), + None, + ) + assert append_button is not None, "Test Append Button not found" + client.button_command(append_button.key) + + try: + state = await asyncio.wait_for(append_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for APPEND log message") + + assert state == "test suffix", ( + f"Append failed: expected 'test suffix', got '{state}'" + ) + + # Test 4: Prepend filter (StringRef-based) + # "prefix " + "test" = "prefix test" + prepend_button = next( + (e for e in entities if "test_prepend_button" in e.object_id.lower()), + None, + ) + assert prepend_button is not None, "Test Prepend Button not found" + client.button_command(prepend_button.key) + + try: + state = await asyncio.wait_for(prepend_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for PREPEND log message") + + assert state == "prefix test", ( + f"Prepend failed: expected 'prefix test', got '{state}'" + ) + + # Test 5: Substitute filter (StringRef-based) + # "foo says hello" with foo->bar, hello->world = "bar says world" + substitute_button = next( + (e for e in entities if "test_substitute_button" in e.object_id.lower()), + None, + ) + assert substitute_button is not None, "Test Substitute Button not found" + client.button_command(substitute_button.key) + + try: + state = await asyncio.wait_for(substitute_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for SUBSTITUTE log message") + + assert state == "bar says world", ( + f"Substitute failed: expected 'bar says world', got '{state}'" + ) + + # Test 6: Map filter - "ON" -> "Active" + map_on_button = next( + (e for e in entities if "test_map_on_button" in e.object_id.lower()), + None, + ) + assert map_on_button is not None, "Test Map ON Button not found" + client.button_command(map_on_button.key) + + try: + state = await asyncio.wait_for(map_on_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for MAP_ON log message") + + assert state == "Active", f"Map ON failed: expected 'Active', got '{state}'" + + # Test 7: Map filter - "OFF" -> "Inactive" + map_off_button = next( + (e for e in entities if "test_map_off_button" in e.object_id.lower()), + None, + ) + assert map_off_button is not None, "Test Map OFF Button not found" + client.button_command(map_off_button.key) + + try: + state = await asyncio.wait_for(map_off_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for MAP_OFF log message") + + assert state == "Inactive", ( + f"Map OFF failed: expected 'Inactive', got '{state}'" + ) + + # Test 8: Map filter - passthrough for unknown values + # "UNKNOWN" -> "UNKNOWN" (no match, passes through unchanged) + map_unknown_button = next( + (e for e in entities if "test_map_unknown_button" in e.object_id.lower()), + None, + ) + assert map_unknown_button is not None, "Test Map Unknown Button not found" + client.button_command(map_unknown_button.key) + + try: + state = await asyncio.wait_for(map_unknown_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for MAP_UNKNOWN log message") + + assert state == "UNKNOWN", ( + f"Map passthrough failed: expected 'UNKNOWN', got '{state}'" + ) + + # Test 9: Chained filters (prepend "[" + append "]") + # "[" + "value" + "]" = "[value]" + chained_button = next( + (e for e in entities if "test_chained_button" in e.object_id.lower()), + None, + ) + assert chained_button is not None, "Test Chained Button not found" + client.button_command(chained_button.key) + + try: + state = await asyncio.wait_for(chained_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for CHAINED log message") + + assert state == "[value]", f"Chained failed: expected '[value]', got '{state}'"