diff --git a/components/esp_system/ld/esp32c3/sections.ld.in b/components/esp_system/ld/esp32c3/sections.ld.in
index 1077f6966d..195f55548b 100644
--- a/components/esp_system/ld/esp32c3/sections.ld.in
+++ b/components/esp_system/ld/esp32c3/sections.ld.in
@@ -19,14 +19,18 @@ SECTIONS
   {
     ALIGNED_SYMBOL(4, _rtc_fast_start)
 
+    HIDDEN(_rtc_code_start = .);
+
     mapping[rtc_text]
 
     *rtc_wake_stub*.*(.text .text.*)
     *(.rtc_text_end_test)
 
-    /* Padding for possible CPU prefetch + alignment for PMS split lines */
-    . += _esp_memprot_prefetch_pad_size;
-    . = ALIGN(_esp_memprot_align_size);
+    HIDDEN(_rtc_code_end = .);
+
+    /* Padding for possible CPU prefetch + 4B alignment for PMS split lines. */
+    . = ((_rtc_code_end - _rtc_code_start) == 0) ?
+        ALIGN(0) : _esp_memprot_prefetch_pad_size + ALIGN(4);
 
     _rtc_text_end = ABSOLUTE(.);
   } > rtc_iram_seg
diff --git a/components/esp_system/ld/esp32s2/sections.ld.in b/components/esp_system/ld/esp32s2/sections.ld.in
index 1b6e992da6..67099c272f 100644
--- a/components/esp_system/ld/esp32s2/sections.ld.in
+++ b/components/esp_system/ld/esp32s2/sections.ld.in
@@ -26,13 +26,16 @@ SECTIONS
   {
     ALIGNED_SYMBOL(4, _rtc_text_start)
 
+    HIDDEN(_rtc_code_start = .);
+
     mapping[rtc_text]
 
     *rtc_wake_stub*.*(.literal .text .literal.* .text.*)
 
-    /* Padding for possible CPU prefetch + alignment for PMS split lines */
-    . += _esp_memprot_prefetch_pad_size;
-    . = ALIGN(_esp_memprot_align_size);
+    HIDDEN(_rtc_code_end = .);
+
+    /* Possibly align + 16B for CPU dummy speculative instruction fetch. */
+    . = ((_rtc_code_end - _rtc_code_start) == 0) ? ALIGN(0) : ALIGN(4) + 16;
 
     _rtc_text_end = ABSOLUTE(.);
   } > rtc_iram_seg
diff --git a/components/esp_system/ld/esp32s3/sections.ld.in b/components/esp_system/ld/esp32s3/sections.ld.in
index 527f8b9350..8b9c9a006e 100644
--- a/components/esp_system/ld/esp32s3/sections.ld.in
+++ b/components/esp_system/ld/esp32s3/sections.ld.in
@@ -20,6 +20,8 @@ SECTIONS
     ALIGNED_SYMBOL(4, _rtc_fast_start)
     ALIGNED_SYMBOL(4, _rtc_text_start)
 
+    HIDDEN(_rtc_code_start = .);
+
     *(.rtc.entry.literal .rtc.entry.text)
 
     mapping[rtc_text]
@@ -27,9 +29,11 @@ SECTIONS
     *rtc_wake_stub*.*(.literal .text .literal.* .text.*)
     *(.rtc_text_end_test)
 
-    /* Padding for possible CPU prefetch + alignment for PMS split lines */
-    . += _esp_memprot_prefetch_pad_size;
-    . = ALIGN(_esp_memprot_align_size);
+    HIDDEN(_rtc_code_end = .);
+
+    /* Padding for possible CPU prefetch + 4B alignment for PMS split lines. */
+    . = ((_rtc_code_end - _rtc_code_start) == 0) ?
+        ALIGN(0) : _esp_memprot_prefetch_pad_size + ALIGN(4);
 
     _rtc_text_end = ABSOLUTE(.);
   } > rtc_iram_seg