diff --git a/homeassistant/components/media_player/intent.py b/homeassistant/components/media_player/intent.py
index 0f36c65023da45..da8da6c2c58682 100644
--- a/homeassistant/components/media_player/intent.py
+++ b/homeassistant/components/media_player/intent.py
@@ -1,5 +1,9 @@
 """Intents for the media_player integration."""
 
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+import time
+
 import voluptuous as vol
 
 from homeassistant.const import (
@@ -8,7 +12,7 @@
     SERVICE_MEDIA_PLAY,
     SERVICE_VOLUME_SET,
 )
-from homeassistant.core import HomeAssistant
+from homeassistant.core import Context, HomeAssistant, State
 from homeassistant.helpers import intent
 
 from . import ATTR_MEDIA_VOLUME_LEVEL, DOMAIN
@@ -19,13 +23,39 @@
 INTENT_MEDIA_NEXT = "HassMediaNext"
 INTENT_SET_VOLUME = "HassSetVolume"
 
-DATA_LAST_PAUSED = f"{DOMAIN}.last_paused"
+
+@dataclass
+class LastPaused:
+    """Information about last media players that were paused by voice."""
+
+    timestamp: float | None = None
+    context: Context | None = None
+    entity_ids: set[str] = field(default_factory=set)
+
+    def clear(self) -> None:
+        """Clear timestamp and entities."""
+        self.timestamp = None
+        self.context = None
+        self.entity_ids.clear()
+
+    def update(self, context: Context | None, entity_ids: Iterable[str]) -> None:
+        """Update last paused group."""
+        self.context = context
+        self.entity_ids = set(entity_ids)
+        if self.entity_ids:
+            self.timestamp = time.time()
+
+    def __bool__(self) -> bool:
+        """Return True if timestamp is set."""
+        return self.timestamp is not None
 
 
 async def async_setup_intents(hass: HomeAssistant) -> None:
     """Set up the media_player intents."""
-    intent.async_register(hass, MediaUnpauseHandler())
-    intent.async_register(hass, MediaPauseHandler())
+    last_paused = LastPaused()
+
+    intent.async_register(hass, MediaUnpauseHandler(last_paused))
+    intent.async_register(hass, MediaPauseHandler(last_paused))
     intent.async_register(
         hass,
         intent.ServiceIntentHandler(
@@ -58,7 +88,7 @@ async def async_setup_intents(hass: HomeAssistant) -> None:
 class MediaPauseHandler(intent.ServiceIntentHandler):
     """Handler for pause intent. Records last paused media players."""
 
-    def __init__(self) -> None:
+    def __init__(self, last_paused: LastPaused) -> None:
         """Initialize handler."""
         super().__init__(
             INTENT_MEDIA_PAUSE,
@@ -68,6 +98,7 @@ def __init__(self) -> None:
             required_features=MediaPlayerEntityFeature.PAUSE,
             required_states={MediaPlayerState.PLAYING},
         )
+        self.last_paused = last_paused
 
     async def async_handle_states(
         self,
@@ -77,11 +108,11 @@ async def async_handle_states(
         match_preferences: intent.MatchTargetsPreferences | None = None,
     ) -> intent.IntentResponse:
         """Record last paused media players."""
-        hass = intent_obj.hass
-
         if match_result.is_match:
             # Save entity ids of paused media players
-            hass.data[DATA_LAST_PAUSED] = {s.entity_id for s in match_result.states}
+            self.last_paused.update(
+                intent_obj.context, (s.entity_id for s in match_result.states)
+            )
 
         return await super().async_handle_states(
             intent_obj, match_result, match_constraints
@@ -91,7 +122,7 @@ async def async_handle_states(
 class MediaUnpauseHandler(intent.ServiceIntentHandler):
     """Handler for unpause/resume intent. Uses last paused media players."""
 
-    def __init__(self) -> None:
+    def __init__(self, last_paused: LastPaused) -> None:
         """Initialize handler."""
         super().__init__(
             INTENT_MEDIA_UNPAUSE,
@@ -100,6 +131,7 @@ def __init__(self) -> None:
             required_domains={DOMAIN},
             required_states={MediaPlayerState.PAUSED},
         )
+        self.last_paused = last_paused
 
     async def async_handle_states(
         self,
@@ -109,21 +141,37 @@ async def async_handle_states(
         match_preferences: intent.MatchTargetsPreferences | None = None,
     ) -> intent.IntentResponse:
         """Unpause last paused media players."""
-        hass = intent_obj.hass
-
-        if (
-            match_result.is_match
-            and (not match_constraints.name)
-            and (last_paused := hass.data.get(DATA_LAST_PAUSED))
-        ):
-            # Resume only the previously paused media players if they are in the
-            # targeted set.
-            targeted_ids = {s.entity_id for s in match_result.states}
-            overlapping_ids = targeted_ids.intersection(last_paused)
-            if overlapping_ids:
-                match_result.states = [
-                    s for s in match_result.states if s.entity_id in overlapping_ids
-                ]
+        if match_result.is_match and (not match_constraints.name) and self.last_paused:
+            assert self.last_paused.timestamp is not None
+
+            # Check for a media player that was paused more recently than the
+            # ones by voice.
+            recent_state: State | None = None
+            for state in match_result.states:
+                if (state.last_changed_timestamp <= self.last_paused.timestamp) or (
+                    state.context == self.last_paused.context
+                ):
+                    continue
+
+                if (recent_state is None) or (
+                    state.last_changed_timestamp > recent_state.last_changed_timestamp
+                ):
+                    recent_state = state
+
+            if recent_state is not None:
+                # Resume the more recently paused media player (outside of voice).
+                match_result.states = [recent_state]
+            else:
+                # Resume only the previously paused media players if they are in the
+                # targeted set.
+                targeted_ids = {s.entity_id for s in match_result.states}
+                overlapping_ids = targeted_ids.intersection(self.last_paused.entity_ids)
+                if overlapping_ids:
+                    match_result.states = [
+                        s for s in match_result.states if s.entity_id in overlapping_ids
+                    ]
+
+            self.last_paused.clear()
 
         return await super().async_handle_states(
             intent_obj, match_result, match_constraints
diff --git a/tests/components/media_player/test_intent.py b/tests/components/media_player/test_intent.py
index 8cce7cff44cda9..e73104eeb39fd6 100644
--- a/tests/components/media_player/test_intent.py
+++ b/tests/components/media_player/test_intent.py
@@ -17,7 +17,7 @@
     STATE_PAUSED,
     STATE_PLAYING,
 )
-from homeassistant.core import HomeAssistant
+from homeassistant.core import Context, HomeAssistant
 from homeassistant.helpers import (
     area_registry as ar,
     entity_registry as er,
@@ -515,3 +515,103 @@ async def test_multiple_media_players(
     hass.states.async_set(
         kitchen_smart_speaker.entity_id, STATE_PLAYING, attributes=attributes
     )
+
+
+async def test_manual_pause_unpause(
+    hass: HomeAssistant,
+    area_registry: ar.AreaRegistry,
+    entity_registry: er.EntityRegistry,
+) -> None:
+    """Test unpausing a media player that was manually paused outside of voice."""
+    await media_player_intent.async_setup_intents(hass)
+
+    attributes = {ATTR_SUPPORTED_FEATURES: MediaPlayerEntityFeature.PAUSE}
+
+    # Create two playing devices
+    device_1 = entity_registry.async_get_or_create("media_player", "test", "device-1")
+    device_1 = entity_registry.async_update_entity(device_1.entity_id, name="device 1")
+    hass.states.async_set(device_1.entity_id, STATE_PLAYING, attributes=attributes)
+
+    device_2 = entity_registry.async_get_or_create("media_player", "test", "device-2")
+    device_2 = entity_registry.async_update_entity(device_2.entity_id, name="device 2")
+    hass.states.async_set(device_2.entity_id, STATE_PLAYING, attributes=attributes)
+
+    # Pause both devices by voice
+    context = Context()
+    calls = async_mock_service(hass, DOMAIN, SERVICE_MEDIA_PAUSE)
+    response = await intent.async_handle(
+        hass,
+        "test",
+        media_player_intent.INTENT_MEDIA_PAUSE,
+        context=context,
+    )
+    await hass.async_block_till_done()
+    assert response.response_type == intent.IntentResponseType.ACTION_DONE
+    assert len(calls) == 2
+
+    hass.states.async_set(
+        device_1.entity_id, STATE_PAUSED, attributes=attributes, context=context
+    )
+    hass.states.async_set(
+        device_2.entity_id, STATE_PAUSED, attributes=attributes, context=context
+    )
+
+    # Unpause both devices by voice
+    context = Context()
+    calls = async_mock_service(hass, DOMAIN, SERVICE_MEDIA_PLAY)
+    response = await intent.async_handle(
+        hass,
+        "test",
+        media_player_intent.INTENT_MEDIA_UNPAUSE,
+        context=context,
+    )
+    await hass.async_block_till_done()
+    assert response.response_type == intent.IntentResponseType.ACTION_DONE
+    assert len(calls) == 2
+
+    hass.states.async_set(
+        device_1.entity_id, STATE_PLAYING, attributes=attributes, context=context
+    )
+    hass.states.async_set(
+        device_2.entity_id, STATE_PLAYING, attributes=attributes, context=context
+    )
+
+    # Pause the first device by voice
+    context = Context()
+    calls = async_mock_service(hass, DOMAIN, SERVICE_MEDIA_PAUSE)
+    response = await intent.async_handle(
+        hass,
+        "test",
+        media_player_intent.INTENT_MEDIA_PAUSE,
+        {"name": {"value": "device 1"}},
+        context=context,
+    )
+    await hass.async_block_till_done()
+    assert response.response_type == intent.IntentResponseType.ACTION_DONE
+    assert len(calls) == 1
+    assert calls[0].data == {"entity_id": device_1.entity_id}
+
+    hass.states.async_set(
+        device_1.entity_id, STATE_PAUSED, attributes=attributes, context=context
+    )
+
+    # "Manually" pause the second device (outside of voice)
+    context = Context()
+    hass.states.async_set(
+        device_2.entity_id, STATE_PAUSED, attributes=attributes, context=context
+    )
+
+    # Unpause with no constraints.
+    # Should resume the more recently (manually) paused device.
+    context = Context()
+    calls = async_mock_service(hass, DOMAIN, SERVICE_MEDIA_PLAY)
+    response = await intent.async_handle(
+        hass,
+        "test",
+        media_player_intent.INTENT_MEDIA_UNPAUSE,
+        context=context,
+    )
+    await hass.async_block_till_done()
+    assert response.response_type == intent.IntentResponseType.ACTION_DONE
+    assert len(calls) == 1
+    assert calls[0].data == {"entity_id": device_2.entity_id}