diff --git a/panel/pipeline.py b/panel/pipeline.py
index 5f7668af0d..ee595a3c36 100644
--- a/panel/pipeline.py
+++ b/panel/pipeline.py
@@ -352,21 +352,24 @@ def _prev_stage(self):
 
     def _update_button(self):
         stage, kwargs = self._stages[self._stage]
-        options = list(self._graph.get(self._stage, []))
+        next_options = list(self._graph.get(self._stage, []))
         next_param = kwargs.get('next_parameter', self.next_parameter)
-        option = getattr(self._state, next_param) if next_param and next_param in stage.param else None
-        if option is None:
-            option = options[0] if options else None
-        self.next_selector.options = options
-        self.next_selector.value = option
-        self.next_selector.disabled = not bool(options)
-        previous = []
+        next_option = getattr(self._state, next_param) if next_param and next_param in stage.param else None
+        if next_option is None:
+            next_option = next_options[0] if next_options else None
+        self.next_selector.options = next_options
+        self.next_selector.value = next_option
+        self.next_selector.disabled = not bool(next_options)
+        previous_options = []
+        previous_option = None
         for src, tgts in self._graph.items():
             if self._stage in tgts:
-                previous.append(src)
-        self.prev_selector.options = previous
-        self.prev_selector.value = self._route[-1] if previous else None
-        self.prev_selector.disabled = not bool(previous)
+                previous_options.append(src)
+                if src in self._route:
+                    previous_option = src
+        self.prev_selector.options = previous_options
+        self.prev_selector.value = previous_option
+        self.prev_selector.disabled = not bool(previous_options)
 
         # Disable previous button
         if self._prev_stage is None:
diff --git a/panel/tests/test_pipeline.py b/panel/tests/test_pipeline.py
index 7c957838b0..bb5cb182e4 100644
--- a/panel/tests/test_pipeline.py
+++ b/panel/tests/test_pipeline.py
@@ -456,3 +456,31 @@ def test_pipeline_ready_parameter_preserved_on_back_navigation(stage1, auto_adva
         assert not pipeline.next_button.disabled
         pipeline._next()
     assert pipeline._stage == 'Stage 2'
+
+
+def test_previous_sets_lower_stage():
+    """
+    Non regression test for https://github.com/holoviz/panel/issues/5687
+
+    When navigating back from stage N to stage N-1, and then hitting previous once
+    again, make sure current state is stage N-2 and not stage N.
+    """
+    pipeline = Pipeline()
+
+    pipeline.add_stage("Stage 1", DummyStage())
+    pipeline.add_stage("Stage 2", DummyStage())
+    pipeline.add_stage("Stage 3", DummyStage())
+
+    pipeline._next()
+    pipeline._next()
+
+    assert pipeline._stage == "Stage 3"
+    assert pipeline._route == ["Stage 1", "Stage 2", "Stage 3"]
+
+    pipeline._previous()
+    assert pipeline._stage == "Stage 2"
+    assert pipeline._route == ["Stage 1", "Stage 2"]
+    pipeline._previous()
+
+    assert pipeline._stage == "Stage 1"
+    assert pipeline._route == ["Stage 1"]