data_store: orphaned n=0 tasks not present in store

[oliver-sanders]

Tasks orphaned (i.e, removed from the workflow config) by either reload or restart do not appear in the data store.

These tasks still exist and still count towards the runahead limit, but are not visible in the GUI causing a hard-to-debug runahead-stall.

Reproducible Example:

This script creates a workflow called test and restores it from a database snapshot in which these tasks have succeeded:

• 1/foo
• 2/foo
• 3/fop

The following tasks are in the pool (waiting):

• 1/bar (orphaned)
• 2/bar (orphaned)
• 3/bar(orphaned)
• 4/bar (orphaned, runahead)
• 4/foo (runahead)

When the workflow starts up, the orphaned tasks do not appear in the n-window.

#!/usr/bin/env bash

set -euxo pipefail

RDIR="$HOME/cylc-run"
WFLOW=test
WRDIR="$RDIR/$WFLOW"

cylc stop --now --now "$WFLOW" || true
cylc clean -y "$WFLOW" || true
mkdir "$WRDIR"
cat > "$WRDIR/flow.cylc" <<__HERE__
[scheduling]
  cycling mode = integer
  initial cycle point = 1
  runahead limit = P2
  [[graph]]
    P1 = foo

[runtime]
  [[foo]]
  [[bar]]
__HERE__

mkdir "$WRDIR/.service"
sqlite3 "$WRDIR/.service/db" <<__HERE__
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE absolute_outputs(cycle TEXT, name TEXT, output TEXT);
CREATE TABLE broadcast_events(time TEXT, change TEXT, point TEXT, namespace TEXT, key TEXT, value TEXT);
CREATE TABLE broadcast_states(point TEXT, namespace TEXT, key TEXT, value TEXT, PRIMARY KEY(point, namespace, key));
CREATE TABLE inheritance(namespace TEXT, inheritance TEXT, PRIMARY KEY(namespace));
INSERT INTO inheritance VALUES('root','["root"]');
INSERT INTO inheritance VALUES('foo','["foo", "root"]');
INSERT INTO inheritance VALUES('bar','["bar", "root"]');
CREATE TABLE task_action_timers(cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key));
CREATE TABLE task_events(name TEXT, cycle TEXT, time TEXT, submit_num INTEGER, event TEXT, message TEXT);
INSERT INTO task_events VALUES('foo','1','2026-01-27T13:29:33Z',0,'succeeded','');
INSERT INTO task_events VALUES('foo','2','2026-01-27T13:29:37Z',0,'succeeded','');
INSERT INTO task_events VALUES('foo','3','2026-01-27T13:29:42Z',0,'succeeded','');
CREATE TABLE task_jobs(cycle TEXT, name TEXT, submit_num INTEGER, flow_nums TEXT, is_manual_submit INTEGER, try_num INTEGER, time_submit TEXT, time_submit_exit TEXT, submit_status INTEGER, time_run TEXT, time_run_exit TEXT, run_signal TEXT, run_status INTEGER, platform_name TEXT, job_runner_name TEXT, job_id TEXT, PRIMARY KEY(cycle, name, submit_num));
CREATE TABLE task_late_flags(cycle TEXT, name TEXT, value INTEGER, PRIMARY KEY(cycle, name));
CREATE TABLE task_outputs(cycle TEXT, name TEXT, flow_nums TEXT, outputs TEXT, PRIMARY KEY(cycle, name, flow_nums));
INSERT INTO task_outputs VALUES('1','bar','[1]','{}');
INSERT INTO task_outputs VALUES('2','bar','[1]','{}');
INSERT INTO task_outputs VALUES('3','bar','[1]','{}');
INSERT INTO task_outputs VALUES('4','bar','[1]','{}');
INSERT INTO task_outputs VALUES('1','foo','[1]','{"submitted": "(manually completed)", "started": "(manually completed)", "succeeded": "(manually completed)"}');
INSERT INTO task_outputs VALUES('2','foo','[1]','{"submitted": "(manually completed)", "started": "(manually completed)", "succeeded": "(manually completed)"}');
INSERT INTO task_outputs VALUES('3','foo','[1]','{"submitted": "(manually completed)", "started": "(manually completed)", "succeeded": "(manually completed)"}');
INSERT INTO task_outputs VALUES('4','foo','[1]','{}');
CREATE TABLE task_pool(cycle TEXT, name TEXT, flow_nums TEXT, status TEXT, is_held INTEGER, PRIMARY KEY(cycle, name, flow_nums));
INSERT INTO task_pool VALUES('1','bar','[1]','waiting',0);
INSERT INTO task_pool VALUES('2','bar','[1]','waiting',0);
INSERT INTO task_pool VALUES('3','bar','[1]','waiting',0);
INSERT INTO task_pool VALUES('4','bar','[1]','waiting',0);
INSERT INTO task_pool VALUES('4','foo','[1]','waiting',0);
CREATE TABLE task_prerequisites(cycle TEXT, name TEXT, flow_nums TEXT, prereq_name TEXT, prereq_cycle TEXT, prereq_output TEXT, satisfied TEXT, PRIMARY KEY(cycle, name, flow_nums, prereq_name, prereq_cycle, prereq_output));
CREATE TABLE task_states(name TEXT, cycle TEXT, flow_nums TEXT, time_created TEXT, time_updated TEXT, submit_num INTEGER, status TEXT, flow_wait INTEGER, is_manual_submit INTEGER, PRIMARY KEY(name, cycle, flow_nums));
INSERT INTO task_states VALUES('bar','1','[1]','2026-01-27T13:29:21Z','2026-01-27T13:29:21Z',0,'waiting',0,0);
INSERT INTO task_states VALUES('bar','2','[1]','2026-01-27T13:29:21Z','2026-01-27T13:29:21Z',0,'waiting',0,0);
INSERT INTO task_states VALUES('bar','3','[1]','2026-01-27T13:29:21Z','2026-01-27T13:29:21Z',0,'waiting',0,0);
INSERT INTO task_states VALUES('bar','4','[1]','2026-01-27T13:29:21Z','2026-01-27T13:29:21Z',0,'waiting',0,0);
INSERT INTO task_states VALUES('foo','1','[1]','2026-01-27T13:29:21Z','2026-01-27T13:29:33Z',0,'succeeded',0,0);
INSERT INTO task_states VALUES('foo','2','[1]','2026-01-27T13:29:21Z','2026-01-27T13:29:37Z',0,'succeeded',0,0);
INSERT INTO task_states VALUES('foo','3','[1]','2026-01-27T13:29:21Z','2026-01-27T13:29:42Z',0,'succeeded',0,0);
INSERT INTO task_states VALUES('foo','4','[1]','2026-01-27T13:29:21Z','2026-01-27T13:29:21Z',0,'waiting',0,0);
CREATE TABLE task_timeout_timers(cycle TEXT, name TEXT, timeout REAL, PRIMARY KEY(cycle, name));
CREATE TABLE tasks_to_hold(name TEXT, cycle TEXT);
CREATE TABLE workflow_flows(flow_num INTEGER, start_time TEXT, description TEXT, PRIMARY KEY(flow_num));
INSERT INTO workflow_flows VALUES(1,'2026-01-27T13:29:21','original flow from 1');
CREATE TABLE workflow_params(key TEXT, value TEXT, PRIMARY KEY(key));
INSERT INTO workflow_params VALUES('uuid_str','b3ad8b6b-e487-482d-a123-637a3aff83ba');
INSERT INTO workflow_params VALUES('cylc_version','8.6.3.dev');
INSERT INTO workflow_params VALUES('UTC_mode','0');
INSERT INTO workflow_params VALUES('n_restart','0');
INSERT INTO workflow_params VALUES('cycle_point_format',NULL);
INSERT INTO workflow_params VALUES('is_paused','1');
INSERT INTO workflow_params VALUES('stop_clock_time',NULL);
INSERT INTO workflow_params VALUES('stop_task',NULL);
INSERT INTO workflow_params VALUES('icp','1');
INSERT INTO workflow_params VALUES('fcp',NULL);
INSERT INTO workflow_params VALUES('startcp',NULL);
INSERT INTO workflow_params VALUES('stopcp',NULL);
INSERT INTO workflow_params VALUES('cycle_point_tz','Z');
INSERT INTO workflow_params VALUES('run_mode','live');
CREATE TABLE workflow_template_vars(key TEXT, value TEXT, PRIMARY KEY(key));
CREATE TABLE xtriggers(signature TEXT, results TEXT, PRIMARY KEY(signature));
COMMIT;
__HERE__

cylc play "$WFLOW" --pause

If you take this script and restore "bar" to the graph:

diff --git a/mess b/mess
index f95c186..716a4e6 100755
--- a/mess
+++ b/mess
@@ -15,7 +15,7 @@ cat > "$WRDIR/flow.cylc" <<__HERE__
   initial cycle point = 1
   runahead limit = P2
   [[graph]]
-    P1 = foo
+    P1 = foo & bar
 
 [runtime]
   [[foo]]

Then the tasks are visible in the GUI as intended.

[oliver-sanders]

I think I may have identified the issue whilst working on #7197