compute: add a lifecycle event log for compute exports

doc/user/content/reference/system-catalog/mz_introspection.md

@@ -485,6 +485,7 @@ The `mz_scheduling_parks_histogram` view describes a histogram of [dataflow] wor
 [query hints]: /sql/select/#query-hints
 
 <!-- RELATION_SPEC_UNDOCUMENTED mz_introspection.mz_compute_hydration_times_per_worker -->
+<!-- RELATION_SPEC_UNDOCUMENTED mz_introspection.mz_compute_lifecycle_events_per_worker -->
 <!-- RELATION_SPEC_UNDOCUMENTED mz_introspection.mz_compute_operator_hydration_statuses_per_worker -->
 <!-- RELATION_SPEC_UNDOCUMENTED mz_introspection.mz_dataflow_operator_reachability -->
 <!-- RELATION_SPEC_UNDOCUMENTED mz_introspection.mz_dataflow_operator_reachability_per_worker -->

src/adapter/src/catalog/open/builtin_schema_migration.rs

@@ -424,11 +424,12 @@ static MIGRATIONS: LazyLock<Vec<MigrationStep>> = LazyLock::new(|| {
             MZ_CATALOG_SCHEMA,
             "mz_views",
         ),
-        // Required because we added the `mz_cluster_replica_resource_usage` builtin log.
-        // make_mz_indexes and make_mz_sources inline the builtin-log set as
-        // VALUES, so adding one changes both MVs' SQL fingerprints. See the NOTE
-        // above: this version must stay at the workspace's current dev version
-        // until the change ships.
+        // Required because we added builtin logs: `mz_cluster_replica_resource_usage` and
+        // `mz_compute_lifecycle_events_per_worker`. make_mz_indexes and make_mz_sources inline
+        // the builtin-log set as VALUES, so adding one changes both MVs' SQL fingerprints. A
+        // replacement step records no fingerprint, so one step per object covers every such
+        // change at this version. See the NOTE above: this version must stay at the workspace's
+        // current dev version until the change ships.
         MigrationStep::replacement(
             "26.40.0-dev.0",
             CatalogItemType::MaterializedView,

src/catalog/src/builtin.rs

@@ -1125,6 +1125,7 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
         Builtin::Log(&MZ_COMPUTE_IMPORT_FRONTIERS_PER_WORKER),
         Builtin::Log(&MZ_COMPUTE_ERROR_COUNTS_RAW),
         Builtin::Log(&MZ_COMPUTE_HYDRATION_TIMES_PER_WORKER),
+        Builtin::Log(&MZ_COMPUTE_LIFECYCLE_EVENTS_PER_WORKER),
         Builtin::Log(&MZ_COMPUTE_OPERATOR_HYDRATION_STATUSES_PER_WORKER),
         Builtin::MaterializedView(&MZ_KAFKA_SINKS),
         Builtin::MaterializedView(&MZ_KAFKA_CONNECTIONS),
@@ -2242,6 +2243,21 @@ mod tests {
             Fingerprint::fingerprint(&&mv_extra),
             "mz_sources fingerprint must change when a builtin source is added"
         );
+
+        // Adding an extra log must also change the fingerprint, because the log set is inlined
+        // alongside the source set. Without this case, adding a builtin log moves the
+        // `mz_sources` fingerprint with nothing on the PR path to announce that it needs a
+        // migration step, and catalog open panics on the upgrade.
+        let extra_log = logs[0];
+        let mv_extra_log = builtin::make_mz_sources(
+            sources.iter().copied(),
+            logs.iter().copied().chain(std::iter::once(extra_log)),
+        );
+        assert_ne!(
+            fp_base,
+            Fingerprint::fingerprint(&&mv_extra_log),
+            "mz_sources fingerprint must change when a builtin log is added"
+        );
     }
 
     /// Verifies that the `mz_indexes` materialized view fingerprint changes

src/catalog/src/builtin/mz_introspection.rs

@@ -357,6 +357,39 @@ pub static MZ_COMPUTE_HYDRATION_TIMES_PER_WORKER: LazyLock<BuiltinLog> =
         }),
     });
 
+pub static MZ_COMPUTE_LIFECYCLE_EVENTS_PER_WORKER: LazyLock<BuiltinLog> =
+    LazyLock::new(|| BuiltinLog {
+        name: "mz_compute_lifecycle_events_per_worker",
+        schema: MZ_INTROSPECTION_SCHEMA,
+        oid: oid::LOG_MZ_COMPUTE_LIFECYCLE_EVENTS_PER_WORKER_OID,
+        variant: LogVariant::Compute(ComputeLog::LifecycleEvent),
+        access: vec![PUBLIC_SELECT],
+        ontology: Some(Ontology {
+            entity_name: "compute_lifecycle_event_per_worker",
+            description: "The lifecycle stages each compute export has reached, with the \
+                          wallclock instant each was reached at, reported by the worker that \
+                          observed it.",
+            links: &const {
+                [OntologyLink {
+                    name: "lifecycle_event_of",
+                    target: "compute_export_per_worker",
+                    properties: LinkProperties::MapsTo {
+                        source_column: "export_id",
+                        target_column: "export_id",
+                        via: None,
+                        from_type: Some(SemanticType::GlobalId),
+                        to_type: Some(SemanticType::GlobalId),
+                        note: Some(
+                            "Both relations are per worker, so the join is on \
+                                 (export_id, worker_id).",
+                        ),
+                    },
+                }]
+            },
+            column_semantic_types: &[("export_id", SemanticType::GlobalId)],
+        }),
+    });
+
 pub static MZ_COMPUTE_OPERATOR_HYDRATION_STATUSES_PER_WORKER: LazyLock<BuiltinLog> =
     LazyLock::new(|| BuiltinLog {
         name: "mz_compute_operator_hydration_statuses_per_worker",

src/catalog/src/durable/transaction.rs

@@ -1022,6 +1022,7 @@ impl<'a> Transaction<'a> {
             LogVariant::Compute(ComputeLog::OperatorHydrationStatus) => 32,
             LogVariant::Compute(ComputeLog::PrometheusMetrics) => 33,
             LogVariant::Compute(ComputeLog::ResourceUsage) => 34,
+            LogVariant::Compute(ComputeLog::LifecycleEvent) => 35,
         };
 
         let mut id: u64 = u64::from(cluster_variant) << 56;

src/compute-client/src/logging.rs

@@ -176,6 +176,8 @@ pub enum ComputeLog {
     ErrorCount,
     /// Hydration times of exported collections.
     HydrationTime,
+    /// Lifecycle events of exported collections.
+    LifecycleEvent,
     /// Hydration status of dataflow operators.
     OperatorHydrationStatus,
     /// Mappings from `GlobalId`/`LirId`` pairs to dataflow addresses.
@@ -372,6 +374,19 @@ impl LogVariant {
                 .with_key(vec![0, 1])
                 .finish(),
 
+            LogVariant::Compute(ComputeLog::LifecycleEvent) => RelationDesc::builder()
+                .with_column("export_id", SqlScalarType::String.nullable(false))
+                .with_column("worker_id", SqlScalarType::UInt64.nullable(false))
+                .with_column("dataflow_id", SqlScalarType::UInt64.nullable(false))
+                .with_column("event", SqlScalarType::String.nullable(false))
+                .with_column(
+                    "occurred_at",
+                    SqlScalarType::TimestampTz { precision: None }.nullable(false),
+                )
+                .with_column("reason", SqlScalarType::String.nullable(true))
+                .with_column("details", SqlScalarType::Jsonb.nullable(true))
+                .finish(),
+
             LogVariant::Compute(ComputeLog::OperatorHydrationStatus) => RelationDesc::builder()
                 .with_column("export_id", SqlScalarType::String.nullable(false))
                 .with_column("lir_id", SqlScalarType::UInt64.nullable(false))