diff --git a/Cargo.lock b/Cargo.lock index 922175881..14f7b11cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5466,6 +5466,7 @@ dependencies = [ name = "weaver_resolved_schema" version = "0.18.0" dependencies = [ + "log", "ordered-float", "schemars", "serde", diff --git a/crates/weaver_forge/expected_output/semconv_jq_fn/semconv_metrics.json b/crates/weaver_forge/expected_output/semconv_jq_fn/semconv_metrics.json index 28c72d06f..c1a7d6abe 100644 --- a/crates/weaver_forge/expected_output/semconv_jq_fn/semconv_metrics.json +++ b/crates/weaver_forge/expected_output/semconv_jq_fn/semconv_metrics.json @@ -193,6 +193,9 @@ "source_group": "registry.url" } }, + "includes_group": [ + "http.client.server_and_port" + ], "provenance": { "path": "data/http.yaml", "registry_id": "default" @@ -438,6 +441,7 @@ "source_group": "attributes.jvm.memory" } }, + "extends_group": "attributes.jvm.memory", "provenance": { "path": "data/jvm-metrics.yaml", "registry_id": "default" @@ -557,6 +561,7 @@ "source_group": "attributes.jvm.memory" } }, + "extends_group": "attributes.jvm.memory", "provenance": { "path": "data/jvm-metrics.yaml", "registry_id": "default" @@ -680,6 +685,7 @@ "source_group": "attributes.jvm.memory" } }, + "extends_group": "attributes.jvm.memory", "provenance": { "path": "data/jvm-metrics.yaml", "registry_id": "default" @@ -775,6 +781,7 @@ "source_group": "attributes.jvm.memory" } }, + "extends_group": "attributes.jvm.memory", "provenance": { "path": "data/jvm-metrics.yaml", "registry_id": "default" diff --git a/crates/weaver_forge/expected_output/semconv_jq_fn/semconv_metrics_not_deprecated.json b/crates/weaver_forge/expected_output/semconv_jq_fn/semconv_metrics_not_deprecated.json index 8776bd7a9..e0ec69e7b 100644 --- a/crates/weaver_forge/expected_output/semconv_jq_fn/semconv_metrics_not_deprecated.json +++ b/crates/weaver_forge/expected_output/semconv_jq_fn/semconv_metrics_not_deprecated.json @@ -193,6 +193,9 @@ "source_group": "registry.url" } }, + "includes_group": [ + "http.client.server_and_port" + ], "provenance": { "path": "data/http.yaml", "registry_id": "default" @@ -428,6 +431,7 @@ "source_group": "attributes.jvm.memory" } }, + "extends_group": "attributes.jvm.memory", "provenance": { "path": "data/jvm-metrics.yaml", "registry_id": "default" @@ -513,6 +517,7 @@ "source_group": "attributes.jvm.memory" } }, + "extends_group": "attributes.jvm.memory", "provenance": { "path": "data/jvm-metrics.yaml", "registry_id": "default" @@ -615,6 +620,7 @@ "source_group": "attributes.jvm.memory" } }, + "extends_group": "attributes.jvm.memory", "provenance": { "path": "data/jvm-metrics.yaml", "registry_id": "default" @@ -700,6 +706,7 @@ "source_group": "attributes.jvm.memory" } }, + "extends_group": "attributes.jvm.memory", "provenance": { "path": "data/jvm-metrics.yaml", "registry_id": "default" diff --git a/crates/weaver_forge/src/lib.rs b/crates/weaver_forge/src/lib.rs index c293bfef2..e2d8bf1fa 100644 --- a/crates/weaver_forge/src/lib.rs +++ b/crates/weaver_forge/src/lib.rs @@ -42,6 +42,7 @@ mod filter; mod formats; pub mod jq; pub mod registry; +pub mod v2; /// Name of the Weaver configuration file. pub const WEAVER_YAML: &str = "weaver.yaml"; diff --git a/crates/weaver_forge/src/registry.rs b/crates/weaver_forge/src/registry.rs index 938924467..01612647d 100644 --- a/crates/weaver_forge/src/registry.rs +++ b/crates/weaver_forge/src/registry.rs @@ -297,6 +297,7 @@ mod tests { body: None, entity_associations: vec![], annotations: None, + visibility: None, }, Group { id: "apple.group".to_owned(), @@ -319,6 +320,7 @@ mod tests { body: None, entity_associations: vec![], annotations: None, + visibility: None, }, Group { id: "middle.group".to_owned(), @@ -341,6 +343,7 @@ mod tests { body: None, entity_associations: vec![], annotations: None, + visibility: None, }, ], }; diff --git a/crates/weaver_forge/src/v2/attribute.rs b/crates/weaver_forge/src/v2/attribute.rs new file mode 100644 index 000000000..28d21d291 --- /dev/null +++ b/crates/weaver_forge/src/v2/attribute.rs @@ -0,0 +1,31 @@ +//! Attribute definitions for template schema. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::{AttributeType, Examples}, + v2::CommonFields, +}; + +/// The definition of an Attribute. +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, PartialEq, Hash, Eq)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "snake_case")] +pub struct Attribute { + /// String that uniquely identifies the attribute. + pub key: String, + /// Either a string literal denoting the type as a primitive or an + /// array type, a template type or an enum definition. + pub r#type: AttributeType, + /// Sequence of example values for the attribute or single example + /// value. They are required only for string and string array + /// attributes. Example values must be of the same type of the + /// attribute. If only a single example is provided, it can directly + /// be reported without encapsulating it into a sequence/dictionary. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub examples: Option, + /// Common fields (like brief, note, attributes). + #[serde(flatten)] + pub common: CommonFields, +} diff --git a/crates/weaver_forge/src/v2/attribute_group.rs b/crates/weaver_forge/src/v2/attribute_group.rs new file mode 100644 index 000000000..48b450019 --- /dev/null +++ b/crates/weaver_forge/src/v2/attribute_group.rs @@ -0,0 +1,20 @@ +//! Version two of attribute groups. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::v2::{signal_id::SignalId, CommonFields}; + +use crate::v2::attribute::Attribute; + +/// Public attribute group. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AttributeGroup { + /// The name of the attribute group, must be unique. + pub id: SignalId, + /// List of attributes. + pub attributes: Vec, + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} diff --git a/crates/weaver_forge/src/v2/entity.rs b/crates/weaver_forge/src/v2/entity.rs new file mode 100644 index 000000000..ea26f8ae4 --- /dev/null +++ b/crates/weaver_forge/src/v2/entity.rs @@ -0,0 +1,47 @@ +//! Event related definitions structs. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::RequirementLevel, + v2::{signal_id::SignalId, CommonFields}, +}; + +use crate::v2::attribute::Attribute; + +/// The definition of an entity signal. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Entity { + /// The type of the entity. + pub r#type: SignalId, + + /// List of attributes that identify this entity. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub identity: Vec, + + /// List of attributes that describe to this entity. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub description: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} + +/// A special type of reference to attributes that remembers entity-specicific information. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct EntityAttribute { + /// Base attribute definitions. + #[serde(flatten)] + pub base: Attribute, + /// Specifies if the attribute is mandatory. Can be "required", + /// "conditionally_required", "recommended" or "opt_in". When omitted, + /// the attribute is "recommended". When set to + /// "conditionally_required", the string provided as MUST + /// specify the conditions under which the attribute is required. + pub requirement_level: RequirementLevel, +} diff --git a/crates/weaver_forge/src/v2/event.rs b/crates/weaver_forge/src/v2/event.rs new file mode 100644 index 000000000..a453e7ac5 --- /dev/null +++ b/crates/weaver_forge/src/v2/event.rs @@ -0,0 +1,69 @@ +//! Event related definitions structs. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::RequirementLevel, + v2::{signal_id::SignalId, CommonFields}, +}; + +use crate::v2::attribute::Attribute; + +/// The definition of an event signal. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Event { + /// The name of the event. + pub name: SignalId, + + /// List of attributes that belong to this event. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + + /// Which resources this event should be associated with. + /// + /// This list is an "any of" list, where a event may be associated with one or more entities, but should + /// be associated with at least one in this list. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub entity_associations: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} + +/// A special type of reference to attributes that remembers event-specicific information. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct EventAttribute { + /// Base attribute definitions. + #[serde(flatten)] + pub base: Attribute, + /// Specifies if the attribute is mandatory. Can be "required", + /// "conditionally_required", "recommended" or "opt_in". When omitted, + /// the attribute is "recommended". When set to + /// "conditionally_required", the string provided as MUST + /// specify the conditions under which the attribute is required. + pub requirement_level: RequirementLevel, +} + +/// A refinement of an event signal, for use in code generation or specific library application. +/// +/// A refinement represents a "view" of an Event that is highly optimised for a particular implementation. +/// e.g. for HTTP events, there may be a refinement that provides only the necessary information for dealing with Java's HTTP +/// client library, and drops optional or extraneous information from the underlying http event. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +pub struct EventRefinement { + /// The identity of the refinement. + pub id: SignalId, + + // TODO - This is a lazy way of doing this. We use `type` to refer + // to the underlying event definition, but override all fields here. + // We probably should copy-paste all the "event" attributes here + // including the `name`. + /// The definition of the event refinement. + #[serde(flatten)] + pub event: Event, +} diff --git a/crates/weaver_forge/src/v2/metric.rs b/crates/weaver_forge/src/v2/metric.rs new file mode 100644 index 000000000..c173b8d56 --- /dev/null +++ b/crates/weaver_forge/src/v2/metric.rs @@ -0,0 +1,81 @@ +//! Metric related definitions structs. + +use crate::v2::attribute::Attribute; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::RequirementLevel, + group::InstrumentSpec, + v2::{signal_id::SignalId, CommonFields}, +}; + +/// The definition of a metric signal. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Metric { + /// The name of the metric. + pub name: SignalId, + /// The instrument type that should be used to record the metric. Note that + /// the semantic conventions must be written using the names of the + /// synchronous instrument types (counter, gauge, updowncounter and + /// histogram). + /// For more details: [Metrics semantic conventions - Instrument types](https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/metrics/semantic_conventions#instrument-types). + pub instrument: InstrumentSpec, + /// The unit in which the metric is measured, which should adhere to the + /// [guidelines](https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/metrics/semantic_conventions#instrument-units). + pub unit: String, + /// List of attributes that should be included on this metric. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + // TODO - Should Entity Associations be "strong" links? + /// Which resources this metric should be associated with. + /// + /// This list is an "any of" list, where a metric may be associated with one or more entities, but should + /// be associated with at least one in this list. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub entity_associations: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} + +/// A special type of reference to attributes that remembers metric-specific information. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MetricAttribute { + /// Base attribute definitions. + #[serde(flatten)] + pub base: Attribute, + /// Specifies if the attribute is mandatory. Can be "required", + /// "conditionally_required", "recommended" or "opt_in". When omitted, + /// the attribute is "recommended". When set to + /// "conditionally_required", the string provided as MUST + /// specify the conditions under which the attribute is required. + /// + /// Note: For attributes that are "recommended" or "opt-in" - not all metric source will + /// create timeseries with these attributes, but for any given timeseries instance, the attributes that *were* present + /// should *remain* present. That is - a metric timeseries cannot drop attributes during its lifetime. + pub requirement_level: RequirementLevel, +} + +/// A refinement of a metric signal, for use in code-gen or specific library application. +/// +/// A refinement represents a "view" of a Metric that is highly optimised for a particular implementation. +/// e.g. for HTTP metrics, there may be a refinement that provides only the necessary information for dealing with Java's HTTP +/// client library, and drops optional or extraneous information from the underlying http metric. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +pub struct MetricRefinement { + /// The identity of the refinement. + pub id: SignalId, + + // TODO - This is a lazy way of doing this. We use `type` to refer + // to the underlying metric definition, but override all fields here. + // We probably should copy-paste all the "metric" attributes here + // including the `ty` + /// The definition of the metric refinement. + #[serde(flatten)] + pub metric: Metric, +} diff --git a/crates/weaver_forge/src/v2/mod.rs b/crates/weaver_forge/src/v2/mod.rs new file mode 100644 index 000000000..6743bf6e4 --- /dev/null +++ b/crates/weaver_forge/src/v2/mod.rs @@ -0,0 +1,9 @@ +//! Version two of weaver model. + +pub mod attribute; +pub mod attribute_group; +pub mod entity; +pub mod event; +pub mod metric; +pub mod registry; +pub mod span; diff --git a/crates/weaver_forge/src/v2/registry.rs b/crates/weaver_forge/src/v2/registry.rs new file mode 100644 index 000000000..f17f36241 --- /dev/null +++ b/crates/weaver_forge/src/v2/registry.rs @@ -0,0 +1,429 @@ +//! Version two of registry specification. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_resolved_schema::attribute::AttributeRef; + +use crate::{ + error::Error, + v2::{ + attribute::Attribute, + attribute_group::AttributeGroup, + entity::{Entity, EntityAttribute}, + event::{Event, EventAttribute, EventRefinement}, + metric::{Metric, MetricAttribute, MetricRefinement}, + span::{Span, SpanAttribute, SpanRefinement}, + }, +}; + +/// A resolved semantic convention registry used in the context of the template and policy +/// engines. +/// +/// This includes all registrys fully fleshed out and ready for codegen. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ForgeResolvedRegistry { + /// The semantic convention registry url. + #[serde(skip_serializing_if = "String::is_empty")] + pub registry_url: String, + /// The raw attributes in this registry. + pub attributes: Vec, + /// The public attribute groups in this registry. + pub attribute_groups: Vec, + // TODO - Attribute Groups + /// The signals defined in this registry. + pub signals: Signals, + /// The set of refinments defined in this registry. + pub refinements: Refinements, +} + +/// The set of all defined signals for a given semantic convention registry. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Signals { + /// The metric signals defined. + pub metrics: Vec, + /// The span signals defined. + pub spans: Vec, + /// The event signals defined. + pub events: Vec, + /// The entity signals defined. + pub entities: Vec, +} + +/// The set of all refinements for a semantic convention registry. +/// +/// A refinement is a specialization of a signal for a particular purpose, +/// e.g. creating a MySQL specific instance of a database span for the purpose +/// of codegeneration for MySQL. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Refinements { + /// The metric refinements defined. + pub metrics: Vec, + /// The span refinements defined. + pub spans: Vec, + /// The event refinements defined. + pub events: Vec, +} + +/// Conversion from Resolved schema to the "template schema". +impl TryFrom for ForgeResolvedRegistry { + type Error = Error; + fn try_from( + value: weaver_resolved_schema::v2::ResolvedTelemetrySchema, + ) -> Result { + ForgeResolvedRegistry::try_from_resolved_schema(value) + } +} + +impl ForgeResolvedRegistry { + /// Create a new template registry from a resolved schema registry. + pub fn try_from_resolved_schema( + schema: weaver_resolved_schema::v2::ResolvedTelemetrySchema, + ) -> Result { + let mut errors = Vec::new(); + + // We create an attribute lookup map. + let mut attributes: Vec = schema + .registry + .attributes + .iter() + .map(|a| Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }) + .collect(); + let attribute_lookup = + |r: &weaver_resolved_schema::v2::attribute::AttributeRef| attributes.get(r.0 as usize); + let mut metrics = Vec::new(); + for metric in schema.registry.metrics { + let attributes = metric + .attributes + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(&ar.base).map(|a| MetricAttribute { + base: Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }, + requirement_level: ar.requirement_level.clone(), + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("metric.{}", &metric.name), + attr_ref: AttributeRef(ar.base.0), + }); + } + attr + }) + .collect(); + metrics.push(Metric { + name: metric.name, + instrument: metric.instrument, + unit: metric.unit, + attributes, + entity_associations: metric.entity_associations, + common: metric.common, + }); + } + metrics.sort_by(|l, r| l.name.cmp(&r.name)); + + let mut metric_refinements: Vec = Vec::new(); + for metric in schema.refinements.metrics { + let attributes = metric + .metric + .attributes + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(&ar.base).map(|a| MetricAttribute { + base: Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }, + requirement_level: ar.requirement_level.clone(), + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("metric.{}", &metric.metric.name), + attr_ref: AttributeRef(ar.base.0), + }); + } + attr + }) + .collect(); + metric_refinements.push(MetricRefinement { + id: metric.id.clone(), + metric: Metric { + name: metric.metric.name, + instrument: metric.metric.instrument, + unit: metric.metric.unit, + attributes, + entity_associations: metric.metric.entity_associations, + common: metric.metric.common, + }, + }); + } + metric_refinements.sort_by(|l, r| l.id.cmp(&r.id)); + + let mut spans = Vec::new(); + for span in schema.registry.spans { + let attributes = span + .attributes + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(&ar.base).map(|a| SpanAttribute { + base: Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }, + requirement_level: ar.requirement_level.clone(), + sampling_relevant: ar.sampling_relevant, + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("span.{}", &span.r#type), + attr_ref: AttributeRef(ar.base.0), + }); + } + attr + }) + .collect(); + spans.push(Span { + r#type: span.r#type, + kind: span.kind, + name: span.name, + attributes, + entity_associations: span.entity_associations, + common: span.common, + }); + } + spans.sort_by(|l, r| l.r#type.cmp(&r.r#type)); + let mut span_refinements = Vec::new(); + for span in schema.refinements.spans { + let attributes = span + .span + .attributes + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(&ar.base).map(|a| SpanAttribute { + base: Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }, + requirement_level: ar.requirement_level.clone(), + sampling_relevant: ar.sampling_relevant, + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("span.{}", &span.id), + attr_ref: AttributeRef(ar.base.0), + }); + } + attr + }) + .collect(); + span_refinements.push(SpanRefinement { + id: span.id, + span: Span { + r#type: span.span.r#type, + kind: span.span.kind, + name: span.span.name, + attributes, + entity_associations: span.span.entity_associations, + common: span.span.common, + }, + }); + } + span_refinements.sort_by(|l, r| l.id.cmp(&r.id)); + + let mut events = Vec::new(); + for event in schema.registry.events { + let attributes = event + .attributes + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(&ar.base).map(|a| EventAttribute { + base: Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }, + requirement_level: ar.requirement_level.clone(), + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("event.{}", &event.name), + attr_ref: AttributeRef(ar.base.0), + }); + } + attr + }) + .collect(); + events.push(Event { + name: event.name, + attributes, + entity_associations: event.entity_associations, + common: event.common, + }); + } + events.sort_by(|l, r| l.name.cmp(&r.name)); + + // convert event refinements. + let mut event_refinements = Vec::new(); + for event in schema.refinements.events { + let attributes = event + .event + .attributes + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(&ar.base).map(|a| EventAttribute { + base: Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }, + requirement_level: ar.requirement_level.clone(), + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("event.{}", &event.id), + attr_ref: AttributeRef(ar.base.0), + }); + } + attr + }) + .collect(); + event_refinements.push(EventRefinement { + id: event.id, + event: Event { + name: event.event.name, + attributes, + entity_associations: event.event.entity_associations, + common: event.event.common, + }, + }); + } + event_refinements.sort_by(|l, r| l.id.cmp(&r.id)); + + let mut entities = Vec::new(); + for e in schema.registry.entities { + let identity = e + .identity + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(&ar.base).map(|a| EntityAttribute { + base: Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }, + requirement_level: ar.requirement_level.clone(), + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("entity.{}", &e.r#type), + attr_ref: AttributeRef(ar.base.0), + }); + } + attr + }) + .collect(); + + let description = e + .description + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(&ar.base).map(|a| EntityAttribute { + base: Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }, + requirement_level: ar.requirement_level.clone(), + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("entity.{}", &e.r#type), + attr_ref: AttributeRef(ar.base.0), + }); + } + attr + }) + .collect(); + entities.push(Entity { + r#type: e.r#type, + identity, + description, + common: e.common, + }); + } + entities.sort_by(|l, r| l.r#type.cmp(&r.r#type)); + + let mut attribute_groups = Vec::new(); + for ag in schema.registry.attribute_groups { + let attributes = ag + .attributes + .iter() + .filter_map(|ar| { + let attr = attribute_lookup(ar).map(|a| Attribute { + key: a.key.clone(), + r#type: a.r#type.clone(), + examples: a.examples.clone(), + common: a.common.clone(), + }); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: format!("attribute_group.{}", &ag.id), + attr_ref: AttributeRef(ar.0), + }); + } + attr + }) + .collect(); + attribute_groups.push(AttributeGroup { + id: ag.id, + attributes, + common: ag.common.clone(), + }); + } + + // Now we sort the attributes, since we aren't looking them up anymore. + attributes.sort_by(|l, r| l.key.cmp(&r.key)); + + if !errors.is_empty() { + return Err(Error::CompoundError(errors)); + } + + Ok(Self { + registry_url: schema.schema_url.clone(), + attributes, + attribute_groups, + signals: Signals { + metrics, + spans, + events, + entities, + }, + refinements: Refinements { + metrics: metric_refinements, + spans: span_refinements, + events: event_refinements, + }, + }) + } +} diff --git a/crates/weaver_forge/src/v2/span.rs b/crates/weaver_forge/src/v2/span.rs new file mode 100644 index 000000000..48d2ba89b --- /dev/null +++ b/crates/weaver_forge/src/v2/span.rs @@ -0,0 +1,77 @@ +//! Span related definitions structs. + +use crate::v2::attribute::Attribute; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::RequirementLevel, + group::SpanKindSpec, + v2::{signal_id::SignalId, span::SpanName, CommonFields}, +}; + +/// The definition of a span signal. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Span { + /// The type of the Span. This denotes the identity + /// of the "shape" of this span, and must be unique. + pub r#type: SignalId, + /// Specifies the kind of the span. + pub kind: SpanKindSpec, + /// The name pattern for the span. + pub name: SpanName, + /// List of attributes that should be included on this span. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + /// Which resources this span should be associated with. + /// + /// This list is an "any of" list, where a span may be associated with one or more entities, but should + /// be associated with at least one in this list. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub entity_associations: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} + +/// A special type of reference to attributes that remembers span-specicific information. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SpanAttribute { + /// Base attribute definitions. + #[serde(flatten)] + pub base: Attribute, + /// Specifies if the attribute is mandatory. Can be "required", + /// "conditionally_required", "recommended" or "opt_in". When omitted, + /// the attribute is "recommended". When set to + /// "conditionally_required", the string provided as MUST + /// specify the conditions under which the attribute is required. + pub requirement_level: RequirementLevel, + + /// Specifies if the attribute is (especially) relevant for sampling + /// and thus should be set at span start. It defaults to false. + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_relevant: Option, +} + +/// A refinement of a span signal, for use in code-gen or specific library application. +/// +/// A refinement represents a "view" of a Span that is highly optimised for a particular implementation. +/// e.g. for HTTP spans, there may be a refinement that provides only the necessary information for dealing with Java's HTTP +/// client library, and drops optional or extraneous information from the underlying http span. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +pub struct SpanRefinement { + /// The identity of the refinement. + pub id: SignalId, + + // TODO - This is a lazy way of doing this. We use `type` to refer + // to the underlying span definition, but override all fields here. + // We probably should copy-paste all the "span" attributes here + // including the `ty` + /// The definition of the metric refinement. + #[serde(flatten)] + pub span: Span, +} diff --git a/crates/weaver_resolved_schema/Cargo.toml b/crates/weaver_resolved_schema/Cargo.toml index 72f75feab..edb842f42 100644 --- a/crates/weaver_resolved_schema/Cargo.toml +++ b/crates/weaver_resolved_schema/Cargo.toml @@ -19,6 +19,7 @@ thiserror.workspace = true serde.workspace = true ordered-float.workspace = true schemars.workspace = true +log.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/crates/weaver_resolved_schema/src/catalog.rs b/crates/weaver_resolved_schema/src/catalog.rs index f74a72173..c84bbb887 100644 --- a/crates/weaver_resolved_schema/src/catalog.rs +++ b/crates/weaver_resolved_schema/src/catalog.rs @@ -21,7 +21,7 @@ use weaver_semconv::stability::Stability; pub struct Catalog { /// Catalog of attributes used in the schema. #[serde(skip_serializing_if = "Vec::is_empty")] - attributes: Vec, + pub(crate) attributes: Vec, } /// Statistics on a catalog. diff --git a/crates/weaver_resolved_schema/src/lib.rs b/crates/weaver_resolved_schema/src/lib.rs index a02c291a1..037dd0138 100644 --- a/crates/weaver_resolved_schema/src/lib.rs +++ b/crates/weaver_resolved_schema/src/lib.rs @@ -29,6 +29,7 @@ pub mod registry; pub mod resource; pub mod signal; pub mod tags; +pub mod v2; pub mod value; /// The registry ID for the OpenTelemetry semantic conventions. @@ -136,6 +137,7 @@ impl ResolvedTelemetrySchema { body: None, annotations: None, entity_associations: vec![], + visibility: None, }); } @@ -156,7 +158,7 @@ impl ResolvedTelemetrySchema { let al = AttributeLineage::new(group_id); lineage.add_attribute_lineage(attr.name.clone(), al); } - let attr_refs = self.catalog.add_attributes(attrs); + let attr_refs: Vec = self.catalog.add_attributes(attrs); self.registry.groups.push(Group { id: group_id.to_owned(), r#type: GroupType::AttributeGroup, @@ -178,6 +180,7 @@ impl ResolvedTelemetrySchema { body: None, annotations: None, entity_associations: vec![], + visibility: None, }); } diff --git a/crates/weaver_resolved_schema/src/lineage.rs b/crates/weaver_resolved_schema/src/lineage.rs index ac9437b11..df0126548 100644 --- a/crates/weaver_resolved_schema/src/lineage.rs +++ b/crates/weaver_resolved_schema/src/lineage.rs @@ -38,6 +38,11 @@ pub struct GroupLineage { /// The provenance of the source file where the group is defined. provenance: Provenance, + /// The group that this group extended, if available. + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub extends_group: Option, + /// The lineage per attribute. /// /// Note: Use a BTreeMap to ensure a deterministic order of attributes. @@ -45,6 +50,11 @@ pub struct GroupLineage { #[serde(skip_serializing_if = "BTreeMap::is_empty")] #[serde(default)] attributes: BTreeMap, + + /// (V2 Only) Attribute groups included in this group. + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub includes_group: Vec, } impl AttributeLineage { @@ -470,10 +480,22 @@ impl GroupLineage { pub fn new(provenance: Provenance) -> Self { Self { provenance, + extends_group: None, attributes: Default::default(), + includes_group: Default::default(), } } + /// Declares this group extended another group. + pub fn extends(&mut self, extends_group: &str) { + self.extends_group = Some(extends_group.to_owned()); + } + + /// Records what attribute groups were included (v2 only). + pub fn includes_group(&mut self, group_id: &str) { + self.includes_group.push(group_id.to_owned()); + } + /// Adds an attribute lineage. pub fn add_attribute_lineage(&mut self, attr_id: String, attribute_lineage: AttributeLineage) { _ = self.attributes.insert(attr_id, attribute_lineage); diff --git a/crates/weaver_resolved_schema/src/registry.rs b/crates/weaver_resolved_schema/src/registry.rs index 224dd300c..3ce1e8322 100644 --- a/crates/weaver_resolved_schema/src/registry.rs +++ b/crates/weaver_resolved_schema/src/registry.rs @@ -20,6 +20,7 @@ use weaver_semconv::deprecated::Deprecated; use weaver_semconv::group::{GroupType, InstrumentSpec, SpanKindSpec}; use weaver_semconv::provenance::Provenance; use weaver_semconv::stability::Stability; +use weaver_semconv::v2::attribute_group::AttributeGroupVisibilitySpec; use weaver_semconv::YamlValue; /// A semantic convention registry. @@ -134,6 +135,12 @@ pub struct Group { #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] pub entity_associations: Vec, + /// Visibility of the attribute group. + /// This is only used for v2 conversion. + #[serde(default)] + #[serde(skip_serializing)] + #[schemars(skip)] + pub visibility: Option, } impl Group { diff --git a/crates/weaver_resolved_schema/src/v2/attribute.rs b/crates/weaver_resolved_schema/src/v2/attribute.rs new file mode 100644 index 000000000..2436aada8 --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/attribute.rs @@ -0,0 +1,45 @@ +//! Attribute definitions for resolved schema. + +use std::fmt::Display; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::{AttributeType, Examples}, + v2::CommonFields, +}; + +/// The definition of an Attribute. +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, PartialEq, Hash, Eq)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "snake_case")] +pub struct Attribute { + /// String that uniquely identifies the attribute. + pub key: String, + /// Either a string literal denoting the type as a primitive or an + /// array type, a template type or an enum definition. + pub r#type: AttributeType, + /// Sequence of example values for the attribute or single example + /// value. They are required only for string and string array + /// attributes. Example values must be of the same type of the + /// attribute. If only a single example is provided, it can directly + /// be reported without encapsulating it into a sequence/dictionary. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub examples: Option, + /// Common fields (like brief, note, attributes). + #[serde(flatten)] + pub common: CommonFields, +} + +/// Reference to an attribute in the catalog. +#[derive( + Serialize, Deserialize, Debug, Copy, Clone, Eq, PartialEq, PartialOrd, Ord, JsonSchema, Hash, +)] +pub struct AttributeRef(pub u32); + +impl Display for AttributeRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "AttributeRef({})", self.0) + } +} diff --git a/crates/weaver_resolved_schema/src/v2/attribute_group.rs b/crates/weaver_resolved_schema/src/v2/attribute_group.rs new file mode 100644 index 000000000..9a09f77f2 --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/attribute_group.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The new way we want to define attribute groups going forward. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::v2::{signal_id::SignalId, CommonFields}; + +use crate::v2::attribute::AttributeRef; + +/// Public attribute group. +/// +/// An attribute group is a grouping of attributes that can be leveraged +/// in codegen. For example, rather than passing attributes on at a time, +/// a temporary structure could be made to contain all of them and report +/// the bundle as a group to different signals. +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, PartialEq)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "snake_case")] +pub struct AttributeGroup { + /// The name of the attribute group, must be unique. + pub id: SignalId, + + /// List of attributes and group references that belong to this group + pub attributes: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} diff --git a/crates/weaver_resolved_schema/src/v2/catalog.rs b/crates/weaver_resolved_schema/src/v2/catalog.rs new file mode 100644 index 000000000..299914479 --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/catalog.rs @@ -0,0 +1,152 @@ +//! Catalog of attributes and other. + +use std::collections::BTreeMap; + +use crate::v2::attribute::{Attribute, AttributeRef}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// A catalog of indexed attributes shared across semconv groups, or signals. +/// Attribute references are used to refer to attributes in the catalog. +/// +/// Note: This is meant to be a temporary datastructure used for creating +/// the registry. +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, Default)] +#[serde(deny_unknown_fields)] +#[must_use] +pub(crate) struct Catalog { + /// Catalog of attributes used in the schema. + #[serde(skip_serializing_if = "Vec::is_empty")] + attributes: Vec, + /// Lookup map to more efficiently find attributes. + lookup: BTreeMap>, +} + +/// Collapses this catalog into the attribute list, preserving order. +impl From for Vec { + fn from(val: Catalog) -> Self { + val.attributes + } +} + +impl Catalog { + /// Creates a catalog from a list of attributes. + pub(crate) fn from_attributes(attributes: Vec) -> Self { + let mut lookup: BTreeMap> = BTreeMap::new(); + for (idx, attr) in attributes.iter().enumerate() { + lookup.entry(attr.key.clone()).or_default().push(idx); + } + Self { attributes, lookup } + } + + /// Converts an attribute from V1 into an AttributeRef + /// on the current list of attributes in the order of this catalog. + #[must_use] + pub(crate) fn convert_ref( + &self, + attribute: &crate::attribute::Attribute, + ) -> Option { + // Note - we do a fast lookup to contentious attributes, + // then linear scan of attributes with same key but different + // other aspects. + self.lookup.get(&attribute.name)?.iter().find_map(|idx| { + self.attributes + .get(*idx) + .filter(|a| { + a.key == attribute.name + && a.r#type == attribute.r#type + && a.examples == attribute.examples + && a.common.brief == attribute.brief + && a.common.note == attribute.note + && a.common.deprecated == attribute.deprecated + && attribute + .stability + .as_ref() + .map(|s| a.common.stability == *s) + .unwrap_or(false) + && attribute + .annotations + .as_ref() + .map(|ans| a.common.annotations == *ans) + .unwrap_or(a.common.annotations.is_empty()) + }) + .map(|_| AttributeRef(*idx as u32)) + }) + } +} + + +#[cfg(test)] +mod test { + use std::collections::BTreeMap; + + use weaver_semconv::attribute::{BasicRequirementLevelSpec, RequirementLevel}; + use weaver_semconv::{attribute::AttributeType, stability::Stability}; + + use crate::v2::attribute::Attribute; + use crate::v2::CommonFields; + use super::Catalog; + + #[test] + fn test_lookup_works() { + let key = "test.key".to_owned(); + let atype = AttributeType::PrimitiveOrArray(weaver_semconv::attribute::PrimitiveOrArrayTypeSpec::String); + let brief = "brief".to_owned(); + let note = "note".to_owned(); + let stability = Stability::Stable; + let annotations = BTreeMap::new(); + let catalog = Catalog::from_attributes(vec![ + Attribute { + key: key.clone(), + r#type: atype.clone(), + examples: None, + common: CommonFields { + brief: brief.clone(), + note: note.clone(), + stability: stability.clone(), + deprecated: None, + annotations: annotations.clone(), + }, + }, + ]); + + let result = catalog.convert_ref(&crate::attribute::Attribute { + name: key.clone(), + r#type: atype.clone(), + brief: brief.clone(), + examples: None, + tag: None, + requirement_level: RequirementLevel::Basic(BasicRequirementLevelSpec::Required), + sampling_relevant: Some(true), + note: note.clone(), + stability: Some(stability.clone()), + deprecated: None, + prefix: false, + tags: None, + annotations: Some(annotations.clone()), + value: None, + role: None, + }); + assert_eq!(result.is_some(), true); + + // Make sure "none" annotations is the same as empty annotations. + let result2 = catalog.convert_ref(&crate::attribute::Attribute { + name: key.clone(), + r#type: atype.clone(), + brief: brief.clone(), + examples: None, + tag: None, + requirement_level: RequirementLevel::Basic(BasicRequirementLevelSpec::Required), + sampling_relevant: Some(true), + note: note.clone(), + stability: Some(stability.clone()), + deprecated: None, + prefix: false, + tags: None, + annotations: None, + value: None, + role: None, + }); + assert_eq!(result2.is_some(), true); + } +} \ No newline at end of file diff --git a/crates/weaver_resolved_schema/src/v2/entity.rs b/crates/weaver_resolved_schema/src/v2/entity.rs new file mode 100644 index 000000000..a37ab406f --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/entity.rs @@ -0,0 +1,43 @@ +//! Entity related definition structs. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::RequirementLevel, + v2::{signal_id::SignalId, CommonFields}, +}; + +use crate::v2::attribute::AttributeRef; + +/// The definition of an Entity signal. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Entity { + /// The type of the Entity. + pub r#type: SignalId, + + /// The attributes that make the identity of the Entity. + pub identity: Vec, + /// The attributes that make the description of the Entity. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub description: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} + +/// A special type of reference to attributes that remembers entity-specicific information. +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct EntityAttributeRef { + /// Reference, by index, to the attribute catalog. + pub base: AttributeRef, + /// Specifies if the attribute is mandatory. Can be "required", + /// "conditionally_required", "recommended" or "opt_in". When omitted, + /// the attribute is "recommended". When set to + /// "conditionally_required", the string provided as MUST + /// specify the conditions under which the attribute is required. + pub requirement_level: RequirementLevel, +} diff --git a/crates/weaver_resolved_schema/src/v2/event.rs b/crates/weaver_resolved_schema/src/v2/event.rs new file mode 100644 index 000000000..0bb7b4698 --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/event.rs @@ -0,0 +1,69 @@ +//! Event related definition structs. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::RequirementLevel, + v2::{signal_id::SignalId, CommonFields}, +}; + +use crate::v2::attribute::AttributeRef; + +/// The definition of an Event signal. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Event { + /// The name of the event. + pub name: SignalId, + + /// List of attributes that belong to this event. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + + // TODO - Should Entity Associations be "strong" links? + /// Which entities this event should be associated with. + /// + /// This list is an "any of" list, where a event may be associated with one or more entities, but should + /// be associated with at least one in this list. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub entity_associations: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} + +/// A special type of reference to attributes that remembers event-specicific information. +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct EventAttributeRef { + /// Reference, by index, to the attribute catalog. + pub base: AttributeRef, + /// Specifies if the attribute is mandatory. Can be "required", + /// "conditionally_required", "recommended" or "opt_in". When omitted, + /// the attribute is "recommended". When set to + /// "conditionally_required", the string provided as MUST + /// specify the conditions under which the attribute is required. + pub requirement_level: RequirementLevel, +} + +/// A refinement of an event, for use in code-gen or specific library application. +/// +/// A refinement represents a "view" of a Event that is highly optimised for a particular implementation. +/// e.g. for HTTP events, there may be a refinement that provides only the necessary information for dealing with Java's HTTP +/// client library, and drops optional or extraneous information from the underlying http event. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +pub struct EventRefinement { + /// The identity of the refinement + pub id: SignalId, + + // TODO - This is a lazy way of doing this. We use `name` to refer + // to the underlying event definition, but override all fields here. + // We probably should copy-paste all the "event" attributes here + // including the `ty` + /// The definition of the event refinement. + #[serde(flatten)] + pub event: Event, +} diff --git a/crates/weaver_resolved_schema/src/v2/metric.rs b/crates/weaver_resolved_schema/src/v2/metric.rs new file mode 100644 index 000000000..d98bd3dec --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/metric.rs @@ -0,0 +1,80 @@ +//! Metric related definitions structs. + +use crate::v2::attribute::AttributeRef; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::RequirementLevel, + group::InstrumentSpec, + v2::{signal_id::SignalId, CommonFields}, +}; + +/// The definition of a metric signal. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Metric { + /// The name of the metric. + pub name: SignalId, + /// The instrument type that should be used to record the metric. Note that + /// the semantic conventions must be written using the names of the + /// synchronous instrument types (counter, gauge, updowncounter and + /// histogram). + /// For more details: [Metrics semantic conventions - Instrument types](https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/metrics/semantic_conventions#instrument-types). + pub instrument: InstrumentSpec, + /// The unit in which the metric is measured, which should adhere to the + /// [guidelines](https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/metrics/semantic_conventions#instrument-units). + pub unit: String, + /// List of attributes that should be included on this metric. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + // TODO - Should Entity Associations be "strong" links? + /// Which entities this metric should be associated with. + /// + /// This list is an "any of" list, where a metric may be associated with one or more entities, but should + /// be associated with at least one in this list. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub entity_associations: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} + +/// A special type of reference to attributes that remembers metric-specicific information. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MetricAttributeRef { + /// Reference, by index, to the attribute catalog. + pub base: AttributeRef, + /// Specifies if the attribute is mandatory. Can be "required", + /// "conditionally_required", "recommended" or "opt_in". When omitted, + /// the attribute is "recommended". When set to + /// "conditionally_required", the string provided as MUST + /// specify the conditions under which the attribute is required. + /// + /// Note: For attributes that are "recommended" or "opt-in" - not all metric source will + /// create timeseries with these attributes, but for any given timeseries instance, the attributes that *were* present + /// should *remain* present. That is - a metric timeseries cannot drop attributes during its lifetime. + pub requirement_level: RequirementLevel, +} + +/// A refinement of a metric signal, for use in code-gen or specific library application. +/// +/// A refinement represents a "view" of a Metric that is highly optimised for a particular implementation. +/// e.g. for HTTP metrics, there may be a refinement that provides only the necessary information for dealing with Java's HTTP +/// client library, and drops optional or extraneous information from the underlying http metric. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +pub struct MetricRefinement { + /// The identity of the refinement. + pub id: SignalId, + + // TODO - This is a lazy way of doing this. We use `name` to refer + // to the underlying metric definition, but override all fields here. + // We probably should copy-paste all the "metric" attributes here + // including the `ty` + /// The definition of the metric refinement. + #[serde(flatten)] + pub metric: Metric, +} diff --git a/crates/weaver_resolved_schema/src/v2/mod.rs b/crates/weaver_resolved_schema/src/v2/mod.rs new file mode 100644 index 000000000..657f275de --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/mod.rs @@ -0,0 +1,681 @@ +//! Version 2 of semantic convention schema. + +use std::collections::{HashMap, HashSet}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + group::GroupType, + v2::{ + attribute_group::AttributeGroupVisibilitySpec, signal_id::SignalId, span::SpanName, + CommonFields, + }, +}; + +use crate::v2::{ + attribute_group::AttributeGroup, + catalog::Catalog, + entity::Entity, + metric::Metric, + refinements::Refinements, + registry::Registry, + span::{Span, SpanRefinement}, +}; + +pub mod attribute; +pub mod attribute_group; +pub mod catalog; +pub mod entity; +pub mod event; +pub mod metric; +pub mod refinements; +pub mod registry; +pub mod span; + +/// A Resolved Telemetry Schema. +/// A Resolved Telemetry Schema is self-contained and doesn't contain any +/// external references to other schemas or semantic conventions. +#[derive(Serialize, Deserialize, Debug, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ResolvedTelemetrySchema { + /// Version of the file structure. + pub file_format: String, + /// Schema URL that this file is published at. + pub schema_url: String, + /// The ID of the registry that this schema belongs to. + pub registry_id: String, + /// The registry that this schema belongs to. + pub registry: Registry, + /// Refinements for the registry + pub refinements: Refinements, + // TODO - versions, dependencies and other options. +} + +/// Easy conversion from v1 to v2. +impl TryFrom for ResolvedTelemetrySchema { + type Error = crate::error::Error; + fn try_from(value: crate::ResolvedTelemetrySchema) -> Result { + let (registry, refinements) = convert_v1_to_v2(value.catalog, value.registry)?; + Ok(ResolvedTelemetrySchema { + // TODO - bump file format? + file_format: value.file_format, + schema_url: value.schema_url, + registry_id: value.registry_id, + registry, + refinements, + }) + } +} + +fn fix_group_id(prefix: &'static str, group_id: &str) -> SignalId { + if group_id.starts_with(prefix) { + group_id.trim_start_matches(prefix).to_owned().into() + } else { + group_id.to_owned().into() + } +} + +fn fix_span_group_id(group_id: &str) -> SignalId { + fix_group_id("span.", group_id) +} + +/// Converts a V1 registry + catalog to V2. +pub fn convert_v1_to_v2( + c: crate::catalog::Catalog, + r: crate::registry::Registry, +) -> Result<(Registry, Refinements), crate::error::Error> { + // When pulling attributes, as we collapse things, we need to filter + // to just unique. + let attributes: HashSet = c + .attributes + .iter() + .cloned() + .map(|a| { + attribute::Attribute { + key: a.name, + r#type: a.r#type, + examples: a.examples, + common: CommonFields { + brief: a.brief, + note: a.note, + // TODO - Check this assumption. + stability: a + .stability + .unwrap_or(weaver_semconv::stability::Stability::Alpha), + deprecated: a.deprecated, + annotations: a.annotations.unwrap_or_default(), + }, + } + }) + .collect(); + + let v2_catalog = Catalog::from_attributes(attributes.into_iter().collect()); + + // Create a lookup so we can check inheritance. + let mut group_type_lookup = HashMap::new(); + for g in r.groups.iter() { + let _ = group_type_lookup.insert(g.id.clone(), g.r#type.clone()); + } + // Pull signals from the registry and create a new span-focused registry. + let mut spans = Vec::new(); + let mut span_refinements = Vec::new(); + let mut metrics = Vec::new(); + let mut metric_refinements = Vec::new(); + let mut events = Vec::new(); + let mut event_refinements = Vec::new(); + let mut entities = Vec::new(); + let mut attribute_groups = Vec::new(); + for g in r.groups.iter() { + match g.r#type { + GroupType::Span => { + // Check if we extend another span. + let is_refinement = g + .lineage + .as_ref() + .and_then(|l| l.extends_group.as_ref()) + .and_then(|parent| group_type_lookup.get(parent)) + .map(|t| *t == GroupType::Span) + .unwrap_or(false); + // Pull all the attribute references. + let mut span_attributes = Vec::new(); + for attr in g.attributes.iter().filter_map(|a| c.attribute(a)) { + if let Some(a) = v2_catalog.convert_ref(attr) { + span_attributes.push(span::SpanAttributeRef { + base: a, + requirement_level: attr.requirement_level.clone(), + sampling_relevant: attr.sampling_relevant, + }); + } else { + // TODO logic error! + log::info!("Logic failue - unable to convert attribute {attr:?}"); + } + } + if !is_refinement { + let span = Span { + r#type: fix_span_group_id(&g.id), + kind: g + .span_kind + .clone() + .unwrap_or(weaver_semconv::group::SpanKindSpec::Internal), + // TODO - Pass advanced name controls through V1 groups. + name: SpanName { + note: g.name.clone().unwrap_or_default(), + }, + entity_associations: g.entity_associations.clone(), + common: CommonFields { + brief: g.brief.clone(), + note: g.note.clone(), + stability: g + .stability + .clone() + .unwrap_or(weaver_semconv::stability::Stability::Alpha), + deprecated: g.deprecated.clone(), + annotations: g.annotations.clone().unwrap_or_default(), + }, + attributes: span_attributes, + }; + spans.push(span.clone()); + span_refinements.push(SpanRefinement { + id: span.r#type.clone(), + span, + }); + } else { + // unwrap should be safe because we verified this is a refinement earlier. + let span_type = g + .lineage + .as_ref() + .and_then(|l| l.extends_group.as_ref()) + .map(|id| fix_span_group_id(id)) + .expect("Refinement extraction issue - this is a logic bug"); + span_refinements.push(SpanRefinement { + id: fix_span_group_id(&g.id), + span: Span { + r#type: span_type, + kind: g + .span_kind + .clone() + .unwrap_or(weaver_semconv::group::SpanKindSpec::Internal), + // TODO - Pass advanced name controls through V1 groups. + name: SpanName { + note: g.name.clone().unwrap_or_default(), + }, + entity_associations: g.entity_associations.clone(), + common: CommonFields { + brief: g.brief.clone(), + note: g.note.clone(), + stability: g + .stability + .clone() + .unwrap_or(weaver_semconv::stability::Stability::Alpha), + deprecated: g.deprecated.clone(), + annotations: g.annotations.clone().unwrap_or_default(), + }, + attributes: span_attributes, + }, + }); + } + } + GroupType::Event => { + let is_refinement = g + .lineage + .as_ref() + .and_then(|l| l.extends_group.as_ref()) + .and_then(|parent| group_type_lookup.get(parent)) + .map(|t| *t == GroupType::Event) + .unwrap_or(false); + let mut event_attributes = Vec::new(); + for attr in g.attributes.iter().filter_map(|a| c.attribute(a)) { + if let Some(a) = v2_catalog.convert_ref(attr) { + event_attributes.push(event::EventAttributeRef { + base: a, + requirement_level: attr.requirement_level.clone(), + }); + } else { + // TODO logic error! + log::info!("Logic failue - unable to convert attribute {attr:?}"); + } + } + let event = event::Event { + name: g + .name + .clone() + .expect("Name must exist on events prior to translation to v2") + .into(), + attributes: event_attributes, + entity_associations: g.entity_associations.clone(), + common: CommonFields { + brief: g.brief.clone(), + note: g.note.clone(), + stability: g + .stability + .clone() + .unwrap_or(weaver_semconv::stability::Stability::Alpha), + deprecated: g.deprecated.clone(), + annotations: g.annotations.clone().unwrap_or_default(), + }, + }; + if !is_refinement { + events.push(event.clone()); + event_refinements.push(event::EventRefinement { + id: event.name.clone(), + event, + }); + } else { + event_refinements.push(event::EventRefinement { + id: fix_group_id("event.", &g.id), + event, + }); + } + } + GroupType::Metric => { + // Check if we extend another metric. + let is_refinement = g + .lineage + .as_ref() + .and_then(|l| l.extends_group.as_ref()) + .and_then(|parent| group_type_lookup.get(parent)) + .map(|t| *t == GroupType::Metric) + .unwrap_or(false); + let mut metric_attributes = Vec::new(); + for attr in g.attributes.iter().filter_map(|a| c.attribute(a)) { + if let Some(a) = v2_catalog.convert_ref(attr) { + metric_attributes.push(metric::MetricAttributeRef { + base: a, + requirement_level: attr.requirement_level.clone(), + }); + } else { + // TODO logic error! + log::info!("Logic failue - unable to convert attribute {attr:?}"); + } + } + // TODO - deal with unwrap errors. + let metric = Metric { + name: g + .metric_name + .clone() + .expect("metric_name must exist on metrics prior to translation to v2") + .into(), + instrument: g + .instrument + .clone() + .expect("instrument must exist on metrics prior to translation to v2"), + unit: g + .unit + .clone() + .expect("unit must exist on metrics prior to translation to v2"), + attributes: metric_attributes, + entity_associations: g.entity_associations.clone(), + common: CommonFields { + brief: g.brief.clone(), + note: g.note.clone(), + stability: g + .stability + .clone() + .unwrap_or(weaver_semconv::stability::Stability::Alpha), + deprecated: g.deprecated.clone(), + annotations: g.annotations.clone().unwrap_or_default(), + }, + }; + if is_refinement { + metric_refinements.push(metric::MetricRefinement { + id: fix_group_id("metric.", &g.id), + metric, + }); + } else { + metrics.push(metric.clone()); + metric_refinements.push(metric::MetricRefinement { + id: metric.name.clone(), + metric, + }); + } + } + GroupType::Entity => { + let mut id_attrs = Vec::new(); + let mut desc_attrs = Vec::new(); + for attr in g.attributes.iter().filter_map(|a| c.attribute(a)) { + if let Some(a) = v2_catalog.convert_ref(attr) { + match attr.role { + Some(weaver_semconv::attribute::AttributeRole::Identifying) => { + id_attrs.push(entity::EntityAttributeRef { + base: a, + requirement_level: attr.requirement_level.clone(), + }); + } + _ => { + desc_attrs.push(entity::EntityAttributeRef { + base: a, + requirement_level: attr.requirement_level.clone(), + }); + } + } + } else { + // TODO logic error! + } + } + entities.push(Entity { + r#type: fix_group_id("entity.", &g.id), + identity: id_attrs, + description: desc_attrs, + common: CommonFields { + brief: g.brief.clone(), + note: g.note.clone(), + stability: g + .stability + .clone() + .unwrap_or(weaver_semconv::stability::Stability::Alpha), + deprecated: g.deprecated.clone(), + annotations: g.annotations.clone().unwrap_or_default(), + }, + }); + } + GroupType::AttributeGroup => { + if g.visibility + .as_ref() + .is_some_and(|v| AttributeGroupVisibilitySpec::Public == *v) + { + // Now we need to convert the group. + let mut attributes = Vec::new(); + // TODO - we need to check lineage and remove parent groups. + for attr in g.attributes.iter().filter_map(|a| c.attribute(a)) { + if let Some(a) = v2_catalog.convert_ref(attr) { + attributes.push(a); + } else { + // TODO logic error! + } + } + attribute_groups.push(AttributeGroup { + id: fix_group_id("attribute_group.", &g.id), + attributes, + common: CommonFields { + brief: g.brief.clone(), + note: g.note.clone(), + stability: g + .stability + .clone() + .unwrap_or(weaver_semconv::stability::Stability::Alpha), + deprecated: g.deprecated.clone(), + annotations: g.annotations.clone().unwrap_or_default(), + }, + }); + } + } + GroupType::MetricGroup | GroupType::Scope | GroupType::Undefined => { + // Ignored for now, we should probably issue warnings. + } + } + } + + let v2_registry = Registry { + registry_url: r.registry_url, + attributes: v2_catalog.into(), + spans, + metrics, + events, + entities, + attribute_groups, + }; + let v2_refinements = Refinements { + spans: span_refinements, + metrics: metric_refinements, + events: event_refinements, + }; + Ok((v2_registry, v2_refinements)) +} + +#[cfg(test)] +mod tests { + + use weaver_semconv::{provenance::Provenance, stability::Stability}; + + use crate::{attribute::Attribute, lineage::GroupLineage, registry::Group}; + + use super::*; + + #[test] + fn test_convert_span_v1_to_v2() { + let mut v1_catalog = crate::catalog::Catalog::from_attributes(vec![]); + let test_refs = v1_catalog.add_attributes([ + Attribute { + name: "test.key".to_owned(), + r#type: weaver_semconv::attribute::AttributeType::PrimitiveOrArray( + weaver_semconv::attribute::PrimitiveOrArrayTypeSpec::String, + ), + brief: "".to_owned(), + examples: None, + tag: None, + requirement_level: weaver_semconv::attribute::RequirementLevel::Basic( + weaver_semconv::attribute::BasicRequirementLevelSpec::Required, + ), + sampling_relevant: None, + note: "".to_owned(), + stability: Some(Stability::Stable), + deprecated: None, + prefix: false, + tags: None, + annotations: None, + value: None, + role: None, + }, + Attribute { + name: "test.key".to_owned(), + r#type: weaver_semconv::attribute::AttributeType::PrimitiveOrArray( + weaver_semconv::attribute::PrimitiveOrArrayTypeSpec::String, + ), + brief: "".to_owned(), + examples: None, + tag: None, + requirement_level: weaver_semconv::attribute::RequirementLevel::Basic( + weaver_semconv::attribute::BasicRequirementLevelSpec::Recommended, + ), + sampling_relevant: Some(true), + note: "".to_owned(), + stability: Some(Stability::Stable), + deprecated: None, + prefix: false, + tags: None, + annotations: None, + value: None, + role: None, + }, + ]); + let mut refinement_span_lineage = GroupLineage::new(Provenance::new("tmp", "tmp")); + refinement_span_lineage.extends("span.my-span"); + let v1_registry = crate::registry::Registry { + registry_url: "my.schema.url".to_owned(), + groups: vec![ + Group { + id: "span.my-span".to_owned(), + r#type: GroupType::Span, + brief: "".to_owned(), + note: "".to_owned(), + prefix: "".to_owned(), + extends: None, + stability: Some(Stability::Stable), + deprecated: None, + attributes: vec![test_refs[1]], + span_kind: Some(weaver_semconv::group::SpanKindSpec::Client), + events: vec![], + metric_name: None, + instrument: None, + unit: None, + name: Some("my span name".to_owned()), + lineage: None, + display_name: None, + body: None, + annotations: None, + entity_associations: vec![], + visibility: None, + }, + Group { + id: "span.custom".to_owned(), + r#type: GroupType::Span, + brief: "".to_owned(), + note: "".to_owned(), + prefix: "".to_owned(), + extends: None, + stability: Some(Stability::Stable), + deprecated: None, + attributes: vec![test_refs[1]], + span_kind: Some(weaver_semconv::group::SpanKindSpec::Client), + events: vec![], + metric_name: None, + instrument: None, + unit: None, + name: Some("my span name".to_owned()), + lineage: Some(refinement_span_lineage), + display_name: None, + body: None, + annotations: None, + entity_associations: vec![], + visibility: None, + }, + ], + }; + + let (v2_registry, v2_refinements) = + convert_v1_to_v2(v1_catalog, v1_registry).expect("Failed to convert v1 to v2"); + // assert only ONE attribute due to sharing. + assert_eq!(v2_registry.attributes.len(), 1); + // assert attribute fields not shared show up on ref in span. + assert_eq!(v2_registry.spans.len(), 1); + if let Some(span) = v2_registry.spans.first() { + assert_eq!(span.r#type, "my-span".to_owned().into()); + // Make sure attribute ref carries sampling relevant. + } + // Assert we have two refinements (e.g. one real span, one refinement). + assert_eq!(v2_refinements.spans.len(), 2); + let span_ref_ids: Vec = v2_refinements + .spans + .iter() + .map(|s| s.id.to_string()) + .collect(); + assert_eq!( + span_ref_ids, + vec!["my-span".to_owned(), "custom".to_owned()] + ); + } + + #[test] + fn test_convert_metric_v1_to_v2() { + let mut v1_catalog = crate::catalog::Catalog::from_attributes(vec![]); + let test_refs = v1_catalog.add_attributes([ + Attribute { + name: "test.key".to_owned(), + r#type: weaver_semconv::attribute::AttributeType::PrimitiveOrArray( + weaver_semconv::attribute::PrimitiveOrArrayTypeSpec::String, + ), + brief: "".to_owned(), + examples: None, + tag: None, + requirement_level: weaver_semconv::attribute::RequirementLevel::Basic( + weaver_semconv::attribute::BasicRequirementLevelSpec::Required, + ), + sampling_relevant: None, + note: "".to_owned(), + stability: Some(Stability::Stable), + deprecated: None, + prefix: false, + tags: None, + annotations: None, + value: None, + role: None, + }, + Attribute { + name: "test.key".to_owned(), + r#type: weaver_semconv::attribute::AttributeType::PrimitiveOrArray( + weaver_semconv::attribute::PrimitiveOrArrayTypeSpec::String, + ), + brief: "".to_owned(), + examples: None, + tag: None, + requirement_level: weaver_semconv::attribute::RequirementLevel::Basic( + weaver_semconv::attribute::BasicRequirementLevelSpec::Recommended, + ), + sampling_relevant: Some(true), + note: "".to_owned(), + stability: Some(Stability::Stable), + deprecated: None, + prefix: false, + tags: None, + annotations: None, + value: None, + role: None, + }, + ]); + let mut refinement_metric_lineage = GroupLineage::new(Provenance::new("tmp", "tmp")); + refinement_metric_lineage.extends("metric.http"); + let v1_registry = crate::registry::Registry { + registry_url: "my.schema.url".to_owned(), + groups: vec![ + Group { + id: "metric.http".to_owned(), + r#type: GroupType::Metric, + brief: "".to_owned(), + note: "".to_owned(), + prefix: "".to_owned(), + extends: None, + stability: Some(Stability::Stable), + deprecated: None, + attributes: vec![test_refs[0]], + span_kind: None, + events: vec![], + metric_name: Some("http".to_owned()), + instrument: Some(weaver_semconv::group::InstrumentSpec::UpDownCounter), + unit: Some("s".to_owned()), + name: None, + lineage: None, + display_name: None, + body: None, + annotations: None, + entity_associations: vec![], + visibility: None, + }, + Group { + id: "metric.http.custom".to_owned(), + r#type: GroupType::Metric, + brief: "".to_owned(), + note: "".to_owned(), + prefix: "".to_owned(), + extends: None, + stability: Some(Stability::Stable), + deprecated: None, + attributes: vec![test_refs[1]], + span_kind: None, + events: vec![], + metric_name: Some("http".to_owned()), + instrument: Some(weaver_semconv::group::InstrumentSpec::UpDownCounter), + unit: Some("s".to_owned()), + name: None, + lineage: Some(refinement_metric_lineage), + display_name: None, + body: None, + annotations: None, + entity_associations: vec![], + visibility: None, + }, + ], + }; + + let (v2_registry, v2_refinements) = + convert_v1_to_v2(v1_catalog, v1_registry).expect("Failed to convert v1 to v2"); + // assert only ONE attribute due to sharing. + assert_eq!(v2_registry.attributes.len(), 1); + // assert attribute fields not shared show up on ref in span. + assert_eq!(v2_registry.metrics.len(), 1); + if let Some(metric) = v2_registry.metrics.first() { + assert_eq!(metric.name, "http".to_owned().into()); + // Make sure attribute ref carries sampling relevant. + } + // Assert we have two refinements (e.g. one real span, one refinement). + assert_eq!(v2_refinements.metrics.len(), 2); + let metric_ref_ids: Vec = v2_refinements + .metrics + .iter() + .map(|s| s.id.to_string()) + .collect(); + assert_eq!( + metric_ref_ids, + vec!["http".to_owned(), "http.custom".to_owned()] + ); + } +} diff --git a/crates/weaver_resolved_schema/src/v2/refinements.rs b/crates/weaver_resolved_schema/src/v2/refinements.rs new file mode 100644 index 000000000..d93f53d15 --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/refinements.rs @@ -0,0 +1,32 @@ +//! A semantic convention refinements. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v2::{event::EventRefinement, metric::MetricRefinement, span::SpanRefinement}; + +/// Semantic convention refinements. +/// +/// Refinements are a specialization of a signal that can be used to optimise documentation, +/// or code generation. A refinement will *always* match the conventions defined by the +/// signal it refines. Refinements cannot be inferred from signals over the wire (e.g. OTLP). +/// This is because any identifying feature of a refinement is used purely for codegen but has +/// no storage location in OTLP. +/// +/// Note: Refinements will always include a "base" refinement for every signal definition. +/// For example, if a Metric signal named `my_metric` is defined, there will be +/// a metric refinement named `my_metric` as well. +/// This allows code generation to *only* interact with refinements, if desired, to +/// provide optimised methods for generating telemetry signals. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Refinements { + /// A list of span refinements. + pub spans: Vec, + + /// A list of metric refinements. + pub metrics: Vec, + + /// A list of event refinements. + pub events: Vec, +} diff --git a/crates/weaver_resolved_schema/src/v2/registry.rs b/crates/weaver_resolved_schema/src/v2/registry.rs new file mode 100644 index 000000000..05fc7fa16 --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/registry.rs @@ -0,0 +1,62 @@ +//! A semantic convention registry. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v2::{ + attribute::{Attribute, AttributeRef}, + attribute_group::AttributeGroup, + entity::Entity, + event::Event, + metric::Metric, + span::Span, +}; + +/// A semantic convention registry. +/// +/// The semantic convention is composed of definitions of +/// attributes, metrics, logs, etc. that will be sent over the wire (e.g. OTLP). +/// +/// Note: The registry does not include signal refinements. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Registry { + /// Catalog of attributes used in the schema. + pub attributes: Vec, + + /// Catalog of (public) attribute groups. + pub attribute_groups: Vec, + + /// The semantic convention registry url. + /// + /// This is the base URL, under which this registry can be found. + pub registry_url: String, + + /// A list of span signal definitions. + pub spans: Vec, + + /// A list of metric signal definitions. + pub metrics: Vec, + + /// A list of event signal definitions. + pub events: Vec, + + /// A list of entity signal definitions. + pub entities: Vec, +} + +impl Registry { + /// Returns the attribute from an attribute ref if it exists. + #[must_use] + pub fn attribute(&self, attribute_ref: &AttributeRef) -> Option<&Attribute> { + self.attributes.get(attribute_ref.0 as usize) + } + /// Returns the attribute name from an attribute ref if it exists + /// in the catalog or None if it does not exist. + #[must_use] + pub fn attribute_key(&self, attribute_ref: &AttributeRef) -> Option<&str> { + self.attributes + .get(attribute_ref.0 as usize) + .map(|attr| attr.key.as_ref()) + } +} diff --git a/crates/weaver_resolved_schema/src/v2/span.rs b/crates/weaver_resolved_schema/src/v2/span.rs new file mode 100644 index 000000000..ddd190aea --- /dev/null +++ b/crates/weaver_resolved_schema/src/v2/span.rs @@ -0,0 +1,79 @@ +//! Span related definitions structs. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use weaver_semconv::{ + attribute::RequirementLevel, + group::SpanKindSpec, + v2::{signal_id::SignalId, span::SpanName, CommonFields}, +}; + +use crate::v2::attribute::AttributeRef; + +/// The definition of a Span signal. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Span { + /// The type of the Span. This denotes the identity + /// of the "shape" of this span, and must be unique. + pub r#type: SignalId, + /// Specifies the kind of the span. + pub kind: SpanKindSpec, + /// The name pattern for the span. + pub name: SpanName, + // TODO - Should we split attributes into "sampling_relevant" and "other" groups here? + /// List of attributes that belong to this span. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + + // TODO - Should Entity Associations be "strong" links? + /// Which entities this span should be associated with. + /// + /// This list is an "any of" list, where a span may be associated with one or more entities, but should + /// be associated with at least one in this list. + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub entity_associations: Vec, + + /// Common fields (like brief, note, annotations). + #[serde(flatten)] + pub common: CommonFields, +} + +/// A special type of reference to attributes that remembers span-specicific information. +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SpanAttributeRef { + /// Reference, by index, to the attribute catalog. + pub base: AttributeRef, + /// Specifies if the attribute is mandatory. Can be "required", + /// "conditionally_required", "recommended" or "opt_in". When omitted, + /// the attribute is "recommended". When set to + /// "conditionally_required", the string provided as MUST + /// specify the conditions under which the attribute is required. + pub requirement_level: RequirementLevel, + /// Specifies if the attribute is (especially) relevant for sampling + /// and thus should be set at span start. It defaults to false. + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_relevant: Option, +} + +/// A refinement of a span, for use in code-gen or specific library application. +/// +/// A refinement represents a "view" of a Span that is highly optimised for a particular implementation. +/// e.g. for HTTP spans, there may be a refinement that provides only the necessary information for dealing with Java's HTTP +/// client library, and drops optional or extraneous information from the underlying http span. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema)] +pub struct SpanRefinement { + /// The identity of the refinement + pub id: SignalId, + + // TODO - This is a lazy way of doing this. We use `type` to refer + // to the underlying span definition, but override all fields here. + // We probably should copy-paste all the "span" attributes here + // including the `ty` + /// The definition of the span refinement. + #[serde(flatten)] + pub span: Span, +} diff --git a/crates/weaver_resolver/data/registry-test-11-prefix-refs-extends/expected-registry.json b/crates/weaver_resolver/data/registry-test-11-prefix-refs-extends/expected-registry.json index f97800f7a..b1b613ff0 100644 --- a/crates/weaver_resolver/data/registry-test-11-prefix-refs-extends/expected-registry.json +++ b/crates/weaver_resolver/data/registry-test-11-prefix-refs-extends/expected-registry.json @@ -129,6 +129,7 @@ "registry_id": "default", "path": "data/registry-test-11-prefix-refs-extends/registry/usage2.yaml" }, + "extends_group": "usage", "attributes": { "client.geo.lat": { "source_group": "registry.client", diff --git a/crates/weaver_resolver/data/registry-test-3-extends/expected-registry.json b/crates/weaver_resolver/data/registry-test-3-extends/expected-registry.json index 4e2ad6bb2..6644307bb 100644 --- a/crates/weaver_resolver/data/registry-test-3-extends/expected-registry.json +++ b/crates/weaver_resolver/data/registry-test-3-extends/expected-registry.json @@ -100,6 +100,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/http-common.yaml" }, + "extends_group": "attributes.http.common", "attributes": { "error.type": { "source_group": "registry.error", @@ -220,6 +221,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/http-common.yaml" }, + "extends_group": "attributes.http.common", "attributes": { "error.type": { "source_group": "registry.error", @@ -443,6 +445,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/metrics-messaging.yaml" }, + "extends_group": "messaging.attributes.common", "attributes": { "error.type": { "source_group": "registry.error", @@ -565,6 +568,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/metrics-messaging.yaml" }, + "extends_group": "metric.messaging.attributes", "attributes": { "error.type": { "source_group": "registry.error", @@ -687,6 +691,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/metrics-messaging.yaml" }, + "extends_group": "metric.messaging.attributes", "attributes": { "error.type": { "source_group": "registry.error", @@ -809,6 +814,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/metrics-messaging.yaml" }, + "extends_group": "metric.messaging.attributes", "attributes": { "error.type": { "source_group": "registry.error", @@ -931,6 +937,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/metrics-messaging.yaml" }, + "extends_group": "metric.messaging.attributes", "attributes": { "error.type": { "source_group": "registry.error", @@ -1053,6 +1060,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/metrics-messaging.yaml" }, + "extends_group": "metric.messaging.attributes", "attributes": { "error.type": { "source_group": "registry.error", @@ -1175,6 +1183,7 @@ "registry_id": "default", "path": "data/registry-test-3-extends/registry/metrics-messaging.yaml" }, + "extends_group": "metric.messaging.attributes", "attributes": { "error.type": { "source_group": "registry.error", diff --git a/crates/weaver_resolver/data/registry-test-7-spans/expected-registry.json b/crates/weaver_resolver/data/registry-test-7-spans/expected-registry.json index 2c89fc942..a1be14868 100644 --- a/crates/weaver_resolver/data/registry-test-7-spans/expected-registry.json +++ b/crates/weaver_resolver/data/registry-test-7-spans/expected-registry.json @@ -381,6 +381,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.connection_string": { "source_group": "registry.db", @@ -613,6 +614,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.cassandra.consistency_level": { "source_group": "registry.db", @@ -914,6 +916,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.connection_string": { "source_group": "registry.db", @@ -1126,6 +1129,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.connection_string": { "source_group": "registry.db", @@ -1339,6 +1343,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.connection_string": { "source_group": "registry.db", @@ -1565,6 +1570,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.connection_string": { "source_group": "registry.db", @@ -1795,6 +1801,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.connection_string": { "source_group": "registry.db", @@ -2073,6 +2080,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.connection_string": { "source_group": "registry.db", @@ -2307,6 +2315,7 @@ "registry_id": "default", "path": "data/registry-test-7-spans/registry/trace-database.yaml" }, + "extends_group": "db", "attributes": { "db.connection_string": { "source_group": "registry.db", diff --git a/crates/weaver_resolver/data/registry-test-8-http/expected-registry.json b/crates/weaver_resolver/data/registry-test-8-http/expected-registry.json index 3ed49a878..a9cd3fac4 100644 --- a/crates/weaver_resolver/data/registry-test-8-http/expected-registry.json +++ b/crates/weaver_resolver/data/registry-test-8-http/expected-registry.json @@ -42,6 +42,7 @@ "registry_id": "default", "path": "data/registry-test-8-http/registry/http-common.yaml" }, + "extends_group": "attributes.http.common", "attributes": { "network.protocol.name": { "source_group": "registry.network", @@ -83,6 +84,7 @@ "registry_id": "default", "path": "data/registry-test-8-http/registry/http.yaml" }, + "extends_group": "attributes.http.server", "attributes": { "network.protocol.name": { "source_group": "registry.network", @@ -128,6 +130,7 @@ "registry_id": "default", "path": "data/registry-test-8-http/registry/http.yaml" }, + "extends_group": "metric_attributes.http.server", "attributes": { "network.protocol.name": { "source_group": "registry.network", diff --git a/crates/weaver_resolver/data/registry-test-9-metric-extends/expected-registry.json b/crates/weaver_resolver/data/registry-test-9-metric-extends/expected-registry.json index 00b25cca7..ab84d8316 100644 --- a/crates/weaver_resolver/data/registry-test-9-metric-extends/expected-registry.json +++ b/crates/weaver_resolver/data/registry-test-9-metric-extends/expected-registry.json @@ -33,6 +33,7 @@ "registry_id": "default", "path": "data/registry-test-9-metric-extends/registry/jvm-metrics.yaml" }, + "extends_group": "attributes.jvm.memory", "attributes": { "jvm.memory.pool.name": { "source_group": "attributes.jvm.memory", diff --git a/crates/weaver_resolver/data/registry-test-lineage-1/expected-registry.json b/crates/weaver_resolver/data/registry-test-lineage-1/expected-registry.json index 690eda330..f2f7e01ea 100644 --- a/crates/weaver_resolver/data/registry-test-lineage-1/expected-registry.json +++ b/crates/weaver_resolver/data/registry-test-lineage-1/expected-registry.json @@ -13,6 +13,7 @@ "registry_id": "default", "path": "data/registry-test-lineage-1/registry/groups.yaml" }, + "extends_group": "intermediate.level", "attributes": { "server.port": { "source_group": "intermediate.level", @@ -39,6 +40,7 @@ "registry_id": "default", "path": "data/registry-test-lineage-1/registry/groups.yaml" }, + "extends_group": "base.level", "attributes": { "server.port": { "source_group": "base.level", diff --git a/crates/weaver_resolver/data/registry-test-lineage-2/expected-registry.json b/crates/weaver_resolver/data/registry-test-lineage-2/expected-registry.json index d117280f2..70d6dd38a 100644 --- a/crates/weaver_resolver/data/registry-test-lineage-2/expected-registry.json +++ b/crates/weaver_resolver/data/registry-test-lineage-2/expected-registry.json @@ -44,6 +44,7 @@ "registry_id": "default", "path": "data/registry-test-lineage-2/registry/groups.yaml" }, + "extends_group": "base.level", "attributes": { "server.port": { "source_group": "base.level", @@ -76,6 +77,7 @@ "registry_id": "default", "path": "data/registry-test-lineage-2/registry/groups.yaml" }, + "extends_group": "intermediate.level", "attributes": { "network.protocol.name": { "source_group": "registry.xyz", diff --git a/crates/weaver_resolver/src/registry.rs b/crates/weaver_resolver/src/registry.rs index 4a2639672..eee09f1ac 100644 --- a/crates/weaver_resolver/src/registry.rs +++ b/crates/weaver_resolver/src/registry.rs @@ -427,6 +427,7 @@ fn group_from_spec(group: GroupSpecWithProvenance) -> UnresolvedGroup { body: group.spec.body, annotations: group.spec.annotations, entity_associations: group.spec.entity_associations, + visibility: group.spec.visibility.clone(), }, attributes: attrs, provenance: group.provenance, @@ -590,6 +591,9 @@ fn resolve_extends_references(ureg: &mut UnresolvedRegistry) -> Result<(), Error vec![(extends, attrs)], unresolved_group.group.lineage.as_mut(), ); + if let Some(lineage) = unresolved_group.group.lineage.as_mut() { + lineage.extends(extends); + } add_resolved_group_to_index( &mut group_index, unresolved_group, @@ -625,6 +629,12 @@ fn resolve_extends_references(ureg: &mut UnresolvedRegistry) -> Result<(), Error } } _ = attrs_by_group.insert(include_group.clone(), attrs); + + // We'll need to reverse engineer if it was a private group later in V2 mapping. + if let Some(lineage) = unresolved_group.group.lineage.as_mut() { + // update lineage so we know a group was included. + lineage.includes_group(include_group); + } } else { errors.push(Error::UnresolvedExtendsRef { group_id: unresolved_group.group.id.clone(), diff --git a/crates/weaver_semconv/src/v2/attribute.rs b/crates/weaver_semconv/src/v2/attribute.rs index ccde647e2..a3bf5dbf1 100644 --- a/crates/weaver_semconv/src/v2/attribute.rs +++ b/crates/weaver_semconv/src/v2/attribute.rs @@ -11,7 +11,7 @@ use crate::{ attribute::{AttributeRole, AttributeSpec, AttributeType, Examples, RequirementLevel}, deprecated::Deprecated, stability::Stability, - v2::CommonFields, + v2::{signal_id::SignalId, CommonFields}, YamlValue, }; @@ -158,7 +158,7 @@ impl AttributeDef { #[serde(deny_unknown_fields)] pub struct GroupRef { /// Reference an existing attribute group by id. - pub ref_group: String, + pub ref_group: SignalId, } /// A reference to either an attribute or an attribute group. @@ -185,7 +185,7 @@ pub fn split_attributes_and_groups( AttributeOrGroupRef::Attribute(attr_ref) => { attributes.push(attr_ref.into_v1_attribute()); } - AttributeOrGroupRef::Group(group_ref) => groups.push(group_ref.ref_group), + AttributeOrGroupRef::Group(group_ref) => groups.push(group_ref.ref_group.into_v1()), } } diff --git a/crates/weaver_semconv/src/v2/mod.rs b/crates/weaver_semconv/src/v2/mod.rs index 95c175f4b..4e3362f91 100644 --- a/crates/weaver_semconv/src/v2/mod.rs +++ b/crates/weaver_semconv/src/v2/mod.rs @@ -28,7 +28,7 @@ pub mod signal_id; pub mod span; /// Common fields we want on all major components of semantic conventions. -#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, PartialEq, Hash, Eq)] #[serde(deny_unknown_fields)] pub struct CommonFields { /// A brief description of the attribute or signal. diff --git a/crates/weaver_semconv/src/v2/signal_id.rs b/crates/weaver_semconv/src/v2/signal_id.rs index c7c15ee9f..c5502e5bc 100644 --- a/crates/weaver_semconv/src/v2/signal_id.rs +++ b/crates/weaver_semconv/src/v2/signal_id.rs @@ -7,7 +7,7 @@ use std::{fmt, ops::Deref}; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; -#[derive(Serialize, JsonSchema, Clone, Debug)] +#[derive(Serialize, JsonSchema, Clone, Debug, PartialEq)] /// An identifier for a signal. Should be `.` separated namespaces and names. pub struct SignalId(String); @@ -19,6 +19,12 @@ impl SignalId { } } +impl From for SignalId { + fn from(value: String) -> Self { + SignalId(value) + } +} + // Allow `&SignalId` to be used for getting `&str`. impl Deref for SignalId { type Target = str; diff --git a/crates/weaver_semconv/src/v2/span.rs b/crates/weaver_semconv/src/v2/span.rs index 86ec5c9e6..1e3d93b7a 100644 --- a/crates/weaver_semconv/src/v2/span.rs +++ b/crates/weaver_semconv/src/v2/span.rs @@ -52,11 +52,7 @@ pub fn split_span_attributes_and_groups( (attribute_refs, groups) } -/// A group defines an attribute group, an entity, or a signal. -/// Supported group types are: `attribute_group`, `span`, `event`, `metric`, `entity`, `scope`. -/// Mandatory fields are: `id` and `brief`. -/// -/// Note: The `resource` type is no longer used and is an alias for `entity`. +/// Defines a new Span signal. #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] #[serde(deny_unknown_fields)] pub struct Span { @@ -117,7 +113,7 @@ impl Span { } /// Specification of the span name. -#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, PartialEq)] #[serde(deny_unknown_fields)] #[serde(rename_all = "snake_case")] pub struct SpanName { diff --git a/src/registry/generate.rs b/src/registry/generate.rs index 4b4a00b74..618ba8cd4 100644 --- a/src/registry/generate.rs +++ b/src/registry/generate.rs @@ -15,7 +15,7 @@ use weaver_forge::file_loader::{FileLoader, FileSystemFileLoader}; use weaver_forge::{OutputDirective, TemplateEngine}; use crate::registry::{Error, PolicyArgs, RegistryArgs}; -use crate::util::prepare_main_registry; +use crate::util::{prepare_main_registry, prepare_main_registry_v2}; use crate::{DiagnosticArgs, ExitDirectives}; use weaver_common::vdir::VirtualDirectory; use weaver_common::vdir::VirtualDirectoryPath; @@ -64,6 +64,10 @@ pub struct RegistryGenerateArgs { #[arg(long, default_value = "false")] pub future: bool, + /// Enable the V2 output for generating templates. + #[arg(long, default_value = "false")] + pub v2: bool, + /// Parameters to specify the diagnostic format. #[command(flatten)] pub diagnostic: DiagnosticArgs, @@ -91,8 +95,17 @@ pub(crate) fn command(args: &RegistryGenerateArgs) -> Result Result { + engine.generate(®istry, args.output.as_path(), &OutputDirective::File)?; + } + None => engine.generate(&v1, args.output.as_path(), &OutputDirective::File)?, + } if !diag_msgs.is_empty() { return Err(diag_msgs); @@ -176,9 +190,10 @@ pub(crate) fn generate_params_shared( #[cfg(test)] mod tests { - use std::path::PathBuf; + use std::path::{Path, PathBuf}; use tempdir::TempDir; + use weaver_diff::diff_dir; use crate::cli::{Cli, Commands}; use crate::registry::generate::RegistryGenerateArgs; @@ -218,6 +233,7 @@ mod tests { display_policy_coverage: false, }, future: false, + v2: false, diagnostic: Default::default(), }), })), @@ -296,6 +312,7 @@ mod tests { display_policy_coverage: false, }, future: false, + v2: false, diagnostic: Default::default(), }), })), @@ -345,6 +362,7 @@ mod tests { display_policy_coverage: false, }, future: false, + v2: false, diagnostic: Default::default(), }), })), @@ -453,6 +471,7 @@ mod tests { display_policy_coverage: false, }, future: false, + v2: false, diagnostic: Default::default(), }), })), @@ -493,4 +512,54 @@ mod tests { ); } } + + #[test] + fn test_registry_generate_v2() { + let temp_output = Path::new("tests/v2_forge/observed_output"); + + // Delete all the files in the observed_output/target directory + // before generating the new files. + std::fs::remove_dir_all(temp_output).unwrap_or_default(); + + let cli = Cli { + debug: 0, + quiet: false, + future: false, + command: Some(Commands::Registry(RegistryCommand { + command: RegistrySubCommand::Generate(RegistryGenerateArgs { + target: "markdown".to_owned(), + output: temp_output.to_path_buf(), + templates: VirtualDirectoryPath::LocalFolder { + path: "tests/v2_forge/templates/".to_owned(), + }, + config: None, + param: None, + params: None, + registry: RegistryArgs { + registry: VirtualDirectoryPath::LocalFolder { + path: "tests/v2_forge/model/".to_owned(), + }, + follow_symlinks: false, + include_unreferenced: false, + }, + policy: PolicyArgs { + policies: vec![], + skip_policies: true, + display_policy_coverage: false, + }, + future: false, + v2: true, + diagnostic: Default::default(), + }), + })), + }; + + let exit_directive = run_command(&cli); + // The command should succeed. + assert_eq!(exit_directive.exit_code, 0); + + // validate expected = observed. + let expected_output = Path::new("tests/v2_forge/expected_output"); + assert!(diff_dir(expected_output, temp_output).unwrap()); + } } diff --git a/src/registry/resolve.rs b/src/registry/resolve.rs index 45f90516c..643e431db 100644 --- a/src/registry/resolve.rs +++ b/src/registry/resolve.rs @@ -11,7 +11,7 @@ use weaver_common::diagnostic::DiagnosticMessages; use crate::format::{apply_format, Format}; use crate::registry::{PolicyArgs, RegistryArgs}; -use crate::util::prepare_main_registry; +use crate::util::{prepare_main_registry, prepare_main_registry_v2}; use crate::{DiagnosticArgs, ExitDirectives}; /// Parameters for the `registry resolve` sub-command @@ -39,6 +39,11 @@ pub struct RegistryResolveArgs { #[arg(short, long, default_value = "yaml")] format: Format, + // TODO - Figure out long term plan for versions here. + /// Whether or not to output version 2 of the schema. + #[arg(long, default_value = "false")] + v2: bool, + /// Policy parameters #[command(flatten)] policy: PolicyArgs, @@ -54,25 +59,50 @@ pub(crate) fn command(args: &RegistryResolveArgs) -> Result Result< + ( + ResolvedRegistry, + weaver_forge::v2::registry::ForgeResolvedRegistry, + Option, + ), + DiagnosticMessages, +> { + let registry_path = ®istry_args.registry; + + let main_registry_repo = RegistryRepo::try_new("main", registry_path)?; + + // Load the semantic convention specs + let main_semconv_specs = load_semconv_specs(&main_registry_repo, registry_args.follow_symlinks) + .capture_non_fatal_errors(diag_msgs)?; + + // Optionally init policy engine + let mut policy_engine = if !policy_args.skip_policies { + // Create and hold all VirtualDirectory instances to keep them from being dropped + let policy_vdirs: Vec = policy_args + .policies + .iter() + .map(|path| { + VirtualDirectory::try_new(path).map_err(|e| { + DiagnosticMessages::from_error(weaver_common::Error::InvalidVirtualDirectory { + path: path.to_string(), + error: e.to_string(), + }) + }) + }) + .collect::>()?; + + // Extract paths from VirtualDirectory instances + let policy_paths: Vec = policy_vdirs + .iter() + .map(|vdir| vdir.path().to_owned()) + .collect(); + + Some(init_policy_engine( + &main_registry_repo, + &policy_paths, + policy_args.display_policy_coverage, + )?) + } else { + None + }; + + // Check pre-resolution policies + if let Some(engine) = policy_engine.as_ref() { + check_policy(engine, &main_semconv_specs) + .inspect(|_, violations| { + if let Some(violations) = violations { + log_success(format!( + "All `before_resolution` policies checked ({} violations found)", + violations.len() + )); + } else { + log_success("No `before_resolution` policy violation"); + } + }) + .capture_non_fatal_errors(diag_msgs)?; + } + + // Resolve the main registry + let mut main_registry = + SemConvRegistry::from_semconv_specs(&main_registry_repo, main_semconv_specs)?; + // Resolve the semantic convention specifications. + // If there are any resolution errors, they should be captured into the ongoing list of + // diagnostic messages and returned immediately because there is no point in continuing + // as the resolution is a prerequisite for the next stages. + let main_resolved_schema = + resolve_semconv_specs(&mut main_registry, registry_args.include_unreferenced) + .capture_non_fatal_errors(diag_msgs)?; + + // This creates the template/json friendly registry. + let main_resolved_registry = ResolvedRegistry::try_from_resolved_registry( + &main_resolved_schema.registry, + main_resolved_schema.catalog(), + ) + .combine_diag_msgs_with(diag_msgs)?; + + // Check post-resolution policies + if let Some(engine) = policy_engine.as_mut() { + check_policy_stage::( + engine, + PolicyStage::AfterResolution, + main_registry_repo.registry_path_repr(), + &main_resolved_registry, + &[], + ) + .inspect(|_, violations| { + if let Some(violations) = violations { + log_success(format!( + "All `after_resolution` policies checked ({} violations found)", + violations.len() + )); + } else { + log_success("No `after_resolution` policy violation"); + } + }) + .capture_non_fatal_errors(diag_msgs)?; + } + + // TODO - fix error passing here so original error is diagnostic. + let v2_schema: weaver_resolved_schema::v2::ResolvedTelemetrySchema = main_resolved_schema + .try_into() + .map_err(|e: weaver_resolved_schema::error::Error| { + weaver_forge::error::Error::TemplateEngineError { + error: e.to_string(), + } + })?; + let v2_resolved_registry = + weaver_forge::v2::registry::ForgeResolvedRegistry::try_from_resolved_schema(v2_schema)?; + Ok((main_resolved_registry, v2_resolved_registry, policy_engine)) +} diff --git a/tests/v2_forge/expected_output/registry.md b/tests/v2_forge/expected_output/registry.md new file mode 100644 index 000000000..d8e90270c --- /dev/null +++ b/tests/v2_forge/expected_output/registry.md @@ -0,0 +1,27 @@ +# Registry + +## Attributes + +- `attr2`: Another attribute + +- `my.attr`: A test attribute + +## Attribute Groups + +- `my`: test group + +## Events + +- `my.event`: A test event + +## Entities + +- `my.entity`: A test entity + +## Metrics + +- `my.metric`: A count of something. + +## Spans + +- `my.span`: A test span diff --git a/tests/v2_forge/model/test.yaml b/tests/v2_forge/model/test.yaml new file mode 100644 index 000000000..dddaa9832 --- /dev/null +++ b/tests/v2_forge/model/test.yaml @@ -0,0 +1,48 @@ +version: "2" +attributes: + - key: my.attr + type: string + brief: A test attribute + stability: stable + - key: attr2 + type: int + brief: Another attribute + stability: stable +attribute_groups: + - id: my + brief: test group + stability: stable + attributes: + - ref: my.attr + visibility: public +events: + - name: my.event + brief: A test event + stability: stable + attributes: + - ref: my.attr +entities: + - type: my.entity + identity: + - ref: my.attr + description: + - ref: attr2 + brief: A test entity + stability: stable +metrics: + - name: my.metric + brief: A count of something. + instrument: counter + unit: "{1}" + stability: stable + attributes: + - ref: my.attr +spans: + - type: my.span + name: + note: Some name pattern + kind: client + attributes: + - ref: my.attr + brief: A test span + stability: stable diff --git a/tests/v2_forge/templates/markdown/registry.md.j2 b/tests/v2_forge/templates/markdown/registry.md.j2 new file mode 100644 index 000000000..e62b32022 --- /dev/null +++ b/tests/v2_forge/templates/markdown/registry.md.j2 @@ -0,0 +1,26 @@ +# Registry + +## Attributes +{% for attr in ctx.attributes %} +- `{{attr.key}}`: {{attr.brief}} +{% endfor %} +## Attribute Groups +{% for g in ctx.attribute_groups %} +- `{{g.id}}`: {{g.brief}} +{% endfor %} +## Events +{% for e in ctx.signals.events %} +- `{{e.name}}`: {{e.brief}} +{% endfor %} +## Entities +{% for e in ctx.signals.entities %} +- `{{e.type}}`: {{e.brief}} +{% endfor %} +## Metrics +{% for m in ctx.signals.metrics %} +- `{{m.name}}`: {{m.brief}} +{% endfor %} +## Spans +{% for s in ctx.signals.spans %} +- `{{s.type}}`: {{s.brief}} +{% endfor %} \ No newline at end of file diff --git a/tests/v2_forge/templates/markdown/weaver.yaml b/tests/v2_forge/templates/markdown/weaver.yaml new file mode 100644 index 000000000..c0e7fbe32 --- /dev/null +++ b/tests/v2_forge/templates/markdown/weaver.yaml @@ -0,0 +1,6 @@ +templates: + - template: "registry.md.j2" + filter: "." + application_mode: single + file_name: registry.md + # TODO - application_mode: each will be broken given the split, we may need new application modes. \ No newline at end of file