[pull] main from guacsec:main

.gitignore

@@ -20,3 +20,4 @@
 .ai_history.txt
 /vendor/
 /vendor.toml
+/.serena

common/src/db/limiter.rs

@@ -4,7 +4,8 @@ use crate::{
 };
 use sea_orm::{
     ConnectionTrait, DbErr, EntityTrait, FromQueryResult, Paginator, PaginatorTrait, QuerySelect,
-    Select, SelectModel, SelectTwo, SelectTwoModel, Selector, SelectorTrait,
+    Select, SelectModel, SelectThree, SelectThreeModel, SelectTwo, SelectTwoModel, Selector,
+    SelectorTrait,
 };
 use std::num::NonZeroU64;
 use tracing::instrument;
@@ -207,3 +208,36 @@ where
         }
     }
 }
+
+impl<'db, C, M1, M2, M3, E1, E2, E3> LimiterTrait<'db, C> for SelectThree<E1, E2, E3>
+where
+    C: ConnectionTrait,
+    E1: EntityTrait<Model = M1>,
+    E2: EntityTrait<Model = M2>,
+    E3: EntityTrait<Model = M3>,
+    M1: FromQueryResult + Sized + Send + Sync + 'db,
+    M2: FromQueryResult + Sized + Send + Sync + 'db,
+    M3: FromQueryResult + Sized + Send + Sync + 'db,
+{
+    type FetchSelector = SelectThreeModel<M1, M2, M3>;
+    type CountSelector = SelectThreeModel<M1, M2, M3>;
+
+    fn limiting(
+        self,
+        db: &'db C,
+        offset: u64,
+        limit: u64,
+    ) -> Limiter<'db, C, Self::FetchSelector, Self::CountSelector> {
+        let selector = self
+            .clone()
+            .limit(NonZeroU64::new(limit).map(|limit| limit.get()))
+            .offset(NonZeroU64::new(offset).map(|offset| offset.get()))
+            .into_model();
+
+        Limiter {
+            db,
+            paginator: self.paginate(db, 1),
+            selector,
+        }
+    }
+}

common/src/db/mod.rs

@@ -21,6 +21,7 @@ use std::{
     fmt::Display,
     ops::{Deref, DerefMut},
     pin::Pin,
+    str::FromStr,
     time::Duration,
 };
 use tracing::instrument;
@@ -87,7 +88,15 @@ impl Database {
         let mut opt = ConnectOptions::new(url);
         opt.max_connections(database.max_conn);
         opt.min_connections(database.min_conn);
+
         opt.sqlx_logging_level(log::LevelFilter::Trace);
+        if let Some(threshold) = std::env::var("TRUSTD_SLOW_SQL_THRESHOLD")
+            .ok()
+            .and_then(|s| humantime::Duration::from_str(&s).ok())
+        {
+            opt.sqlx_logging(true);
+            opt.sqlx_slow_statements_logging_settings(log::LevelFilter::Warn, *threshold);
+        }
 
         opt.connect_timeout(Duration::from_secs(database.connect_timeout));
         opt.acquire_timeout(Duration::from_secs(database.acquire_timeout));

common/src/db/query/filtering.rs

@@ -1,7 +1,7 @@
 // SeaORM impls
 
 use super::{Columns, Error, IntoColumns, Query, Sort};
-use sea_orm::{EntityTrait, QueryFilter, QueryOrder, Select, SelectTwo};
+use sea_orm::{EntityTrait, QueryFilter, QueryOrder, Select, SelectThree, SelectTwo};
 
 /// Pass a Query instance for filtering
 pub trait Filtering<T: EntityTrait> {
@@ -51,3 +51,20 @@ where
         )
     }
 }
+
+/// SeaORM SelectThree
+impl<E, F, G> Filtering<E> for SelectThree<E, F, G>
+where
+    E: EntityTrait,
+    F: EntityTrait,
+    G: EntityTrait,
+{
+    fn filtering(self, search: Query) -> Result<Self, Error> {
+        self.filtering_with(
+            search,
+            Columns::from_entity::<E>()
+                .add_columns(Columns::from_entity::<F>())
+                .add_columns(Columns::from_entity::<G>()),
+        )
+    }
+}

common/src/id.rs

@@ -1,7 +1,7 @@
 use crate::purl::PurlErr;
 use hex::ToHex;
 use ring::digest::Digest;
-use sea_orm::{EntityTrait, QueryFilter, Select, UpdateMany};
+use sea_orm::{EntityTrait, QueryFilter, Select, SelectThree, SelectTwo, UpdateMany};
 use sea_query::Condition;
 use serde::{
     Deserialize, Deserializer, Serialize, Serializer,
@@ -65,6 +65,27 @@ where
     }
 }
 
+impl<E, F> TrySelectForId for SelectTwo<E, F>
+where
+    E: EntityTrait + TryFilterForId,
+    F: EntityTrait,
+{
+    fn try_filter(self, id: Id) -> Result<Self, IdError> {
+        Ok(self.filter(E::try_filter(id)?))
+    }
+}
+
+impl<E, F, G> TrySelectForId for SelectThree<E, F, G>
+where
+    E: EntityTrait + TryFilterForId,
+    F: EntityTrait,
+    G: EntityTrait,
+{
+    fn try_filter(self, id: Id) -> Result<Self, IdError> {
+        Ok(self.filter(E::try_filter(id)?))
+    }
+}
+
 impl<E> TrySelectForId for UpdateMany<E>
 where
     E: EntityTrait + TryFilterForId,

common/src/model.rs

@@ -78,6 +78,15 @@ pub struct PaginatedResults<R> {
     pub total: u64,
 }
 
+impl<T> Default for PaginatedResults<T> {
+    fn default() -> Self {
+        Self {
+            items: vec![],
+            total: 0,
+        }
+    }
+}
+
 impl<R> PaginatedResults<R> {
     /// Create a new paginated result
     pub async fn new<C, S1, S2>(

docs/env-vars.md

@@ -48,6 +48,7 @@
 | `TRUSTD_S3_BUCKET`                       | S3 bucket name                                                                      |                                         |
 | `TRUSTD_S3_REGION`                       | S3 region name                                                                      |                                         |
 | `TRUSTD_S3_SECRET_KEY`                   | S3 secret key                                                                       |                                         |
+| `TRUSTD_SLOW_SQL_THRESHOLD`              | Override threshold for slow SQL statements (humantime)                              | `1m`                                    |
 | `TRUSTD_STORAGE_FS_PATH`                 | Path for storage file system strategy                                               | `./.trustify/storage`                   |
 | `TRUSTD_STORAGE_STRATEGY`                | Specifies the storage strategy to use                                               | `File system`                           |
 | `UI_CLIENT_ID`                           | Client ID used by the UI                                                            | `frontend`                              |

migration/src/lib.rs

@@ -52,6 +52,8 @@ mod m0002100_analysis_perf_indexes;
 mod m0002110_license_query_performance;
 mod m0002120_normalize_expanded_license;
 mod m0002130_add_csaf_product_id;
+mod m0002140_p2p_right_index;
+mod m0002150_fix_advisory_labels_index;
 
 pub trait MigratorExt: Send {
     fn build_migrations() -> Migrations;
@@ -120,6 +122,8 @@ impl MigratorExt for Migrator {
             .normal(m0002110_license_query_performance::Migration)
             .normal(m0002120_normalize_expanded_license::Migration)
             .normal(m0002130_add_csaf_product_id::Migration)
+            .normal(m0002140_p2p_right_index::Migration)
+            .normal(m0002150_fix_advisory_labels_index::Migration)
     }
 }
 

migration/src/m0002140_p2p_right_index.rs

@@ -0,0 +1,53 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+#[allow(deprecated)]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .create_index(
+                Index::create()
+                    .if_not_exists()
+                    .table(PackageRelatesToPackage::Table)
+                    .name(Indexes::IdxPackageRelatesToPackageRightId.to_string())
+                    .col(PackageRelatesToPackage::SbomId)
+                    .col(PackageRelatesToPackage::Relationship)
+                    .col(PackageRelatesToPackage::RightNodeId)
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // Drop indexes in reverse order
+        manager
+            .drop_index(
+                Index::drop()
+                    .if_exists()
+                    .table(PackageRelatesToPackage::Table)
+                    .name(Indexes::IdxPackageRelatesToPackageRightId.to_string())
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum Indexes {
+    IdxPackageRelatesToPackageRightId,
+}
+
+#[derive(DeriveIden)]
+enum PackageRelatesToPackage {
+    Table,
+    SbomId,
+    Relationship,
+    RightNodeId,
+}

migration/src/m0002150_fix_advisory_labels_index.rs

@@ -0,0 +1,72 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+#[allow(deprecated)]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .drop_index(
+                Index::drop()
+                    .if_exists()
+                    .table(Sbom::Table)
+                    .name("advisory_labels_idx")
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_index(
+                Index::create()
+                    .if_not_exists()
+                    .table(Advisory::Table)
+                    .name("advisory_labels_idx")
+                    .col(Advisory::Labels)
+                    .index_type(IndexType::Custom(Alias::new("GIN").into_iden()))
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .drop_index(
+                Index::drop()
+                    .if_exists()
+                    .table(Advisory::Table)
+                    .name("advisory_labels_idx")
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_index(
+                Index::create()
+                    .if_not_exists()
+                    .table(Sbom::Table)
+                    .name("advisory_labels_idx")
+                    .col(Sbom::Labels)
+                    .index_type(IndexType::Custom(Alias::new("GIN").into_iden()))
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum Advisory {
+    Table,
+    Labels,
+}
+
+#[derive(DeriveIden)]
+enum Sbom {
+    Table,
+    Labels,
+}

modules/analysis/src/service/load/mod.rs

@@ -29,7 +29,7 @@ use std::{
 };
 use time::OffsetDateTime;
 use tokio::sync::oneshot;
-use tracing::{Level, instrument};
+use tracing::{Instrument, Level, info_span, instrument};
 use trustify_common::{
     cpe::Cpe as TrustifyCpe,
     db::query::{Columns, Filtering, IntoColumns},
@@ -497,6 +497,7 @@ impl InnerService {
                 // there is an operation in progress, await and return
 
                 return rx
+                    .instrument(info_span!("waiting for first loading"))
                     .await
                     // error awaiting
                     .map_err(|_| Error::Internal("failed to await loading operation".into()))?

modules/analysis/src/service/render/mod.rs

@@ -12,6 +12,7 @@ pub enum Renderer {
 }
 
 impl AnalysisService {
+    #[instrument(skip_all)]
     pub fn render(&self, graph: &PackageGraph, renderer: Renderer) -> Option<(String, String)> {
         match renderer {
             Renderer::Graphviz => self.walk(graph, graphviz::Renderer::new()),

modules/fundamental/src/advisory/endpoints/mod.rs

@@ -240,7 +240,7 @@ pub async fn upload(
     tag = "advisory",
     operation_id = "downloadAdvisory",
     params(
-        ("key" = Id, Path),
+        ("key" = Id, Path, description = "Identifier of the advisory, either `urn:uuid:<uuid>` or a digest e.g. `sha256:<hex>`"),
     ),
     responses(
         (status = 200, description = "Download a an advisory", body = inline(BinaryData)),

modules/fundamental/src/common/service.rs

@@ -2,9 +2,10 @@ use crate::{Error, common::LicenseRefMapping, source_document::model::SourceDocu
 use sea_orm::{ConnectionTrait, DbBackend, FromQueryResult, PaginatorTrait, Statement};
 use spdx_expression;
 use std::collections::BTreeMap;
+use tracing::instrument;
 use trustify_module_storage::service::{StorageBackend, StorageKey, dispatch::DispatchBackend};
 
-#[derive(Copy, Clone, Eq, PartialEq)]
+#[derive(Copy, Clone, Eq, PartialEq, Debug)]
 pub enum DocumentType {
     Advisory,
     Sbom,
@@ -13,6 +14,7 @@ pub enum DocumentType {
 /// Fetch all unique key/value labels matching the `filter_text` for all the `r#type` entities, i.e. `DocumentType::Advisory` or `DocumentType::Sbom`
 ///
 /// If limit=0 then all data will be fetched
+#[instrument(skip(connection), err(level=tracing::Level::INFO))]
 pub async fn fetch_labels<C: ConnectionTrait>(
     r#type: DocumentType,
     filter_text: String,