feat(views): added views and subviews for some KPI

 - number of eservices without tokens
 - number of clients without tokens
 - number of eservices without tokens in last N days
 - number of clients without tokens in last N days
 - ratio, for each consumer tenant, between subscribed eservices and used eservices
 - average delta time between administrative usage request and first JWT issued, for each eservice.
 - ratio between declared maximum total daily call for each eservice and sum of daily call promised for each agreement

Author: marco-vit-pagopa

views/domains/clients_without_tokens/R__domains_mv_00_client_without_token__raw_data.sql

@@ -0,0 +1,41 @@
+CREATE SCHEMA IF NOT EXISTS sub_views;
+
+GRANT USAGE ON SCHEMA sub_views TO GROUP readonly_group;
+
+DROP MATERIALIZED VIEW sub_views.mv_00_client_without_token__raw_data CASCADE;
+
+-- We need to know which client have generated token and which haven't.
+-- But we can't use lateral (left or right) join; due to a limitation 
+--  of redshift materilized views incremental refresh.
+-- As a workaround we generate a record for each active client and 
+--  one for each JWT of that clients; each record has usage_weight field that 
+--  can assume 0 or 1 value. Making sum of the usage_weight field as a result 
+--  we can assume that clients with total usage_weight equals 0 are clients 
+--  never used to detach some JWT.
+CREATE MATERIALIZED VIEW sub_views.mv_00_client_without_token__raw_data AUTO REFRESH YES AS 
+  select 
+    t.name as consumer_name,
+    c.id as client_id,
+    0 as usage_weight
+  from
+    domains.client c
+    join domains.tenant t on c.consumer_id = t.id
+  where
+      not coalesce( c.deleted, false )
+    and
+      not coalesce( t.deleted, false )
+  union all
+  select
+    t.name as consumer_name,
+    c.id as client_id,
+    1 as usage_weight
+  from
+    domains.client c
+    join domains.tenant t on c.consumer_id = t.id
+    join jwt.generated_token_audit jwt
+         on jwt.client_id = c.id 
+  where
+      not coalesce( c.deleted, false )
+    and
+      not coalesce( t.deleted, false )
+;

views/domains/clients_without_tokens/R__domains_mv_01_client_without_token__usage_count.sql

@@ -0,0 +1,18 @@
+CREATE SCHEMA IF NOT EXISTS sub_views;
+
+GRANT USAGE ON SCHEMA sub_views TO GROUP readonly_group;
+
+DROP MATERIALIZED VIEW sub_views.mv_01_client_without_token__usage_count CASCADE;
+
+-- number_of_usage is the number of detached JWT.
+CREATE MATERIALIZED VIEW sub_views.mv_01_client_without_token__usage_count AS
+select
+  consumer_name, 
+  client_id,
+  sum( usage_weight) as number_of_usage
+from
+  sub_views.mv_00_client_without_token__raw_data
+group by
+  consumer_name, 
+  client_id
+;

views/domains/clients_without_tokens/R__domains_mv_02_client_without_token__final.sql

@@ -0,0 +1,31 @@
+CREATE SCHEMA IF NOT EXISTS views;
+
+GRANT USAGE ON SCHEMA views TO GROUP readonly_group;
+GRANT USAGE ON SCHEMA views TO interop_analytics_quicksight_user;
+
+DROP MATERIALIZED VIEW views.mv_02_client_without_token__final CASCADE;
+
+CREATE MATERIALIZED VIEW views.mv_02_client_without_token__final AS
+select
+  consumer_name, 
+  count( client_id ) as client__count,
+  sum( case when number_of_usage = 0 then 1 else 0 end ) as client_not_used__count,
+  round(
+      100 
+    *
+      sum( case when number_of_usage = 0 then 1 else 0 end )
+    /
+      cast( count( client_id) as decimal )
+    ,
+    2
+  )
+   as client_not_used__percent
+from
+  sub_views.mv_01_client_without_token__usage_count
+group by
+  consumer_name
+;
+
+comment on view views_test.mv_02_client_without_token__final 
+is 'This view show, for each consumer tenant: how many active client are registered; \n how many of them have requested at least one token; the ratio in percentage format. '
+

views/domains/clients_without_tokens__last_n_days/R__domains_mv_00_client_not_used_after_ts__raw_data.sql

@@ -0,0 +1,42 @@
+CREATE SCHEMA IF NOT EXISTS sub_views;
+
+GRANT USAGE ON SCHEMA sub_views TO GROUP readonly_group;
+
+DROP MATERIALIZED VIEW sub_views.mv_00_client_not_used_after_ts__raw_data CASCADE;
+
+-- We need to know which client have generated token and which haven't.
+-- But we can't use lateral (left or right) join; due to a limitation 
+--  of redshift materilized views incremental refresh.
+-- As a workaround we generate a record for each active client and 
+--  one for each JWT of that clients; each record has issued_at field that 
+--  can assume -1 value for records not associated to any JWT. As a result clients 
+--  that have never detached any JWT have max(issued_at) == -1
+CREATE MATERIALIZED VIEW sub_views.mv_00_client_not_used_after_ts__raw_data AUTO REFRESH YES AS 
+select 
+  t.name as consumer_name,
+  c.id as client_id,
+  c."name" as client_name,
+  -1 as issued_at
+from
+  domains.client c
+  join domains.tenant t on t.id = c.consumer_id
+where
+    not coalesce( c.deleted, false )
+  and
+    not coalesce( t.deleted, false )
+union all
+select 
+  t.name as consumer_name,
+  c.id as client_id,
+  c."name" as client_name,
+  jwt.issued_at as issued_at
+from
+  domains.client c 
+  join domains.tenant t on t.id = c.consumer_id
+  join jwt.generated_token_audit jwt
+       on jwt.client_id = c.id
+where
+    not coalesce( c.deleted, false )
+  and
+    not coalesce( t.deleted, false )
+;

views/domains/clients_without_tokens__last_n_days/R__domains_mv_01_client_not_used_after_ts__last_issued_at.sql

@@ -0,0 +1,24 @@
+CREATE SCHEMA IF NOT EXISTS views;
+
+GRANT USAGE ON SCHEMA views TO GROUP readonly_group;
+GRANT USAGE ON SCHEMA views TO interop_analytics_quicksight_user;
+
+DROP MATERIALIZED VIEW IF EXISTS views.mv_01_client_not_used_after_ts__last_issued_at CASCADE;
+
+CREATE MATERIALIZED VIEW views.mv_01_client_not_used_after_ts__last_issued_at AS
+select
+  consumer_name,
+  client_id,
+  min(client_name) as client_name,
+  max(issued_at) as last_issued_at,
+  timestamp 'epoch' + ( max(issued_at) / 1000 ) * interval '1 second' as last_issued_at_ts
+from
+  sub_views.mv_00_client_not_used_after_ts__raw_data
+group by
+  consumer_name,
+  client_id  
+;
+
+COMMENT ON VIEW views.mv_01_client_not_used_after_ts__last_issued_at 
+is 'This view show, for each client, the epoch timestamp of last issued token. \n The last_issued_at field assume value -1 for clients that have never detachd a JWT.'
+;

views/domains/consumers_subscription_usage_ratio/R__domains_mv_00_not_used_subscribed_eservice__raw_data.sql

@@ -0,0 +1,38 @@
+CREATE SCHEMA IF NOT EXISTS sub_views;
+
+GRANT USAGE ON SCHEMA sub_views TO GROUP readonly_group;
+
+DROP MATERIALIZED VIEW sub_views.mv_00_not_used_subscribed_eservice__raw_data CASCADE;
+
+-- We need to know which eservice was authorized and which wasn't.
+-- But we can't use lateral (left or right) join; due to a limitation 
+--  of redshift materilized views incremental refresh.
+-- As a workaround we generate a record for each active eservice and 
+--  one for each JWT was authorizing that eservice; each record has required_jwt field that 
+--  can assume 0 or 1 value, 0 if the record is not related to any JWT. The result is
+--  that eservices with sum(required_jwt) == 0 are never used.
+CREATE MATERIALIZED VIEW sub_views.mv_00_not_used_subscribed_eservice__raw_data AUTO REFRESH YES AS
+select
+  t.name as consumer_name,
+  p.eservice_id,
+  0 as required_jwt
+from 
+  domains.purpose p
+  join domains.purpose_version pv on pv.purpose_id = p.id
+  join domains.tenant t on t.id = p.consumer_id
+where 
+  pv.state in ('Active')
+union all
+select
+  t.name as consumer_name,
+  p.eservice_id,
+  1 as required_jwt
+from 
+  domains.purpose p
+  join domains.purpose_version pv on pv.purpose_id = p.id
+  join domains.tenant t on t.id = p.consumer_id
+  join jwt.generated_token_audit jwt 
+       on jwt.purpose_version_id = pv.id
+where 
+    pv.state in ('Active')
+;

views/domains/consumers_subscription_usage_ratio/R__domains_mv_01_not_used_subscribed_eservice__usage_count.sql

@@ -0,0 +1,18 @@
+CREATE SCHEMA IF NOT EXISTS sub_views;
+
+GRANT USAGE ON SCHEMA sub_views TO GROUP readonly_group;
+
+DROP MATERIALIZED VIEW sub_views.mv_01_not_used_subscribed_eservice__usage_count CASCADE;
+
+-- For each eservice show the number of JWT that provide authorization for that eservice calls.
+CREATE MATERIALIZED VIEW sub_views.mv_01_not_used_subscribed_eservice__usage_count AS 
+select
+  consumer_name,
+  eservice_id,
+  sum(required_jwt) as how_many_required_jwt
+from
+  sub_views.mv_00_not_used_subscribed_eservice__raw_data 
+group by
+  consumer_name,
+  eservice_id
+;

views/domains/consumers_subscription_usage_ratio/R__domains_mv_02_not_used_subscribed_eservice__final.sql

@@ -0,0 +1,33 @@
+CREATE SCHEMA IF NOT EXISTS views;
+
+GRANT USAGE ON SCHEMA views TO GROUP readonly_group;
+GRANT USAGE ON SCHEMA views TO interop_analytics_quicksight_user;
+
+DROP MATERIALIZED VIEW IF EXISTS views.mv_02_not_used_subscribed_eservice__final CASCADE;
+
+CREATE MATERIALIZED VIEW views.mv_02_not_used_subscribed_eservice__final AS 
+select
+  consumer_name,
+  sum( case when how_many_required_jwt > 0 then 1 else 0 end ) as used_eservices,
+  count( * ) as subscribed_eservices,
+  round(
+      100
+    *
+      (
+          sum( case when how_many_required_jwt > 0 then 1 else 0 end ) 
+        / 
+          cast( count( * ) as decimal )
+      )
+    ,
+    2 -- keep two decimal digit
+  )
+   as used_eservices_percent
+from
+  sub_views.mv_01_not_used_subscribed_eservice__usage_count
+group by
+  consumer_name
+;
+
+COMMENT ON VIEW views.mv_02_not_used_subscribed_eservice__final
+is 'For each consumer show the number of subscribed eservices; \n the number of distinct eservices with, at least, one detached token;\n the ration as percentage.\n N.B.: tenant self invocations are included'
+;

views/domains/consumers_usage_request_first_token_delta/R__domains_mv_00_avg_first_token_delta_by_eservice__first_issued_token.sql

@@ -0,0 +1,33 @@
+CREATE SCHEMA IF NOT EXISTS sub_views;
+
+GRANT USAGE ON SCHEMA sub_views TO GROUP readonly_group;
+
+DROP MATERIALIZED VIEW sub_views.mv_00_avg_first_token_delta_by_eservice__first_issued_token CASCADE;
+
+CREATE MATERIALIZED VIEW sub_views.mv_00_avg_first_token_delta_by_eservice__first_issued_token AUTO REFRESH YES AS
+select
+  t.name as producer_name,
+  a.eservice_id,
+  a.id as agreement_id,
+  min( a_s.when ) as agreement_activation_ts,
+  min( jwt.issued_at_tz ) as first_jwt_issued_at_tz
+from 
+  domains.agreement a
+  join domains.agreement_stamp a_s on a_s.agreement_id = a.id
+  join domains.tenant t on t.id = a.producer_id
+  join jwt.generated_token_audit jwt on jwt.agreement_id = a.id
+where
+    a.state in ('Active', 'Suspended', 'Archived', 'MissingCertifiedAttributes' ) -- Keep every agreement that was "Active" fore some time.
+  and
+    a_s.kind = 'activation' -- Select the agreement activation timestamp
+  and
+    not coalesce( a.deleted, false)
+  and
+    not coalesce( a_s.deleted, false)
+  and
+    not coalesce( t.deleted, false)
+group by
+  t.name,
+  a.eservice_id,
+  a.id
+;

views/domains/consumers_usage_request_first_token_delta/R__domains_mv_01_avg_first_token_delta_by_eservice__final.sql

@@ -0,0 +1,30 @@
+CREATE SCHEMA IF NOT EXISTS views;
+
+GRANT USAGE ON SCHEMA views TO GROUP readonly_group;
+GRANT USAGE ON SCHEMA views TO interop_analytics_quicksight_user;
+
+DROP MATERIALIZED VIEW IF EXISTS views.mv_01_avg_first_token_delta_by_eservice__final CASCADE;
+
+CREATE MATERIALIZED VIEW views.mv_01_avg_first_token_delta_by_eservice__final AS 
+select
+  producer_name,
+  eservice_id,
+  avg(
+    datediff(
+      seconds,
+      agreement_activation_ts at time zone 'GMT',
+      first_jwt_issued_at_tz at time zone 'GMT'
+    )
+  ) as avg_first_token_delta_time__seconds,
+  count(*) as num_of_agreement
+from 
+  sub_views.mv_00_avg_first_token_delta_by_eservice__first_issued_token
+group by
+  producer_name,
+  eservice_id
+;
+
+
+COMMENT ON VIEW views.mv_01_avg_first_token_delta_by_eservice__final
+is 'This view show, for each eservice, the average delta time between agreement accepted time and fist detached JWT issued_at time.\n Self tenant invocation are included.'
+;