Merge pull request #377 from dbt-labs/epapineau/undocumented-sources

Epapineau/undocumented sources

Author: Elize Papineau

docs/rules.md

@@ -23,6 +23,8 @@ hide:
 |Testing                                       |[Test Coverage](../rules/testing/#test-coverage)                                                                       |`fct_test_coverage`|
 |Documentation                                 |[Undocumented Models](../rules/documentation/#undocumented-models)                                                     |`fct_undocumented_models`|
 |Documentation                                 |[Documentation Coverage](../rules/documentation/#documentation-coverage)                                               |`fct_documentation_coverage`|
+|Documentation                                 |[Undocumented Source Tables](../rules/documentation/#undocumented-source-tables)                                                     |`fct_undocumented_source_tables`|
+|Documentation                                 |[Undocumented Sources](../rules/documentation/#undocumented-sources)                                               |`fct_documentation_sources`|
 |Structure                                     |[Test Directories](../rules/structure/#test-directories)                                                               |`fct_test_directories`|
 |Structure                                     |[Model Naming Conventions](../rules/structure/#model-naming-conventions)                                               |`fct_model_naming_conventions`|
 |Structure                                     |[Source Directories](../rules/structure/#source-directories)                                                           |`fct_source_directories`|

docs/rules/documentation.md

@@ -46,3 +46,43 @@ function in the model's `.yml` entry.
 !!! note "Tip"
 
     We recommend that every model in your dbt project has at minimum a model-level description. This ensures that each model's purpose is clear to other developers and stakeholders when viewing the dbt docs site. Missing documentation should be addressed first for marts models, then for the rest of your project, to ensure that stakeholders in the organization can understand the data which is surfaced to them.
+
+## Undocumented Source Tables
+
+`fct_undocumented_source_tables` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/documentation/fct_undocumented_source_tables.sql)) lists every source table with no description configured.
+
+**Reason to Flag**
+
+Good documentation for your dbt sources will help contributors to your project understand how and when data is loaded into your warehouse.
+
+**How to Remediate**
+
+Apply a text [description](https://docs.getdbt.com/docs/building-a-dbt-project/documentation) in the table's `.yml` entry, or create a [docs block](https://docs.getdbt.com/docs/building-a-dbt-project/documentation#using-docs-blocks) in a markdown file, and use the `{{ doc() }}`
+function in the table's `.yml` entry.
+```
+sources:
+  - name: my_source
+    tables:
+      - name: my_table
+        description: This is the source table description
+```
+
+## Undocumented Sources
+
+`fct_undocumented_sources` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/documentation/fct_undocumented_sources.sql)) lists every source with no description configured.
+
+**Reason to Flag**
+
+Good documentation for your dbt sources will help contributors to your project understand how and when data is loaded into your warehouse.
+
+**How to Remediate**
+
+Apply a text [description](https://docs.getdbt.com/docs/building-a-dbt-project/documentation) in the source's `.yml` entry, or create a [docs block](https://docs.getdbt.com/docs/building-a-dbt-project/documentation#using-docs-blocks) in a markdown file, and use the `{{ doc() }}`
+function in the source's `.yml` entry.
+```
+sources:
+  - name: my_source
+    description: This is the source description
+    tables:
+      - name: my_table
+```

integration_tests/models/staging/source_1/source.yml

@@ -2,6 +2,7 @@ version: 2
 
 sources:
   - name: source_1
+    description: this is source 1.
     schema: real_schema
     # database: real_database
     tables:

integration_tests/seeds/docs/docs_seeds.yml

@@ -30,4 +30,22 @@ seeds:
             - staging_documentation_coverage_pct
             - intermediate_documentation_coverage_pct
             - marts_documentation_coverage_pct
-            - other_documentation_coverage_pct
+            - other_documentation_coverage_pct
+  
+  - name: test_fct_undocumented_source_tables
+    config:
+      tags:
+        - docs
+    tests:
+      - dbt_utils.equality:
+          name: equality_fct_undocumented_source_tables
+          compare_model: ref('fct_undocumented_source_tables')
+
+  - name: test_fct_undocumented_sources
+    config:
+      tags:
+        - docs
+    tests:
+      - dbt_utils.equality:
+          name: equality_fct_undocumented_sources
+          compare_model: ref('fct_undocumented_sources')

integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv

@@ -0,0 +1,6 @@
+resource_name
+source_1.table_4
+source_1.table_5
+source_2.table_3
+source_1.raw_table_5
+source_1.table_2

integration_tests/seeds/docs/test_fct_undocumented_sources.csv

@@ -0,0 +1,2 @@
+source_name
+source_2

models/marts/documentation/documentation.yml

@@ -16,5 +16,15 @@ models:
   - name: fct_undocumented_models
     description: >
       This model contains all models that do not have a description configured in a YML file. 
+    tests:
+      - is_empty
+  - name: fct_undocumented_source_tables
+    description: >
+      This model contains all source tables that do not have a description configured in a YML file. 
+    tests:
+      - is_empty
+  - name: fct_undocumented_sources
+    description: >
+      This model contains all sources that do not have a description configured in a YML file. 
     tests:
       - is_empty

models/marts/documentation/fct_undocumented_source_tables.sql

@@ -0,0 +1,21 @@
+with
+
+all_resources as (
+    select * from {{ ref('int_all_graph_resources') }}
+    where not is_excluded
+
+),
+
+final as (
+
+    select
+        resource_name
+
+    from all_resources
+    where not is_described and resource_type = 'source'
+
+)
+
+select * from final
+
+{{ filter_exceptions(model.name) }}

models/marts/documentation/fct_undocumented_sources.sql

@@ -0,0 +1,21 @@
+with
+
+all_resources as (
+    select * from {{ ref('int_all_graph_resources') }}
+    where not is_excluded
+
+),
+
+final as (
+
+    select distinct
+        source_name
+
+    from all_resources
+    where not is_source_described and resource_type = 'source'
+
+)
+
+select * from final
+
+{{ filter_exceptions(model.name) }}