Feature/freshness updates (#140)

fivetran-joemarkiewicz · fivetran-avinash · fivetran-jamie · web-flow · commit 301e706c5171 · 2025-06-26T12:08:05.000-05:00
* Pre-release for fixing cast value issues

* PR review

* Update casting to string for sprint ids

* Update casting to string for sprint ids

* feature/freshness-updates

* pr fixes

* Update packages.yml

Co-authored-by: Jamie Rodriguez &lt;65564846+fivetran-jamie@users.noreply.github.com&gt;

---------

Co-authored-by: Avinash Kunnath &lt;avinash.kunnath@fivetran.com&gt;
Co-authored-by: Jamie Rodriguez &lt;65564846+fivetran-jamie@users.noreply.github.com&gt;
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -3,7 +3,7 @@ steps:
     key: "run-dbt-postgres"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -18,7 +18,7 @@ steps:
     key: "run_dbt_snowflake"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -35,7 +35,7 @@ steps:
     key: "run_dbt_bigquery"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -47,7 +47,7 @@ steps:
     key: "run_dbt_redshift"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -62,7 +62,7 @@ steps:
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
@@ -77,7 +77,7 @@ steps:
     key: "run_dbt_databricks_sql"
     plugins:
       - docker#v3.13.0:
-          image: "python:3.8"
+          image: "python:3.10.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -19,6 +19,7 @@ dbt deps
 
 if [ "$db" = "databricks-sql" ]; then
 dbt seed --vars '{jira_schema: jira_integrations_tests_sqlw}' --target "$db" --full-refresh
+dbt source freshness --vars '{jira_schema: jira_integrations_tests_sqlw}' --target "$db" || echo "...Only verifying freshness runs…"
 dbt compile --vars '{jira_schema: jira_integrations_tests_sqlw}' --target "$db"
 dbt run --vars '{jira_schema: jira_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt run --vars '{jira_schema: jira_integrations_tests_sqlw}' --target "$db"
@@ -29,6 +30,7 @@ dbt test --vars '{jira_schema: jira_integrations_tests_sqlw}' --target "$db"
 
 else
 dbt seed --target "$db" --full-refresh
+dbt source freshness --target "$db" || echo "...Only verifying freshness runs…"
 dbt compile --target "$db"
 dbt run --target "$db" --full-refresh
 dbt run --target "$db"
diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md
@@ -1,29 +1,35 @@
-## PR Overview
-**This PR will address the following Issue/Feature:**
+<!--
+Pre-Submission Reminders  
+Before marking this PR as "ready for review":
 
-**This PR will result in the following new package version:**
-<!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
+- `dbt run --full-refresh && dbt test`  
+- `dbt run` && `dbt test` (if incremental models are present)  
+- The related issue is linked, tagged, and appropriately assigned  
+- Documentation and version updates are included, if applicable  
+- `docs` have been regenerated (unless there are no code or YAML changes)  
+- BuildKite integration tests are passing
+-->
 
-**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:**
-<!--- Copy/paste the CHANGELOG for this version below. -->
+## PR Overview 
+**Package version introduced in this PR:** 
+ 
+**This PR addresses the following Issue/Feature(s):**
+<!-- Add Issue # or internal ticket reference -->
 
-## PR Checklist
-### Basic Validation
-Please acknowledge that you have successfully performed the following commands locally:
-- [ ] dbt run –full-refresh && dbt test
-- [ ] dbt run (if incremental models are present) && dbt test
+**Summary of changes:**  
+<!-- 1-2 sentences describing PR changes. -->
 
-Before marking this PR as "ready for review" the following have been applied:
-- [ ] The appropriate issue has been linked, tagged, and properly assigned
-- [ ] All necessary documentation and version upgrades have been applied
-- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
-- [ ] BuildKite integration tests are passing
-- [ ] Detailed validation steps have been provided below
 
-### Detailed Validation
-Please share any and all of your validation steps:
-<!--- Provide the steps you took to validate your changes below. -->
+### Submission Checklist  
+- [ ] Alignment meeting with the reviewer (if needed)  
+  - [ ] Timeline and validation requirements discussed  
+- [ ] Provide validation details:  
+  - [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests)
+  - [ ] **Testing Instructions:** Confirm the change addresses the issue(s)
+  - [ ] **Focus Areas:** Complex logic or queries that need extra attention  
 
-### If you had to summarize this PR in an emoji, which would it be?
-<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
-:dancer:
+### Changelog  
+<!-- Recommend drafting changelog notes, then refining via ChatGPT using:  
+"Draft a changelog entry based on the following notes." -->
+- [ ] Draft changelog for PR  
+- [ ] Final changelog for release review
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,54 @@
+# dbt_jira v0.21.0
+
+[PR #140](https://github.com/fivetran/dbt_jira/pull/140) includes the following updates:
+
+## Breaking Change for dbt Core < 1.9.6
+
+> *Note: This is not relevant to Fivetran Quickstart users.*
+
+Migrated `freshness` from a top-level source property to a source `config` in alignment with [recent updates](https://github.com/dbt-labs/dbt-core/issues/11506) from dbt Core ([Jira Source v0.9.0](https://github.com/fivetran/dbt_jira_source/releases/tag/v0.9.0)). This will resolve the following deprecation warning that users running dbt >= 1.9.6 may have received:
+
+```
+[WARNING]: Deprecated functionality
+Found `freshness` as a top-level property of `jira` in file
+`models/src_jira.yml`. The `freshness` top-level property should be moved
+into the `config` of `jira`.
+```
+
+**IMPORTANT:** Users running dbt Core < 1.9.6 will not be able to utilize freshness tests in this release or any subsequent releases, as older versions of dbt will not recognize freshness as a source `config` and therefore not run the tests.
+
+If you are using dbt Core < 1.9.6 and want to continue running Jira freshness tests, please elect **one** of the following options:
+  1. (Recommended) Upgrade to dbt Core >= 1.9.6
+  2. Do not upgrade your installed version of the `jira` package. Pin your dependency on v0.20.0 in your `packages.yml` file.
+  3. Utilize a dbt [override](https://docs.getdbt.com/reference/resource-properties/overrides) to overwrite the package's `jira` source and apply freshness via the previous release top-level property route. This will require you to copy and paste the entirety of the previous release `src_jira.yml` file and add an `overrides: jira_source` property.
+
+## Bug Fix
+- Changed casting of `sprint_id` to a string rather than an integer in `jira__daily_sprint_issue_history` to resolve casting errors.
+
+## Under the Hood
+- Updates to ensure integration tests use latest version of dbt.
+
+# dbt_jira v0.20.1-a2
+The latest version of [PR #138](https://github.com/fivetran/dbt_jira/pull/138) contains the following updates:
+
+## Bug Fix
+- Changed casting of `sprint_id` to a string rather than an integer in `jira__daily_sprint_issue_history` to resolve casting errors.
+
+## Under the Hood
+- Updated the package maintainer pull request template.
+- Updated `consistency_issue_enhanced` test to exclude columns that depend on calculations involving the current timestamp to remove failures.
+- Updated the sprint integrity tests to handle the new `sprint_id` casting.
+
+# dbt_jira v0.20.1-a1
+[PR #138](https://github.com/fivetran/dbt_jira/pull/138) contains the following updates:
+
+## Bug Fix
+- Moved casting of integer of `sprint_id` in `jira__daily_sprint_issue_history` into a later CTE to avoid casting errors where `field_value` is a string.
+
+## Under the Hood
+- Updated the package maintainer pull request template.
+- Updated `consistency_issue_enhanced` test to exclude columns that depend on calculations involving the current timestamp to remove failures.
+
 # dbt_jira v0.20.0
 This release includes the following updates.
 
diff --git a/README.md b/README.md
@@ -72,7 +72,7 @@ Include the following jira package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/jira
-    version: [">=0.20.0", "<0.21.0"]
+    version: [">=0.21.0", "<0.22.0"]
 
 ```
 ### Step 3: Define database and schema variables
@@ -188,7 +188,7 @@ This dbt package is dependent on the following dbt packages. These dependencies
 ```yml
 packages:
     - package: fivetran/jira_source
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=0.9.0", "<0.10.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'jira'
-version: '0.20.0'
+version: '0.21.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -2,10 +2,6 @@
 # HEY! This file is used in the dbt package integrations tests with Buildkite.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
-config:
-    send_anonymous_usage_stats: False
-    use_colors: True
-
 integration_tests:
   target: redshift
   outputs:
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'jira_integration_tests'
-version: '0.20.0'
+version: '0.21.0'
 config-version: 2
 profile: 'integration_tests'
 
@@ -86,3 +86,6 @@ seeds:
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']
+
+flags:
+  send_anonymous_usage_stats: False
diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt
@@ -5,4 +5,4 @@ dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
 dbt-databricks>=1.3.0,<2.0.0
-certifi==2025.1.31
+certifi==2025.1.31
diff --git a/integration_tests/tests/consistency/consistency_issue_enhanced.sql b/integration_tests/tests/consistency/consistency_issue_enhanced.sql
@@ -5,7 +5,7 @@
 ) }}
 
 {# Exclude columns that depend on calculations involving the current time in seconds or aggregate strings in a random order, as they will differ between runs. #}
-{% set exclude_columns = [] %}
+{% set exclude_columns = ['open_duration_seconds', 'any_assignment_duration_seconds', 'last_assignment_duration_seconds'] %}
 with prod as (
     select {{ dbt_utils.star(from=ref('jira__issue_enhanced'), except=exclude_columns) }}
     from {{ target.schema }}_jira_prod.jira__issue_enhanced
diff --git a/integration_tests/tests/integrity/vertical_integrity_daily_sprint_issue_history.sql b/integration_tests/tests/integrity/vertical_integrity_daily_sprint_issue_history.sql
@@ -63,6 +63,6 @@ select
     first_date_end
 from end_model
 full outer join source_model
-    on end_model.sprint_id = source_model.sprint_id
+    on end_model.sprint_id = cast(source_model.sprint_id as string)
     and end_model.issue_id = source_model.issue_id
 where first_date_source != first_date_end 
diff --git a/integration_tests/tests/integrity/vertical_integrity_sprint_enhanced.sql b/integration_tests/tests/integrity/vertical_integrity_sprint_enhanced.sql
@@ -60,5 +60,5 @@ select
     sprint_issues_end
 from end_model
 full outer join source_model
-    on end_model.sprint_id = source_model.sprint_id
+    on end_model.sprint_id = cast(source_model.sprint_id as string)
 where sprint_issues_source != sprint_issues_end 
diff --git a/models/jira__daily_sprint_issue_history.sql b/models/jira__daily_sprint_issue_history.sql
@@ -17,13 +17,14 @@ sprint_issue_pairing as (
 
     select
         issue_id,
-        cast(field_value as {{ dbt.type_int() }}) as sprint_id,
+        field_value as sprint_id,
         updated_at,
         is_active
     from {{ ref('int_jira__issue_multiselect_history') }}
     where field_name = 'sprint'
         and field_value is not null
 ),
+
 sprint_activity_window as (
 
     select
@@ -55,7 +56,7 @@ ranked_sprint_updates as (
         ) as row_num
     from sprint_issue_pairing
     left join sprint_activity_window
-        on sprint_activity_window.sprint_id = sprint_issue_pairing.sprint_id
+        on cast(sprint_activity_window.sprint_id as {{ dbt.type_string() }}) = sprint_issue_pairing.sprint_id
     left join daily_issue_field_history 
         on sprint_issue_pairing.issue_id = daily_issue_field_history.issue_id
         -- Ensure tracking starts at the correct earliest date
@@ -84,7 +85,7 @@ filtered_issue_sprint_history as (
     inner join {{ ref('jira__issue_enhanced') }} issue
         on ranked_sprint_updates.issue_id = issue.issue_id
     inner join {{ var('sprint') }} sprint
-        on ranked_sprint_updates.sprint_id = sprint.sprint_id
+        on ranked_sprint_updates.sprint_id = cast(sprint.sprint_id as {{ dbt.type_string() }})
     where row_num = 1 --Keep only the last update per sprint-issue-date_day
 ),
 
diff --git a/packages.yml b/packages.yml
@@ -1,3 +1,3 @@
 packages:
-- package: fivetran/jira_source
-  version: [">=0.8.0", "<0.9.0"]
+  - package: fivetran/jira_source
+    version: [">=0.9.0", "<0.10.0"]
