Skip to content

Commit 9590c3a

Browse files
C00ldudeNoonanC00ldudeNoonan
authored
Merge pull request #22 from C00ldudeNoonan/ingestion-enhance
Ingestion enhance
2 parents 9c174eb + a7b779e commit 9590c3a

23 files changed

Lines changed: 1921 additions & 591 deletions

‎.sqlfluff‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
[sqlfluff]
2+
dialect = duckdb
3+
templater = dbt
4+
# Allow longer lines for complex SQL (default is 80)
5+
max_line_length = 120
6+
7+
[sqlfluff:templater:dbt]
8+
project_dir = dbt_project
9+
profiles_dir = dbt_project
10+
profile = econ_database
11+
12+
[sqlfluff:rules:layout.long_lines]
13+
# Ignore long lines in comments and certain clauses
14+
ignore_comment_lines = True
15+
ignore_comment_clauses = True
16+

‎dbt_project/.sqlfluff‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
[sqlfluff]
22
dialect = duckdb
3-
templater = dbt
3+
# Note: templater setting must be in root .sqlfluff file, not in subdirectories
44

5-
[sqlfluff:templater:dbt]
6-
project_dir = .
7-
profiles_dir = .
8-
profile = econ_database
5+
# Additional SQLFluff rules can be specified here if needed
96

‎dbt_project/models/analysis/leading_econ_return_indicator.sql‎

Lines changed: 148 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -7,132 +7,170 @@
77

88
-- Analysis of Economic Data Rate of Change vs Future Stock Returns
99
WITH economic_changes AS (
10-
SELECT
11-
bha.symbol,
12-
bha.month_date,
13-
bha.series_name,
14-
bha.category,
15-
fsm.category as economic_category,
16-
bha.value as current_econ_value,
17-
bha.monthly_avg_close,
18-
bha.pct_change_q1_forward,
19-
bha.pct_change_q2_forward,
20-
bha.pct_change_q3_forward,
21-
22-
-- Calculate month-over-month change in economic data
23-
LAG(bha.value, 1) OVER (PARTITION BY bha.symbol, bha.series_name ORDER BY bha.month_date) as prev_econ_value,
24-
25-
-- Calculate rate of change (month-over-month %)
26-
CASE
27-
WHEN LAG(bha.value, 1) OVER (PARTITION BY bha.symbol, bha.series_name ORDER BY bha.month_date) IS NOT NULL
28-
AND LAG(bha.value, 1) OVER (PARTITION BY bha.symbol, bha.series_name ORDER BY bha.month_date) != 0
29-
THEN ((bha.value - LAG(bha.value, 1) OVER (PARTITION BY bha.symbol, bha.series_name ORDER BY bha.month_date)) /
30-
LAG(bha.value, 1) OVER (PARTITION BY bha.symbol, bha.series_name ORDER BY bha.month_date)) * 100
31-
ELSE NULL
32-
END as econ_mom_change_pct,
33-
34-
-- Calculate 3-month rolling average of economic change
35-
AVG(bha.value) OVER (
36-
PARTITION BY bha.symbol, bha.series_name
37-
ORDER BY bha.month_date
38-
ROWS BETWEEN 2 PRECEDING AND CURRENT ROW
39-
) as econ_3mo_avg
40-
41-
FROM {{ ref('base_historical_analysis') }} bha
42-
LEFT JOIN {{ ref('fred_series_mapping') }} fsm
43-
ON bha.series_name = fsm.series_name
44-
WHERE bha.value IS NOT NULL
10+
SELECT
11+
bha.symbol,
12+
bha.month_date,
13+
bha.series_name,
14+
bha.category,
15+
fsm.category AS economic_category,
16+
bha.value AS current_econ_value,
17+
bha.monthly_avg_close,
18+
bha.pct_change_q1_forward,
19+
bha.pct_change_q2_forward,
20+
bha.pct_change_q3_forward,
21+
22+
-- Calculate month-over-month change in economic data
23+
LAG(bha.value, 1)
24+
OVER (
25+
PARTITION BY bha.symbol, bha.series_name ORDER BY bha.month_date
26+
)
27+
AS prev_econ_value,
28+
29+
-- Calculate rate of change (month-over-month %)
30+
CASE
31+
WHEN
32+
LAG(bha.value, 1)
33+
OVER (
34+
PARTITION BY bha.symbol, bha.series_name
35+
ORDER BY bha.month_date
36+
)
37+
IS NOT NULL
38+
AND LAG(bha.value, 1)
39+
OVER (
40+
PARTITION BY bha.symbol, bha.series_name
41+
ORDER BY bha.month_date
42+
)
43+
!= 0
44+
THEN (
45+
(
46+
bha.value
47+
- LAG(bha.value, 1)
48+
OVER (
49+
PARTITION BY bha.symbol, bha.series_name
50+
ORDER BY bha.month_date
51+
)
52+
)
53+
/ LAG(bha.value, 1)
54+
OVER (
55+
PARTITION BY bha.symbol, bha.series_name
56+
ORDER BY bha.month_date
57+
)
58+
) * 100
59+
END AS econ_mom_change_pct,
60+
61+
-- Calculate 3-month rolling average of economic change
62+
AVG(bha.value) OVER (
63+
PARTITION BY bha.symbol, bha.series_name
64+
ORDER BY bha.month_date
65+
ROWS BETWEEN 2 PRECEDING AND CURRENT ROW
66+
) AS econ_3mo_avg
67+
68+
FROM {{ ref('base_historical_analysis') }} AS bha
69+
LEFT JOIN {{ ref('fred_series_mapping') }} AS fsm
70+
ON bha.series_name = fsm.series_name
71+
WHERE bha.value IS NOT NULL
4572
),
4673

4774
correlation_analysis AS (
48-
SELECT
49-
symbol,
50-
series_name,
51-
category,
52-
economic_category,
53-
54-
-- Basic correlation metrics between economic change and future returns
55-
COUNT(*) as observation_count,
56-
57-
-- Correlation between economic MoM change and Q1 forward returns
58-
CORR(econ_mom_change_pct, pct_change_q1_forward) as corr_econ_q1_returns,
59-
CORR(econ_mom_change_pct, pct_change_q2_forward) as corr_econ_q2_returns,
60-
CORR(econ_mom_change_pct, pct_change_q3_forward) as corr_econ_q3_returns,
61-
62-
-- Average returns when economic data is growing vs declining
63-
AVG(CASE WHEN econ_mom_change_pct > 0 THEN pct_change_q1_forward END) as avg_q1_return_when_econ_growing,
64-
AVG(CASE WHEN econ_mom_change_pct < 0 THEN pct_change_q1_forward END) as avg_q1_return_when_econ_declining,
65-
66-
-- Standard deviations
67-
STDDEV(econ_mom_change_pct) as econ_change_volatility,
68-
STDDEV(pct_change_q1_forward) as q1_return_volatility,
69-
70-
-- Economic data statistics
71-
AVG(econ_mom_change_pct) as avg_econ_change_pct,
72-
MIN(econ_mom_change_pct) as min_econ_change_pct,
73-
MAX(econ_mom_change_pct) as max_econ_change_pct
74-
75-
FROM economic_changes
76-
WHERE econ_mom_change_pct IS NOT NULL
77-
GROUP BY symbol, series_name, category, economic_category
75+
SELECT
76+
symbol,
77+
series_name,
78+
category,
79+
economic_category,
80+
81+
-- Basic correlation metrics between economic change and future returns
82+
COUNT(*) AS observation_count,
83+
84+
-- Correlation between economic MoM change and Q1 forward returns
85+
CORR(econ_mom_change_pct, pct_change_q1_forward)
86+
AS corr_econ_q1_returns,
87+
CORR(econ_mom_change_pct, pct_change_q2_forward)
88+
AS corr_econ_q2_returns,
89+
CORR(econ_mom_change_pct, pct_change_q3_forward)
90+
AS corr_econ_q3_returns,
91+
92+
-- Average returns when economic data is growing vs declining
93+
AVG(CASE WHEN econ_mom_change_pct > 0 THEN pct_change_q1_forward END)
94+
AS avg_q1_return_when_econ_growing,
95+
AVG(CASE WHEN econ_mom_change_pct < 0 THEN pct_change_q1_forward END)
96+
AS avg_q1_return_when_econ_declining,
97+
98+
-- Standard deviations
99+
STDDEV(econ_mom_change_pct) AS econ_change_volatility,
100+
STDDEV(pct_change_q1_forward) AS q1_return_volatility,
101+
102+
-- Economic data statistics
103+
AVG(econ_mom_change_pct) AS avg_econ_change_pct,
104+
MIN(econ_mom_change_pct) AS min_econ_change_pct,
105+
MAX(econ_mom_change_pct) AS max_econ_change_pct
106+
107+
FROM economic_changes
108+
WHERE econ_mom_change_pct IS NOT NULL
109+
GROUP BY symbol, series_name, category, economic_category
78110
),
79111

80112
detailed_monthly_view AS (
81-
SELECT
113+
SELECT
114+
symbol,
115+
month_date,
116+
series_name,
117+
category,
118+
economic_category,
119+
econ_mom_change_pct,
120+
pct_change_q1_forward,
121+
pct_change_q2_forward,
122+
pct_change_q3_forward,
123+
124+
-- Quintile ranking of economic changes
125+
NTILE(5)
126+
OVER (PARTITION BY symbol, series_name ORDER BY econ_mom_change_pct)
127+
AS econ_change_quintile,
128+
129+
-- Lead/lag analysis - how does economic data relate to past and future returns
130+
LAG(pct_change_q1_forward, 1)
131+
OVER (PARTITION BY symbol ORDER BY month_date)
132+
AS prev_month_q1_return,
133+
LEAD(pct_change_q1_forward, 1)
134+
OVER (PARTITION BY symbol ORDER BY month_date)
135+
AS next_month_q1_return
136+
137+
FROM economic_changes
138+
WHERE econ_mom_change_pct IS NOT NULL
139+
)
140+
141+
-- Main Results: Show correlations and key insights
142+
SELECT
143+
'Correlation Analysis' AS analysis_type,
82144
symbol,
83-
month_date,
84145
series_name,
85146
category,
86147
economic_category,
87-
econ_mom_change_pct,
88-
pct_change_q1_forward,
89-
pct_change_q2_forward,
90-
pct_change_q3_forward,
91-
92-
-- Quintile ranking of economic changes
93-
NTILE(5) OVER (PARTITION BY symbol, series_name ORDER BY econ_mom_change_pct) as econ_change_quintile,
94-
95-
-- Lead/lag analysis - how does economic data relate to past and future returns
96-
LAG(pct_change_q1_forward, 1) OVER (PARTITION BY symbol ORDER BY month_date) as prev_month_q1_return,
97-
LEAD(pct_change_q1_forward, 1) OVER (PARTITION BY symbol ORDER BY month_date) as next_month_q1_return
98-
99-
FROM economic_changes
100-
WHERE econ_mom_change_pct IS NOT NULL
101-
)
102-
103-
-- Main Results: Show correlations and key insights
104-
SELECT
105-
'Correlation Analysis' as analysis_type,
106-
symbol,
107-
series_name,
108-
category,
109-
economic_category,
110-
observation_count,
111-
ROUND(corr_econ_q1_returns, 4) as correlation_econ_vs_q1_returns,
112-
ROUND(corr_econ_q2_returns, 4) as correlation_econ_vs_q2_returns,
113-
ROUND(corr_econ_q3_returns, 4) as correlation_econ_vs_q3_returns,
114-
ROUND(avg_q1_return_when_econ_growing, 2) as avg_q1_return_econ_up,
115-
ROUND(avg_q1_return_when_econ_declining, 2) as avg_q1_return_econ_down,
116-
ROUND(avg_q1_return_when_econ_declining, 2) as return_difference
148+
observation_count,
149+
ROUND(corr_econ_q1_returns, 4) AS correlation_econ_vs_q1_returns,
150+
ROUND(corr_econ_q2_returns, 4) AS correlation_econ_vs_q2_returns,
151+
ROUND(corr_econ_q3_returns, 4) AS correlation_econ_vs_q3_returns,
152+
ROUND(avg_q1_return_when_econ_growing, 2) AS avg_q1_return_econ_up,
153+
ROUND(avg_q1_return_when_econ_declining, 2) AS avg_q1_return_econ_down,
154+
ROUND(avg_q1_return_when_econ_declining, 2) AS return_difference
117155
FROM correlation_analysis
118156
WHERE observation_count >= 10 -- Filter for meaningful sample sizes
119157

120158
UNION ALL
121159

122160
-- Quintile Analysis: Performance by economic data change quintiles
123-
SELECT
124-
'Quintile Analysis' as analysis_type,
125-
symbol,
126-
series_name,
127-
category,
128-
economic_category,
129-
NULL as observation_count,
130-
econ_change_quintile as correlation_econ_vs_q1_returns,
131-
NULL as correlation_econ_vs_q2_returns,
132-
NULL as correlation_econ_vs_q3_returns,
133-
ROUND(AVG(pct_change_q1_forward), 2) as avg_q1_return_econ_up,
134-
COUNT(*) as avg_q1_return_econ_down,
135-
ROUND(AVG(econ_mom_change_pct), 2) as return_difference
161+
SELECT
162+
'Quintile Analysis' AS analysis_type,
163+
symbol,
164+
series_name,
165+
category,
166+
economic_category,
167+
NULL AS observation_count,
168+
econ_change_quintile AS correlation_econ_vs_q1_returns,
169+
NULL AS correlation_econ_vs_q2_returns,
170+
NULL AS correlation_econ_vs_q3_returns,
171+
ROUND(AVG(pct_change_q1_forward), 2) AS avg_q1_return_econ_up,
172+
COUNT(*) AS avg_q1_return_econ_down,
173+
ROUND(AVG(econ_mom_change_pct), 2) AS return_difference
136174
FROM detailed_monthly_view
137175
GROUP BY symbol, series_name, category, economic_category, econ_change_quintile
138176
HAVING COUNT(*) >= 3

0 commit comments

Comments
 (0)