fix: Repair data pipeline with StatCan CMHC rental data

- Add StatCan CMHC parser to fetch rental data from Statistics Canada API
- Create year spine (2014-2025) as time dimension driver instead of census
- Add CMA-level rental and income intermediate models
- Update mart_neighbourhood_overview to use rental years as base
- Fix neighbourhood_service queries to match dbt schema
- Add CMHC data loading to pipeline script

Data now flows correctly: 158 neighbourhoods × 12 years = 1,896 records
Rent data available 2019-2025, crime data 2014-2024

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

dbt/models/intermediate/int_census__toronto_cma.sql

@@ -0,0 +1,60 @@
+-- Intermediate: Toronto CMA census statistics by year
+-- Provides city-wide averages for metrics not available at neighbourhood level
+-- Used when neighbourhood-level data is unavailable (e.g., median household income)
+-- Grain: One row per year
+
+with years as (
+    select * from {{ ref('int_year_spine') }}
+),
+
+census as (
+    select * from {{ ref('stg_toronto__census') }}
+),
+
+-- Census data is only available for 2016 and 2021
+-- Map each analysis year to the appropriate census year
+year_to_census as (
+    select
+        y.year,
+        case
+            when y.year <= 2018 then 2016
+            else 2021
+        end as census_year
+    from years y
+),
+
+-- Toronto CMA median household income from Statistics Canada
+-- Source: Census Profile Table 98-316-X2021001
+-- 2016: $65,829 (from Census Profile)
+-- 2021: $84,000 (from Census Profile)
+cma_income as (
+    select 2016 as census_year, 65829 as median_household_income union all
+    select 2021 as census_year, 84000 as median_household_income
+),
+
+-- City-wide aggregates from loaded neighbourhood data
+city_aggregates as (
+    select
+        census_year,
+        sum(population) as total_population,
+        avg(population_density) as avg_population_density,
+        avg(unemployment_rate) as avg_unemployment_rate
+    from census
+    where population is not null
+    group by census_year
+),
+
+final as (
+    select
+        y.year,
+        y.census_year,
+        ci.median_household_income,
+        ca.total_population,
+        ca.avg_population_density,
+        ca.avg_unemployment_rate
+    from year_to_census y
+    left join cma_income ci on y.census_year = ci.census_year
+    left join city_aggregates ca on y.census_year = ca.census_year
+)
+
+select * from final

dbt/models/intermediate/int_neighbourhood__amenity_scores.sql

@@ -34,7 +34,7 @@ amenity_scores as (
         n.population,
         n.land_area_sqkm,
 
-        a.year,
+        coalesce(a.year, 2021) as year,
 
         -- Raw counts
         a.parks_count,

dbt/models/intermediate/int_neighbourhood__crime_summary.sql

@@ -64,15 +64,17 @@ crime_summary as (
         w.robbery_count,
         w.theft_over_count,
         w.homicide_count,
-        w.avg_rate_per_100k,
         w.yoy_change_pct,
 
-        -- Crime rate per 100K population
-        case
-            when n.population > 0
-            then round(w.total_incidents::numeric / n.population * 100000, 2)
-            else null
-        end as crime_rate_per_100k
+        -- Crime rate per 100K population (use source data avg, or calculate if population available)
+        coalesce(
+            w.avg_rate_per_100k,
+            case
+                when n.population > 0
+                then round(w.total_incidents::numeric / n.population * 100000, 2)
+                else null
+            end
+        ) as crime_rate_per_100k
 
     from neighbourhoods n
     inner join with_yoy w on n.neighbourhood_id = w.neighbourhood_id

dbt/models/intermediate/int_neighbourhood__demographics.sql

@@ -17,7 +17,8 @@ demographics as (
         n.geometry,
         n.land_area_sqkm,
 
-        c.census_year,
+        -- Use census_year from census data, or fall back to dim_neighbourhood's year
+        coalesce(c.census_year, n.census_year, 2021) as census_year,
         c.population,
         c.population_density,
         c.median_household_income,

dbt/models/intermediate/int_neighbourhood__housing.sql

@@ -20,7 +20,7 @@ housing as (
         n.neighbourhood_name,
         n.geometry,
 
-        coalesce(r.year, c.census_year) as year,
+        coalesce(r.year, c.census_year, 2021) as year,
 
         -- Census housing metrics
         c.pct_owner_occupied,

dbt/models/intermediate/int_rentals__toronto_cma.sql

@@ -0,0 +1,25 @@
+-- Intermediate: Toronto CMA rental metrics by year
+-- Aggregates rental data to city-wide averages by year
+-- Source: StatCan CMHC data at CMA level
+-- Grain: One row per year
+
+with rentals as (
+    select * from {{ ref('stg_cmhc__rentals') }}
+),
+
+-- Pivot bedroom types to columns
+yearly_rentals as (
+    select
+        year,
+        max(case when bedroom_type = 'bachelor' then avg_rent end) as avg_rent_bachelor,
+        max(case when bedroom_type = '1bed' then avg_rent end) as avg_rent_1bed,
+        max(case when bedroom_type = '2bed' then avg_rent end) as avg_rent_2bed,
+        max(case when bedroom_type = '3bed' then avg_rent end) as avg_rent_3bed,
+        -- Use 2-bedroom as standard reference
+        max(case when bedroom_type = '2bed' then avg_rent end) as avg_rent_standard,
+        max(vacancy_rate) as vacancy_rate
+    from rentals
+    group by year
+)
+
+select * from yearly_rentals

dbt/models/intermediate/int_year_spine.sql

@@ -0,0 +1,11 @@
+-- Intermediate: Year spine for analysis
+-- Creates a row for each year from 2014-2025
+-- Used to drive time-series analysis across all data sources
+
+with years as (
+    -- Generate years from available data sources
+    -- Crime data: 2014-2024, Rentals: 2019-2025
+    select generate_series(2014, 2025) as year
+)
+
+select year from years