diff --git a/dbt/models/intermediate/_intermediate.yml b/dbt/models/intermediate/_intermediate.yml index ab62db8..9fd7fd9 100644 --- a/dbt/models/intermediate/_intermediate.yml +++ b/dbt/models/intermediate/_intermediate.yml @@ -1,17 +1,6 @@ version: 2 models: - - name: int_purchases__monthly - description: "Purchase data enriched with time and district dimensions" - columns: - - name: purchase_id - tests: - - unique - - not_null - - name: district_code - tests: - - not_null - - name: int_rentals__annual description: "Rental data enriched with time and zone dimensions" columns: diff --git a/dbt/models/intermediate/int_purchases__monthly.sql b/dbt/models/intermediate/int_purchases__monthly.sql deleted file mode 100644 index b03f5d8..0000000 --- a/dbt/models/intermediate/int_purchases__monthly.sql +++ /dev/null @@ -1,62 +0,0 @@ --- Intermediate: Monthly purchase data enriched with dimensions --- Joins purchases with time and district dimensions for analysis - -with purchases as ( - select * from {{ ref('stg_trreb__purchases') }} -), - -time_dim as ( - select * from {{ ref('stg_dimensions__time') }} -), - -district_dim as ( - select * from {{ ref('stg_dimensions__trreb_districts') }} -), - -enriched as ( - select - p.purchase_id, - - -- Time attributes - t.date_key, - t.full_date, - t.year, - t.month, - t.quarter, - t.month_name, - - -- District attributes - d.district_key, - d.district_code, - d.district_name, - d.area_type, - - -- Metrics - p.sales_count, - p.dollar_volume, - p.avg_price, - p.median_price, - p.new_listings, - p.active_listings, - p.days_on_market, - p.sale_to_list_ratio, - - -- Calculated metrics - case - when p.active_listings > 0 - then round(p.sales_count::numeric / p.active_listings, 3) - else null - end as absorption_rate, - - case - when p.sales_count > 0 - then round(p.active_listings::numeric / p.sales_count, 1) - else null - end as months_of_inventory - - from purchases p - inner join time_dim t on p.date_key = t.date_key - inner join district_dim d on p.district_key = d.district_key -) - -select * from enriched diff --git a/dbt/models/marts/_marts.yml b/dbt/models/marts/_marts.yml index b6419a2..1f89efe 100644 --- a/dbt/models/marts/_marts.yml +++ b/dbt/models/marts/_marts.yml @@ -1,15 +1,6 @@ version: 2 models: - - name: mart_toronto_purchases - description: "Final mart for Toronto purchase/sales analysis by district and time" - columns: - - name: purchase_id - description: "Unique purchase record identifier" - tests: - - unique - - not_null - - name: mart_toronto_rentals description: "Final mart for Toronto rental market analysis by zone and time" columns: @@ -18,6 +9,3 @@ models: tests: - unique - not_null - - - name: mart_toronto_market_summary - description: "Combined market summary aggregating purchases and rentals at Toronto level" diff --git a/dbt/models/marts/mart_toronto_market_summary.sql b/dbt/models/marts/mart_toronto_market_summary.sql deleted file mode 100644 index cec3c77..0000000 --- a/dbt/models/marts/mart_toronto_market_summary.sql +++ /dev/null @@ -1,81 +0,0 @@ --- Mart: Toronto Market Summary --- Aggregated view combining purchase and rental market indicators --- Grain: One row per year-month - -with purchases_agg as ( - select - year, - month, - month_name, - quarter, - - -- Aggregate purchase metrics across all districts - sum(sales_count) as total_sales, - sum(dollar_volume) as total_dollar_volume, - round(avg(avg_price), 0) as avg_price_all_districts, - round(avg(median_price), 0) as median_price_all_districts, - sum(new_listings) as total_new_listings, - sum(active_listings) as total_active_listings, - round(avg(days_on_market), 0) as avg_days_on_market, - round(avg(sale_to_list_ratio), 2) as avg_sale_to_list_ratio, - round(avg(absorption_rate), 3) as avg_absorption_rate, - round(avg(months_of_inventory), 1) as avg_months_of_inventory, - round(avg(avg_price_yoy_pct), 2) as avg_price_yoy_pct - - from {{ ref('mart_toronto_purchases') }} - group by year, month, month_name, quarter -), - -rentals_agg as ( - select - year, - - -- Aggregate rental metrics across all zones (all bedroom types) - round(avg(avg_rent), 0) as avg_rent_all_zones, - round(avg(vacancy_rate), 2) as avg_vacancy_rate, - round(avg(rent_change_pct), 2) as avg_rent_change_pct, - sum(rental_universe) as total_rental_universe - - from {{ ref('mart_toronto_rentals') }} - group by year -), - -final as ( - select - p.year, - p.month, - p.month_name, - p.quarter, - - -- Purchase market indicators - p.total_sales, - p.total_dollar_volume, - p.avg_price_all_districts, - p.median_price_all_districts, - p.total_new_listings, - p.total_active_listings, - p.avg_days_on_market, - p.avg_sale_to_list_ratio, - p.avg_absorption_rate, - p.avg_months_of_inventory, - p.avg_price_yoy_pct, - - -- Rental market indicators (annual, so join on year) - r.avg_rent_all_zones, - r.avg_vacancy_rate, - r.avg_rent_change_pct, - r.total_rental_universe, - - -- Affordability indicator (price to rent ratio) - case - when r.avg_rent_all_zones > 0 - then round(p.avg_price_all_districts / (r.avg_rent_all_zones * 12), 1) - else null - end as price_to_annual_rent_ratio - - from purchases_agg p - left join rentals_agg r on p.year = r.year -) - -select * from final -order by year desc, month desc diff --git a/dbt/models/marts/mart_toronto_purchases.sql b/dbt/models/marts/mart_toronto_purchases.sql deleted file mode 100644 index 80c5766..0000000 --- a/dbt/models/marts/mart_toronto_purchases.sql +++ /dev/null @@ -1,79 +0,0 @@ --- Mart: Toronto Purchase Market Analysis --- Final analytical table for purchase/sales data visualization --- Grain: One row per district per month - -with purchases as ( - select * from {{ ref('int_purchases__monthly') }} -), - --- Add year-over-year calculations -with_yoy as ( - select - p.*, - - -- Previous year same month values - lag(p.avg_price, 12) over ( - partition by p.district_code - order by p.date_key - ) as avg_price_prev_year, - - lag(p.sales_count, 12) over ( - partition by p.district_code - order by p.date_key - ) as sales_count_prev_year, - - lag(p.median_price, 12) over ( - partition by p.district_code - order by p.date_key - ) as median_price_prev_year - - from purchases p -), - -final as ( - select - purchase_id, - date_key, - full_date, - year, - month, - quarter, - month_name, - district_key, - district_code, - district_name, - area_type, - sales_count, - dollar_volume, - avg_price, - median_price, - new_listings, - active_listings, - days_on_market, - sale_to_list_ratio, - absorption_rate, - months_of_inventory, - - -- Year-over-year changes - case - when avg_price_prev_year > 0 - then round(((avg_price - avg_price_prev_year) / avg_price_prev_year) * 100, 2) - else null - end as avg_price_yoy_pct, - - case - when sales_count_prev_year > 0 - then round(((sales_count - sales_count_prev_year)::numeric / sales_count_prev_year) * 100, 2) - else null - end as sales_count_yoy_pct, - - case - when median_price_prev_year > 0 - then round(((median_price - median_price_prev_year) / median_price_prev_year) * 100, 2) - else null - end as median_price_yoy_pct - - from with_yoy -) - -select * from final diff --git a/dbt/models/staging/_sources.yml b/dbt/models/staging/_sources.yml index ff92376..1fffd9a 100644 --- a/dbt/models/staging/_sources.yml +++ b/dbt/models/staging/_sources.yml @@ -2,20 +2,10 @@ version: 2 sources: - name: toronto_housing - description: "Toronto housing data loaded from TRREB and CMHC sources" + description: "Toronto housing data loaded from CMHC and City of Toronto sources" database: portfolio schema: public tables: - - name: fact_purchases - description: "TRREB monthly purchase/sales statistics by district" - columns: - - name: id - description: "Primary key" - - name: date_key - description: "Foreign key to dim_time" - - name: district_key - description: "Foreign key to dim_trreb_district" - - name: fact_rentals description: "CMHC annual rental survey data by zone and bedroom type" columns: @@ -32,14 +22,6 @@ sources: - name: date_key description: "Primary key (YYYYMMDD format)" - - name: dim_trreb_district - description: "TRREB district dimension with geometry" - columns: - - name: district_key - description: "Primary key" - - name: district_code - description: "TRREB district code" - - name: dim_cmhc_zone description: "CMHC zone dimension with geometry" columns: @@ -49,7 +31,7 @@ sources: description: "CMHC zone code" - name: dim_neighbourhood - description: "City of Toronto neighbourhoods (reference only)" + description: "City of Toronto neighbourhoods (158 official boundaries)" columns: - name: neighbourhood_id description: "Primary key" diff --git a/dbt/models/staging/_staging.yml b/dbt/models/staging/_staging.yml index a3458a6..1fb83f4 100644 --- a/dbt/models/staging/_staging.yml +++ b/dbt/models/staging/_staging.yml @@ -1,23 +1,6 @@ version: 2 models: - - name: stg_trreb__purchases - description: "Staged TRREB purchase/sales data from fact_purchases" - columns: - - name: purchase_id - description: "Unique identifier for purchase record" - tests: - - unique - - not_null - - name: date_key - description: "Date dimension key (YYYYMMDD)" - tests: - - not_null - - name: district_key - description: "TRREB district dimension key" - tests: - - not_null - - name: stg_cmhc__rentals description: "Staged CMHC rental market data from fact_rentals" columns: @@ -44,20 +27,6 @@ models: - unique - not_null - - name: stg_dimensions__trreb_districts - description: "Staged TRREB district dimension" - columns: - - name: district_key - description: "District dimension key" - tests: - - unique - - not_null - - name: district_code - description: "TRREB district code (e.g., W01, C01)" - tests: - - unique - - not_null - - name: stg_dimensions__cmhc_zones description: "Staged CMHC zone dimension" columns: diff --git a/dbt/models/staging/stg_dimensions__trreb_districts.sql b/dbt/models/staging/stg_dimensions__trreb_districts.sql deleted file mode 100644 index c0e5dc6..0000000 --- a/dbt/models/staging/stg_dimensions__trreb_districts.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Staged TRREB district dimension --- Source: dim_trreb_district table --- Grain: One row per district - -with source as ( - select * from {{ source('toronto_housing', 'dim_trreb_district') }} -), - -staged as ( - select - district_key, - district_code, - district_name, - area_type, - geometry - from source -) - -select * from staged diff --git a/dbt/models/staging/stg_trreb__purchases.sql b/dbt/models/staging/stg_trreb__purchases.sql deleted file mode 100644 index 3694d71..0000000 --- a/dbt/models/staging/stg_trreb__purchases.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Staged TRREB purchase/sales data --- Source: fact_purchases table loaded from TRREB Market Watch PDFs --- Grain: One row per district per month - -with source as ( - select * from {{ source('toronto_housing', 'fact_purchases') }} -), - -staged as ( - select - id as purchase_id, - date_key, - district_key, - sales_count, - dollar_volume, - avg_price, - median_price, - new_listings, - active_listings, - avg_dom as days_on_market, - avg_sp_lp as sale_to_list_ratio - from source -) - -select * from staged