lmiranda
053acf6436
Add schemas, parsers, loaders, and models for Toronto neighbourhood-centric data including census profiles, crime statistics, and amenities. Schemas: - NeighbourhoodRecord, CensusRecord, CrimeRecord, CrimeType - AmenityType, AmenityRecord, AmenityCount Models: - BridgeCMHCNeighbourhood (zone-to-neighbourhood mapping with weights) - FactCensus, FactCrime, FactAmenities Parsers: - TorontoOpenDataParser (CKAN API for neighbourhoods, census, amenities) - TorontoPoliceParser (crime rates, MCI data) Loaders: - load_census_data, load_crime_data, load_amenities - build_cmhc_neighbourhood_crosswalk (PostGIS area weights) Also updates CLAUDE.md with projman plugin workflow documentation. Closes #53, #54, #55, #56, #57, #58, #59 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
69 lines
2.3 KiB
Python
69 lines
2.3 KiB
Python
"""Loader for census data to fact_census table."""
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from portfolio_app.toronto.models import FactCensus
|
|
from portfolio_app.toronto.schemas import CensusRecord
|
|
|
|
from .base import get_session, upsert_by_key
|
|
|
|
|
|
def load_census_data(
|
|
records: list[CensusRecord],
|
|
session: Session | None = None,
|
|
) -> int:
|
|
"""Load census records to fact_census table.
|
|
|
|
Args:
|
|
records: List of validated CensusRecord schemas.
|
|
session: Optional existing session.
|
|
|
|
Returns:
|
|
Number of records loaded (inserted + updated).
|
|
"""
|
|
|
|
def _load(sess: Session) -> int:
|
|
models = []
|
|
for r in records:
|
|
model = FactCensus(
|
|
neighbourhood_id=r.neighbourhood_id,
|
|
census_year=r.census_year,
|
|
population=r.population,
|
|
population_density=float(r.population_density)
|
|
if r.population_density
|
|
else None,
|
|
median_household_income=float(r.median_household_income)
|
|
if r.median_household_income
|
|
else None,
|
|
average_household_income=float(r.average_household_income)
|
|
if r.average_household_income
|
|
else None,
|
|
unemployment_rate=float(r.unemployment_rate)
|
|
if r.unemployment_rate
|
|
else None,
|
|
pct_bachelors_or_higher=float(r.pct_bachelors_or_higher)
|
|
if r.pct_bachelors_or_higher
|
|
else None,
|
|
pct_owner_occupied=float(r.pct_owner_occupied)
|
|
if r.pct_owner_occupied
|
|
else None,
|
|
pct_renter_occupied=float(r.pct_renter_occupied)
|
|
if r.pct_renter_occupied
|
|
else None,
|
|
median_age=float(r.median_age) if r.median_age else None,
|
|
average_dwelling_value=float(r.average_dwelling_value)
|
|
if r.average_dwelling_value
|
|
else None,
|
|
)
|
|
models.append(model)
|
|
|
|
inserted, updated = upsert_by_key(
|
|
sess, FactCensus, models, ["neighbourhood_id", "census_year"]
|
|
)
|
|
return inserted + updated
|
|
|
|
if session:
|
|
return _load(session)
|
|
with get_session() as sess:
|
|
return _load(sess)
|