"""Parser for Toronto Open Data CKAN API.

Fetches neighbourhood boundaries, census profiles, and amenities data
from the City of Toronto's Open Data Portal.

API Documentation: https://open.toronto.ca/dataset/
"""

import contextlib
import json
import logging
from decimal import Decimal
from pathlib import Path
from typing import Any

import httpx

from portfolio_app.toronto.schemas import (
    AmenityRecord,
    AmenityType,
    CensusRecord,
    NeighbourhoodRecord,
)

logger = logging.getLogger(__name__)


class TorontoOpenDataParser:
    """Parser for Toronto Open Data CKAN API.

    Provides methods to fetch and parse neighbourhood boundaries, census profiles,
    and amenities (parks, schools, childcare) from the Toronto Open Data portal.
    """

    BASE_URL = "https://ckan0.cf.opendata.inter.prod-toronto.ca"
    API_PATH = "/api/3/action"

    # Dataset package IDs
    DATASETS = {
        "neighbourhoods": "neighbourhoods",
        "neighbourhood_profiles": "neighbourhood-profiles",
        "parks": "parks",
        "schools": "school-locations-all-types",
        "childcare": "licensed-child-care-centres",
    }

    def __init__(
        self,
        cache_dir: Path | None = None,
        timeout: float = 30.0,
    ) -> None:
        """Initialize parser.

        Args:
            cache_dir: Optional directory for caching API responses.
            timeout: HTTP request timeout in seconds.
        """
        self._cache_dir = cache_dir
        self._timeout = timeout
        self._client: httpx.Client | None = None
        self._neighbourhood_name_map: dict[str, int] | None = None

    @property
    def client(self) -> httpx.Client:
        """Lazy-initialize HTTP client."""
        if self._client is None:
            self._client = httpx.Client(
                base_url=self.BASE_URL,
                timeout=self._timeout,
                headers={"Accept": "application/json"},
            )
        return self._client

    def close(self) -> None:
        """Close HTTP client."""
        if self._client is not None:
            self._client.close()
            self._client = None

    def _get_neighbourhood_name_map(self) -> dict[str, int]:
        """Build and cache a mapping of neighbourhood names to IDs.

        Returns:
            Dictionary mapping normalized neighbourhood names to area_id.
        """
        if self._neighbourhood_name_map is not None:
            return self._neighbourhood_name_map

        neighbourhoods = self.get_neighbourhoods()
        self._neighbourhood_name_map = {}

        for n in neighbourhoods:
            # Add multiple variations of the name for flexible matching
            name_lower = n.area_name.lower().strip()
            self._neighbourhood_name_map[name_lower] = n.area_id

            # Also add without common suffixes/prefixes
            for suffix in [" neighbourhood", " area", "-"]:
                if suffix in name_lower:
                    alt_name = name_lower.replace(suffix, "").strip()
                    self._neighbourhood_name_map[alt_name] = n.area_id

        logger.debug(
            f"Built neighbourhood name map with {len(self._neighbourhood_name_map)} entries"
        )
        return self._neighbourhood_name_map

    def _match_neighbourhood_id(self, name: str) -> int | None:
        """Match a neighbourhood name to its ID.

        Args:
            name: Neighbourhood name from census data.

        Returns:
            Neighbourhood ID or None if not found.
        """
        name_map = self._get_neighbourhood_name_map()
        name_lower = name.lower().strip()

        # Direct match
        if name_lower in name_map:
            return name_map[name_lower]

        # Try removing parenthetical content
        if "(" in name_lower:
            base_name = name_lower.split("(")[0].strip()
            if base_name in name_map:
                return name_map[base_name]

        # Try fuzzy matching with first few chars
        for key, area_id in name_map.items():
            if key.startswith(name_lower[:10]) or name_lower.startswith(key[:10]):
                return area_id

        return None

    def __enter__(self) -> "TorontoOpenDataParser":
        return self

    def __exit__(self, *args: Any) -> None:
        self.close()

    def _get_package(self, package_id: str) -> dict[str, Any]:
        """Fetch package metadata from CKAN.

        Args:
            package_id: The package/dataset ID.

        Returns:
            Package metadata dictionary.
        """
        response = self.client.get(
            f"{self.API_PATH}/package_show",
            params={"id": package_id},
        )
        response.raise_for_status()
        result = response.json()

        if not result.get("success"):
            raise ValueError(f"CKAN API error: {result.get('error', 'Unknown error')}")

        return dict(result["result"])

    def _get_resource_url(
        self,
        package_id: str,
        format_filter: str = "geojson",
    ) -> str:
        """Get the download URL for a resource in a package.

        Args:
            package_id: The package/dataset ID.
            format_filter: Resource format to filter by (e.g., 'geojson', 'csv').

        Returns:
            Resource download URL.

        Raises:
            ValueError: If no matching resource is found.
        """
        package = self._get_package(package_id)
        resources = package.get("resources", [])

        for resource in resources:
            resource_format = resource.get("format", "").lower()
            if format_filter.lower() in resource_format:
                return str(resource["url"])

        available = [r.get("format") for r in resources]
        raise ValueError(
            f"No {format_filter} resource in {package_id}. Available: {available}"
        )

    def _fetch_geojson(self, package_id: str) -> dict[str, Any]:
        """Fetch GeoJSON data from a package.

        Handles both pure GeoJSON responses and CSV responses with embedded
        geometry columns (common in Toronto Open Data).

        Args:
            package_id: The package/dataset ID.

        Returns:
            GeoJSON FeatureCollection.
        """
        # Check cache first
        if self._cache_dir:
            cache_file = self._cache_dir / f"{package_id}.geojson"
            if cache_file.exists():
                logger.debug(f"Loading {package_id} from cache")
                with open(cache_file, encoding="utf-8") as f:
                    return dict(json.load(f))

        url = self._get_resource_url(package_id, format_filter="geojson")
        logger.info(f"Fetching GeoJSON from {url}")

        response = self.client.get(url)
        response.raise_for_status()

        # Try to parse as JSON first
        try:
            data = response.json()
            # If it's already a valid GeoJSON FeatureCollection, return it
            if isinstance(data, dict) and data.get("type") == "FeatureCollection":
                if self._cache_dir:
                    self._cache_dir.mkdir(parents=True, exist_ok=True)
                    cache_file = self._cache_dir / f"{package_id}.geojson"
                    with open(cache_file, "w", encoding="utf-8") as f:
                        json.dump(data, f)
                return dict(data)
        except json.JSONDecodeError:
            pass

        # If JSON parsing failed, it's likely CSV with embedded geometry
        # Parse CSV and convert to GeoJSON FeatureCollection
        logger.info("Response is CSV format, converting to GeoJSON...")
        import csv
        import io

        # Increase field size limit for large geometry columns
        csv.field_size_limit(10 * 1024 * 1024)  # 10 MB

        csv_text = response.text
        reader = csv.DictReader(io.StringIO(csv_text))

        features = []
        for row in reader:
            # Extract geometry from the 'geometry' column if present
            geometry = None
            if "geometry" in row and row["geometry"]:
                with contextlib.suppress(json.JSONDecodeError):
                    geometry = json.loads(row["geometry"])

            # Build properties from all other columns
            properties = {k: v for k, v in row.items() if k != "geometry"}

            features.append(
                {
                    "type": "Feature",
                    "geometry": geometry,
                    "properties": properties,
                }
            )

        geojson_data: dict[str, Any] = {
            "type": "FeatureCollection",
            "features": features,
        }

        # Cache the converted response
        if self._cache_dir:
            self._cache_dir.mkdir(parents=True, exist_ok=True)
            cache_file = self._cache_dir / f"{package_id}.geojson"
            with open(cache_file, "w", encoding="utf-8") as f:
                json.dump(geojson_data, f)

        return geojson_data

    def _fetch_csv_as_json(self, package_id: str) -> list[dict[str, Any]]:
        """Fetch CSV data as JSON records via CKAN datastore.

        Args:
            package_id: The package/dataset ID.

        Returns:
            List of records as dictionaries.
        """
        package = self._get_package(package_id)
        resources = package.get("resources", [])

        # Find a datastore-enabled resource
        for resource in resources:
            if resource.get("datastore_active"):
                resource_id = resource["id"]
                break
        else:
            raise ValueError(f"No datastore resource in {package_id}")

        # Fetch all records via datastore_search
        records: list[dict[str, Any]] = []
        offset = 0
        limit = 1000

        while True:
            response = self.client.get(
                f"{self.API_PATH}/datastore_search",
                params={"id": resource_id, "limit": limit, "offset": offset},
            )
            response.raise_for_status()
            result = response.json()

            if not result.get("success"):
                raise ValueError(f"Datastore error: {result.get('error')}")

            batch = result["result"]["records"]
            records.extend(batch)

            if len(batch) < limit:
                break
            offset += limit

        return records

    def get_neighbourhoods(self) -> list[NeighbourhoodRecord]:
        """Fetch 158 Toronto neighbourhood boundaries.

        Returns:
            List of validated NeighbourhoodRecord objects.
        """
        geojson = self._fetch_geojson(self.DATASETS["neighbourhoods"])
        features = geojson.get("features", [])

        records = []
        for feature in features:
            props = feature.get("properties", {})
            geometry = feature.get("geometry")

            # Use AREA_SHORT_CODE as the primary ID (1-158 range)
            # AREA_ID is a large internal identifier not useful for our schema
            short_code = props.get("AREA_SHORT_CODE") or props.get(
                "area_short_code", ""
            )
            if short_code:
                area_id = int("".join(c for c in str(short_code) if c.isdigit()) or "0")
            else:
                # Fallback to _id (row number) if AREA_SHORT_CODE not available
                area_id = int(props.get("_id", 0))

            if area_id == 0:
                logger.warning(f"Skipping neighbourhood with no valid ID: {props}")
                continue

            area_name = (
                props.get("AREA_NAME")
                or props.get("area_name")
                or f"Neighbourhood {area_id}"
            )

            records.append(
                NeighbourhoodRecord(
                    area_id=area_id,
                    area_name=str(area_name),
                    area_short_code=str(short_code) if short_code else None,
                    geometry=geometry,
                )
            )

        logger.info(f"Parsed {len(records)} neighbourhoods")
        return records

    # Mapping of indicator names to CensusRecord fields
    # Keys are partial matches (case-insensitive) found in the "Characteristic" column
    # Order matters - first match wins, so more specific patterns come first
    # Note: owner/renter counts are raw numbers, not percentages - calculated in dbt
    CENSUS_INDICATOR_MAPPING: dict[str, str] = {
        "population, 2021": "population",
        "population, 2016": "population",
        "population density per square kilometre": "population_density",
        "median total income of households in": "median_household_income",
        "average total income of households in": "average_household_income",
        "unemployment rate": "unemployment_rate",
        "bachelor's degree or higher": "pct_bachelors_or_higher",
        "average age": "median_age",
        "average value of dwellings": "average_dwelling_value",
    }

    def get_census_profiles(self, year: int = 2021) -> list[CensusRecord]:
        """Fetch neighbourhood census profiles.

        The Toronto Open Data neighbourhood profiles dataset is pivoted:
        - Rows are demographic indicators (e.g., "Population", "Median Income")
        - Columns are neighbourhoods (e.g., "Agincourt North", "Alderwood")

        This method transposes the data to create one CensusRecord per neighbourhood.

        Args:
            year: Census year (2016 or 2021).

        Returns:
            List of validated CensusRecord objects.
        """
        try:
            raw_records = self._fetch_csv_as_json(
                self.DATASETS["neighbourhood_profiles"]
            )
        except ValueError as e:
            logger.warning(f"Could not fetch census profiles: {e}")
            return []

        if not raw_records:
            logger.warning("Census profiles dataset is empty")
            return []

        logger.info(f"Fetched {len(raw_records)} census profile rows")

        # Find the characteristic/indicator column name
        # Prioritize "Characteristic" over "Category" since both may exist
        sample_row = raw_records[0]
        char_col = None

        # First try exact match for Characteristic
        if "Characteristic" in sample_row:
            char_col = "Characteristic"
        else:
            # Fall back to pattern matching
            for col in sample_row:
                col_lower = col.lower()
                if "characteristic" in col_lower:
                    char_col = col
                    break

            # Last resort: try Category
            if not char_col:
                for col in sample_row:
                    if "category" in col.lower():
                        char_col = col
                        break

        if not char_col:
            # Try other common column names
            for candidate in ["Topic", "_id"]:
                if candidate in sample_row:
                    char_col = candidate
                    break

        if not char_col:
            logger.warning("Could not find characteristic column in census data")
            return []

        # Identify neighbourhood columns (exclude metadata columns)
        exclude_cols = {
            char_col,
            "_id",
            "Topic",
            "Data Source",
            "Characteristic",
            "Category",
        }
        neighbourhood_cols = [col for col in sample_row if col not in exclude_cols]

        logger.info(f"Found {len(neighbourhood_cols)} neighbourhood columns")

        # Build a lookup: neighbourhood_name -> {field: value}
        neighbourhood_data: dict[str, dict[str, Decimal | int | None]] = {
            col: {} for col in neighbourhood_cols
        }

        # Process each row to extract indicator values
        for row in raw_records:
            characteristic = str(row.get(char_col, "")).lower().strip()

            # Check if this row matches any indicator we care about
            for indicator_pattern, field_name in self.CENSUS_INDICATOR_MAPPING.items():
                if indicator_pattern in characteristic:
                    # Extract values for each neighbourhood
                    for col in neighbourhood_cols:
                        value = row.get(col)
                        if value is not None and value != "":
                            try:
                                # Clean and convert value
                                str_val = str(value).replace(",", "").replace("$", "")
                                str_val = str_val.replace("%", "").strip()
                                if str_val and str_val not in ("x", "X", "F", ".."):
                                    numeric_val = Decimal(str_val)
                                    # Only store if not already set (first match wins)
                                    if field_name not in neighbourhood_data[col]:
                                        neighbourhood_data[col][
                                            field_name
                                        ] = numeric_val
                            except (ValueError, TypeError):
                                pass
                    break  # Move to next row after matching

        # Convert to CensusRecord objects
        records = []
        unmatched = []

        for neighbourhood_name, data in neighbourhood_data.items():
            if not data:
                continue

            # Match neighbourhood name to ID
            neighbourhood_id = self._match_neighbourhood_id(neighbourhood_name)
            if neighbourhood_id is None:
                unmatched.append(neighbourhood_name)
                continue

            try:
                pop_val = data.get("population")
                population = int(pop_val) if pop_val is not None else None

                record = CensusRecord(
                    neighbourhood_id=neighbourhood_id,
                    census_year=year,
                    population=population,
                    population_density=data.get("population_density"),
                    median_household_income=data.get("median_household_income"),
                    average_household_income=data.get("average_household_income"),
                    unemployment_rate=data.get("unemployment_rate"),
                    pct_bachelors_or_higher=data.get("pct_bachelors_or_higher"),
                    pct_owner_occupied=data.get("pct_owner_occupied"),
                    pct_renter_occupied=data.get("pct_renter_occupied"),
                    median_age=data.get("median_age"),
                    average_dwelling_value=data.get("average_dwelling_value"),
                )
                records.append(record)
            except Exception as e:
                logger.debug(f"Skipping neighbourhood {neighbourhood_name}: {e}")

        if unmatched:
            logger.warning(
                f"Could not match {len(unmatched)} neighbourhoods: {unmatched[:5]}..."
            )

        logger.info(f"Parsed {len(records)} census records for year {year}")
        return records

    def get_parks(self) -> list[AmenityRecord]:
        """Fetch park locations.

        Returns:
            List of validated AmenityRecord objects.
        """
        return self._fetch_amenities(
            self.DATASETS["parks"],
            AmenityType.PARK,
            name_field="ASSET_NAME",
            address_field="ADDRESS_FULL",
        )

    def get_schools(self) -> list[AmenityRecord]:
        """Fetch school locations.

        Returns:
            List of validated AmenityRecord objects.
        """
        return self._fetch_amenities(
            self.DATASETS["schools"],
            AmenityType.SCHOOL,
            name_field="NAME",
            address_field="ADDRESS_FULL",
        )

    def get_childcare_centres(self) -> list[AmenityRecord]:
        """Fetch licensed childcare centre locations.

        Returns:
            List of validated AmenityRecord objects.
        """
        return self._fetch_amenities(
            self.DATASETS["childcare"],
            AmenityType.CHILDCARE,
            name_field="LOC_NAME",
            address_field="ADDRESS",
        )

    def _fetch_amenities(
        self,
        package_id: str,
        amenity_type: AmenityType,
        name_field: str,
        address_field: str,
    ) -> list[AmenityRecord]:
        """Fetch and parse amenity data from GeoJSON.

        Args:
            package_id: CKAN package ID.
            amenity_type: Type of amenity.
            name_field: Property name containing amenity name.
            address_field: Property name containing address.

        Returns:
            List of AmenityRecord objects.
        """
        try:
            geojson = self._fetch_geojson(package_id)
        except (httpx.HTTPError, ValueError) as e:
            logger.warning(f"Could not fetch {package_id}: {e}")
            return []

        features = geojson.get("features", [])
        records = []

        for feature in features:
            props = feature.get("properties", {})
            geometry = feature.get("geometry")

            # Get coordinates from geometry
            lat, lon = None, None
            if geometry and geometry.get("type") == "Point":
                coords = geometry.get("coordinates", [])
                if len(coords) >= 2:
                    lon, lat = coords[0], coords[1]

            # Try to determine neighbourhood_id
            # Many datasets include AREA_ID or similar
            neighbourhood_id = (
                props.get("AREA_ID")
                or props.get("area_id")
                or props.get("NEIGHBOURHOOD_ID")
                or 0  # Will need spatial join if not available
            )

            name = props.get(name_field) or props.get(name_field.lower()) or "Unknown"
            address = props.get(address_field) or props.get(address_field.lower())

            # Skip if we don't have a neighbourhood assignment
            if neighbourhood_id == 0:
                continue

            records.append(
                AmenityRecord(
                    neighbourhood_id=int(neighbourhood_id),
                    amenity_type=amenity_type,
                    amenity_name=str(name)[:200],
                    address=str(address)[:300] if address else None,
                    latitude=Decimal(str(lat)) if lat else None,
                    longitude=Decimal(str(lon)) if lon else None,
                )
            )

        logger.info(f"Parsed {len(records)} {amenity_type.value} records")
        return records
