"""Fetch AI-related standards metadata from ISO/IEC JTC 1/SC 42.

ISO provides open data (JSON/CSV/Parquet) for metadata but full text is paywalled.
We use the open data portal to catalog SC 42 standards and supplement with
publicly available scope/abstract from the ISO Online Browsing Platform (OBP).
"""

from __future__ import annotations

import csv
import io
import re
import time as time_mod

import httpx
from rich.console import Console

from ..config import Config
from .base import SourceDocument

console = Console()

# ISO Open Data Portal — deliverables metadata
ISO_OPEN_DATA_CSV = "https://isopublicstorageprod.blob.core.windows.net/opendata/_latest/iso_deliverables_metadata/csv/iso_deliverables_metadata.csv"

# SC 42 is the AI committee under JTC 1
SC42_COMMITTEE = "ISO/IEC JTC 1/SC 42"

# Additional AI-relevant committees
AI_COMMITTEES = [
    "ISO/IEC JTC 1/SC 42",  # Artificial Intelligence
    "ISO/IEC JTC 1/SC 27",  # Information security (AI trust/privacy overlap)
]

# Known key SC 42 standards with abstracts (since open data lacks abstracts)
ISO_AI_CATALOG = [
    ("ISO/IEC 42001:2023", "Information technology — Artificial intelligence — Management system",
     "Specifies requirements for establishing, implementing, maintaining, and continually improving an AI management system within organizations."),
    ("ISO/IEC 22989:2022", "Information technology — Artificial intelligence — AI concepts and terminology",
     "Establishes terminology and describes concepts in the field of artificial intelligence."),
    ("ISO/IEC 23894:2023", "Information technology — Artificial intelligence — Guidance on risk management",
     "Provides guidance on managing risk related to development and use of AI systems."),
    ("ISO/IEC 23053:2022", "Framework for Artificial Intelligence (AI) Systems Using Machine Learning (ML)",
     "Establishes a framework describing a generic AI system using ML technology."),
    ("ISO/IEC 38507:2022", "Information technology — Governance of IT — Governance implications of the use of AI",
     "Provides guidance on the governance implications of AI for governing bodies of organizations."),
    ("ISO/IEC 5338:2023", "Information technology — AI system life cycle processes",
     "Defines processes and their associated activities for the development and operation of AI systems."),
    ("ISO/IEC 5339:2024", "Information technology — AI — Guidance for AI applications",
     "Provides guidance on how to apply AI within organizations, including use case analysis and risk assessment."),
    ("ISO/IEC 42005:2025", "Information technology — AI — AI system impact assessment",
     "Provides guidance for assessing the potential positive and negative impacts of AI systems."),
    ("ISO/IEC TR 24028:2020", "Overview of trustworthiness in artificial intelligence",
     "Provides an overview of trustworthiness in AI, including risks, challenges, and approaches."),
    ("ISO/IEC TR 24029-1:2021", "Assessment of the robustness of neural networks — Part 1: Overview",
     "Provides background on properties of neural network robustness and methods for assessment."),
    ("ISO/IEC TR 24030:2021", "Artificial intelligence — Use cases",
     "Provides a collection of representative use cases of AI applications across various domains."),
    ("ISO/IEC TS 6254:2024", "Objectives and approaches for explainability of ML models and AI systems",
     "Describes objectives and approaches for explainability of machine learning models and AI systems."),
    ("ISO/IEC 12792:2024", "Transparency taxonomy of AI systems",
     "Establishes a transparency taxonomy for AI systems to support understanding and governance."),
]

ISO_OBP_URL = "https://www.iso.org/standard/"


def _iso_id_to_name(iso_id: str) -> str:
    """Convert ISO ID to slug. E.g. 'ISO/IEC 42001:2023' -> 'iso-iec-42001-2023'."""
    slug = iso_id.lower().replace("/", "-").replace(" ", "-").replace(":", "-")
    return slug


class ISOFetcher:
    """Fetch AI-related standards from ISO/IEC.

    Combines:
    1. Open data CSV for discovering SC 42 standards
    2. Curated catalog with known abstracts
    3. OBP scraping for scope text of discovered standards
    """

    def __init__(self, config: Config | None = None):
        self.config = config or Config.load()
        self.client = httpx.Client(timeout=30, follow_redirects=True)

    def search(
        self, keywords: list[str], since: str | None = None
    ) -> list[SourceDocument]:
        """Return AI-relevant ISO/IEC standards."""
        seen: dict[str, SourceDocument] = {}

        # Strategy 1: Curated catalog (with real abstracts)
        console.print("  Loading ISO/IEC SC 42 catalog...")
        for iso_id, title, abstract in ISO_AI_CATALOG:
            name = _iso_id_to_name(iso_id)
            seen[name] = SourceDocument(
                name=name,
                title=f"{iso_id}: {title}",
                abstract=abstract,
                source="iso",
                source_id=iso_id,
                source_url=f"https://www.iso.org/standard/{iso_id.split(':')[0].replace('/', '%2F').replace(' ', '%20')}.html",
                time=self._extract_year(iso_id),
                doc_status="published",
            )

        # Strategy 2: Try to download open data CSV for additional SC 42 standards
        console.print("  Fetching ISO open data for SC 42 standards...")
        open_data_docs = self._fetch_open_data(keywords, since)
        for doc in open_data_docs:
            if doc.name not in seen:
                seen[doc.name] = doc

        console.print(f"  Found [bold green]{len(seen)}[/] ISO/IEC AI standards")
        return list(seen.values())

    def _extract_year(self, iso_id: str) -> str:
        """Extract year from ISO ID like 'ISO/IEC 42001:2023'."""
        if ":" in iso_id:
            return iso_id.split(":")[-1]
        return ""

    def _fetch_open_data(self, keywords: list[str], since: str | None) -> list[SourceDocument]:
        """Fetch ISO open data CSV and filter for AI standards."""
        docs = []
        try:
            console.print("  Downloading ISO deliverables catalog (CSV)...")
            resp = self.client.get(ISO_OPEN_DATA_CSV, timeout=60)
            resp.raise_for_status()

            reader = csv.DictReader(io.StringIO(resp.text))
            ai_keywords = ["artificial intelligence", "machine learning", "neural network",
                           "ai system", "trustworth", "autonomous"]

            for row in reader:
                title = row.get("title.en", "")
                committee = row.get("ownerCommittee", "")
                ref = row.get("reference", "")
                status = row.get("currentStage", "")
                pub_date = row.get("publicationDate", "")
                scope = row.get("scope.en", "")

                # Filter: SC 42 committee OR AI keywords in title
                is_sc42 = "SC 42" in committee
                has_ai_keyword = any(kw in title.lower() for kw in ai_keywords)
                if not (is_sc42 or has_ai_keyword):
                    continue

                if since and pub_date and pub_date < since:
                    continue

                name = _iso_id_to_name(ref)
                abstract = scope[:2000] if scope else f"ISO/IEC standard: {title}. Committee: {committee}."
                docs.append(SourceDocument(
                    name=name,
                    title=f"{ref}: {title}",
                    abstract=abstract,
                    source="iso",
                    source_id=ref,
                    source_url=f"https://www.iso.org/standard/{ref.split(':')[0].replace('/', '%2F').replace(' ', '%20')}.html",
                    time=pub_date or self._extract_year(ref),
                    doc_status=status.lower() if status else "published",
                ))

        except httpx.HTTPError as e:
            console.print(f"[yellow]Could not fetch ISO open data: {e}[/]")
        except Exception as e:
            console.print(f"[yellow]Error parsing ISO CSV: {e}[/]")

        return docs

    def download_text(self, doc: SourceDocument) -> str | None:
        """ISO full text is paywalled. Return abstract/scope only."""
        # Try to scrape scope from ISO website
        iso_id = doc.source_id.split(":")[0]  # e.g. "ISO/IEC 42001"
        try:
            # The OBP has scope text for some standards
            url = f"https://www.iso.org/standard/{iso_id.replace('/', '%2F').replace(' ', '%20')}.html"
            resp = self.client.get(url)
            if resp.status_code == 200:
                # Extract scope/abstract from page
                scope_match = re.search(
                    r'<div[^>]*id="item-abstract"[^>]*>(.*?)</div>',
                    resp.text, re.DOTALL,
                )
                if scope_match:
                    scope = re.sub(r'<[^>]+>', '', scope_match.group(1)).strip()
                    if len(scope) > 30:
                        return scope[:5000]
            time_mod.sleep(0.5)
        except httpx.HTTPError:
            pass
        return None

    def close(self) -> None:
        self.client.close()
