"""Check the complete matrix against its full paths and model baselines.

Run from any directory with Python 3; no third-party packages are required.
This validates the released outputs, not a fresh terrain calculation. See
Supplementary_Methods.pdf for full calculation commands.
"""
from pathlib import Path
from collections import Counter
from html.parser import HTMLParser
from urllib.parse import urlsplit, unquote
import json, math, hashlib, re, argparse
from zipfile import ZipFile
from xml.etree import ElementTree as ET
import posixpath

parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--served-html', action='store_true', help='Check a downloaded deployment, allowing only recorded Netlify HTML variants and omission of its two private configuration files.')
args = parser.parse_args()
SITE = Path(__file__).resolve().parents[1]
def read(name):
    return json.loads((SITE / name).read_text())
def close(a, b):
    assert math.isclose(a, b, rel_tol=1e-12, abs_tol=1e-10), (a, b)

def worksheet_records(filename, sheet_name):
    """Read the stored values of an XLSX table using the Python standard library."""
    ns = {'s': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
    rel_ns = '{http://schemas.openxmlformats.org/package/2006/relationships}'
    rid = '{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id'
    with ZipFile(SITE / filename) as archive:
        workbook = ET.fromstring(archive.read('xl/workbook.xml'))
        sheet = next(s for s in workbook.findall('s:sheets/s:sheet', ns) if s.get('name') == sheet_name)
        rels = ET.fromstring(archive.read('xl/_rels/workbook.xml.rels'))
        target = next(r.get('Target') for r in rels.findall(rel_ns + 'Relationship') if r.get('Id') == sheet.get(rid))
        path = target.lstrip('/') if target.startswith('/') else posixpath.normpath('xl/' + target)
        shared = []
        if 'xl/sharedStrings.xml' in archive.namelist():
            shared_root = ET.fromstring(archive.read('xl/sharedStrings.xml'))
            shared = [''.join(t.text or '' for t in item.findall('.//s:t', ns)) for item in shared_root]
        root = ET.fromstring(archive.read(path))
        rows = []
        for row in root.findall('s:sheetData/s:row', ns):
            cells = {}
            for cell in row.findall('s:c', ns):
                letters = re.match(r'[A-Z]+', cell.get('r')).group()
                column = 0
                for letter in letters:
                    column = 26 * column + ord(letter) - ord('A') + 1
                kind = cell.get('t'); value = cell.findtext('s:v', namespaces=ns)
                if kind == 'inlineStr':
                    value = ''.join(t.text or '' for t in cell.findall('.//s:t', ns))
                elif kind == 's':
                    value = shared[int(value)]
                elif kind == 'b':
                    value = value == '1'
                elif value is not None and kind not in ('str', 'e'):
                    value = float(value)
                cells[column] = value
            if any(value not in (None, '') for value in cells.values()):
                rows.append(cells)
    header = rows[0]
    assert len(set(header.values())) == len(header), sheet_name
    return [dict((key, row.get(column)) for column, key in header.items()) for row in rows[1:]]

def same_cell(actual, expected, context):
    if isinstance(expected, (list, dict)):
        actual = json.loads(actual)
    if actual in (None, '') and expected in (None, ''):
        return
    if isinstance(expected, bool):
        assert actual is expected, context
    elif isinstance(expected, (float, int)):
        assert isinstance(actual, (float, int)) and not isinstance(actual, bool), context
        close(actual, expected)
    else:
        assert actual == expected, (context, actual, expected)

local_records = worksheet_records('data/Supplementary_Table_S1.xlsx', 'Local_Cases')
assert len(local_records) == 24
local_index = {r['site_id']: r for r in local_records}
assert len(local_index) == 24
evidence_cells_checked = 0
for export in ['data/evidence/local_cases.json', 'source/local_cases.json']:
    cases = read(export)
    assert len(cases) == 24 and {c['site_id'] for c in cases} == set(local_index)
    for case in cases:
        row = local_index[case['site_id']]
        assert set(case) <= set(row), (case['site_id'], set(case) - set(row))
        # The workbook also has source-check notes that are not map export
        # fields. Check every exported field against its worksheet value.
        for key, expected in case.items():
            same_cell(row[key], expected, (export, case['site_id'], key))
            evidence_cells_checked += 1

routes = read('source/map_routes.json')
paths = read('data/complete_matrix/primary_complete_paths.json')['routes']
metrics = read('data/complete_matrix/distance_metrics_all_76.json')
table = read('source/map_table_s2.json')
summary = read('data/complete_matrix/analysis_summary.json')
table_cells_checked = 0
for sheet_name, records, key in [('Table S2', table['rows'], 'route_key'), ('Source contrasts', table['contrasts'], 'candidate_route')]:
    rows = worksheet_records('data/Supplementary_Table_S2.xlsx', sheet_name)
    assert len(rows) == len(records), sheet_name
    row_index = {r[key]: r for r in rows}
    assert len(row_index) == len(records), sheet_name
    for record in records:
        row = row_index[record[key]]
        assert set(row) == set(record), (sheet_name, record[key])
        for field, expected in record.items():
            same_cell(row[field], expected, (sheet_name, record[key], field))
            table_cells_checked += 1
assert len(routes) == len(paths) == len(table['rows']) == 22
assert Counter(p['analysis_role'] for p in paths) == {'principal': 9, 'supplementary': 9, 'regional comparator': 4}
index = {p['key']: p for p in paths}
assert len(index) == 22
for target in ['harbetsuvan', 'karahan', 'gobekli', 'sefer', 'tasli', 'hamzan']:
    selected = [p for p in paths if p['target_key'] == target]
    assert {p['source_key'] for p in selected} == {'gollu', 'bingol', 'nemrut'}
    assert len(selected) == 3
    role = 'principal' if target in ['harbetsuvan', 'karahan', 'gobekli'] else 'supplementary'
    assert all(p['analysis_role'] == role for p in selected)
    if target in ['tasli', 'hamzan']:
        assert all(p['endpoint_status'] == 'illustrative' for p in selected)
for r in routes:
    p = index[r['key']]
    assert r['path'] == p['archived_display_coordinates']
    assert r['analysis_role'] == p['analysis_role']
    for field in ['grid_path_length_km', 'source_connector_km', 'target_connector_km', 'total_path_length_km']:
        close(r[field], p[field])
    close(r['composite_cost_index'], p['reproduced_composite_cost'])
    close(p['total_path_length_km'], p['grid_path_length_km'] + p['source_connector_km'] + p['target_connector_km'])
for row in table['rows']:
    p = index[row['route_key']]
    for field in ['grid_path_length_km', 'source_connector_km', 'target_connector_km', 'total_path_length_km', 'straight_line_km']:
        close(row[field], p[field])
    close(row['endpoint_connectors_km'], p['source_connector_km'] + p['target_connector_km'])
    close(row['composite_cost_index'], p['reproduced_composite_cost'])
    close(row['cost_relative_to_harb_central_100'], 100 * p['reproduced_composite_cost'] / index['harb-central']['reproduced_composite_cost'])
    assert row['analysis_role'] == p['analysis_role']
assert len(metrics) == 76
runs = Counter(m['run'] for m in metrics)
assert runs.pop('primary') == 22 and len(runs) == 6 and set(runs.values()) == {9}
by_run = {(m['run'], m['key']): m for m in metrics}
assert len(by_run) == 76
for run in runs:
    assert {m['key'] for m in metrics if m['run'] == run} == {p['key'] for p in paths if p['analysis_role'] == 'principal'}
# Check every stored path against the corresponding summary row. The complete
# outputs include the 56 baseline scenarios, so no superseded audit is needed.
full_by_run = {}
for run in ['primary', *runs]:
    output = read(f'data/complete_matrix/{run}_complete_paths.json')
    assert output['configuration']['key'] == run
    for p in output['routes']:
        key = (run, p['key'])
        assert key not in full_by_run
        full_by_run[key] = p
        m = by_run[key]
        for field, value in m.items():
            assert p[field] == value, (key, field)
        assert p['expanded_vertex_count'] == len(p['expanded_grid_cells']) == len(p['expanded_grid_coordinates'])
        assert p['parent_vertex_count'] == len(p['parent_grid_cells'])
        close(p['total_path_length_km'], p['grid_path_length_km'] + p['source_connector_km'] + p['target_connector_km'])
        close(p['costed_edge_sum'], p['reproduced_composite_cost'])
assert len(full_by_run) == 76
baselines = [read('source/terrain_model/least_cost_path_outputs.json')]
baselines += read('source/terrain_model/least_cost_path_sensitivity.json')['runs']
baseline_checks = 0
for baseline_index, baseline in enumerate(baselines):
    run = 'primary' if baseline_index == 0 else baseline['run_label']
    for old in baseline['routes']:
        key = (run, old['key'])
        p = full_by_run[key]
        assert p['historical_scenario'] and p['display_geometry_exact_match']
        assert p['archived_display_coordinates'] == old['path'], key
        close(p['reproduced_composite_cost'], old['composite_cost_index'])
        close(p['archived_composite_cost'], old['composite_cost_index'])
        close(p['archived_display_length_km'], old['length_km'])
        close(p['reproduced_display_length_km'], old['length_km'])
        baseline_checks += 1
assert baseline_checks == 56 == sum(p['historical_scenario'] for p in full_by_run.values())
contrasts = read('data/complete_matrix/source_contrasts.json')
assert len(contrasts) == 42 and len(table['contrasts']) == 6
for c in contrasts:
    a = by_run[c['run'], c['near_route']]['total_path_length_km']
    b = by_run[c['run'], c['reference_route']]['total_path_length_km']
    close(c['new_total_reduction_pct'], 100 * (1 - a / b))
    assert a < b
for c in table['contrasts']:
    close(c['total_path_reduction_percent'], 100 * (1 - index[c['candidate_route']]['total_path_length_km'] / index[c['reference_route']]['total_path_length_km']))
assert summary['original_files_unchanged'] and summary['maximum_cost_difference'] == 0
assert summary['sensitivity_ordering_retained']
fig = read('figures/figure_manifest.json')['figure_2']
assert fig['displayed_scenarios'] == 14 and fig['calculated_scenarios'] == 22
assert set(fig['display_subset_keys']) == {r['key'] for r in routes if r['figure_2_displayed']}
assert Counter(index[k]['analysis_role'] for k in fig['display_subset_keys']) == {'principal': 7, 'supplementary': 3, 'regional comparator': 4}

class Links(HTMLParser):
    def __init__(self):
        super().__init__(); self.local = set()
    def handle_starttag(self, tag, attrs):
        for key, value in attrs:
            if key in ['href', 'src'] and value and not urlsplit(value).scheme and not value.startswith(('#', '//')):
                self.local.add(unquote(urlsplit(value).path))
html = (SITE / 'index.html').read_text()
if args.served_html:
    # Restore only the observed link spellings for local file-existence checks.
    # The original bytes are checked separately against the manifest below.
    for served, authored in [("href='/figures'", 'href="figures.html"'),
                             ("href='/readme'", 'href="README.html"'),
                             ("href='/'", 'href="index.html"')]:
        html = html.replace(served, authored)
links = Links(); links.feed(html)
# Source-list links are added by JavaScript and are not HTML href attributes.
for group in read('source/map_sourceIndex.json'):
    for item in group['items']:
        value = item.get('url', '')
        if value and not urlsplit(value).scheme and not value.startswith(('#', '//')):
            links.local.add(unquote(urlsplit(value).path))
assert all((SITE / p).is_file() for p in links.local), sorted(p for p in links.local if not (SITE / p).is_file())
assert '22 least-cost path scenarios' in html and 'nine principal' in html
assert 'Figure 2 displays 14 paths: seven principal, three supplementary and four regional comparator paths.' in html
assert not re.search(r'\b(?:fourteen scenarios|14 scenarios)\b', html, flags=re.I)
manifest = SITE / 'build_manifest.json'
checked_hashes = 0
skipped_private_configuration = []
if manifest.exists():
    m = read('build_manifest.json')
    for file, expected in m['asset_sha256'].items():
        if args.served_html and file in ('_redirects', 'netlify.toml') and not (SITE / file).exists():
            skipped_private_configuration.append(file)
            continue
        data = (SITE / file).read_bytes()
        accepted = {expected}
        if file.endswith('.html') and args.served_html:
            data = re.sub(rb'(?m)^<!-- This site is hosted on Netlify\. Anyone can build and deploy a site\r?\n     like this one for free: https://netlify\.new/\?[^\r\n<>]*\r?\n     Netlify hosting facts for this site: static/SSR served via Netlify Edge\. -->\r?\n', b'', data)
            served_hash = m.get('netlify_served_html_sha256', {}).get(file)
            if served_hash:
                accepted.add(served_hash)
        assert hashlib.sha256(data).hexdigest() in accepted, file
        checked_hashes += 1
report = {'base_paths': 22, 'principal_paths': 9, 'supplementary_paths': 9, 'regional_paths': 4,
          'sensitivity_runs': 54, 'principal_contrasts': 6, 'model_baselines_checked': baseline_checks,
          'figure_2_display_subset': 14, 'all_complete_paths_and_table_values_match': True,
          'all_eastern_central_orderings_retained': True, 'local_links_checked': len(links.local),
          'local_evidence_export_cells_checked': evidence_cells_checked,
          'table_s2_excel_cells_checked': table_cells_checked,
          'manifest_hashes_checked': checked_hashes,
          'private_configuration_files_not_served': skipped_private_configuration}
print(json.dumps(report, indent=2))