"""
Extract positional data from Calgary Highlanders War Diary Sep 44 (pages 7-57).
Outputs a JSON array of position objects.
"""

import re
import json
from html.parser import HTMLParser

INPUT_FILE = r"C:\Users\natha\IdeaProjects\AI-Prototype\Inputs\ocr-output\Calgary-Highlanders_War-Diary_Sep44_olmocr.md"
OUTPUT_FILE = r"C:\Users\natha\IdeaProjects\AI-Prototype\outputs\Calgary-Highlanders_Sep44_positions.json"

# ── helpers ──────────────────────────────────────────────────────────────────

def expand_grid(raw: str) -> tuple[str, bool]:
    """
    Return (6-figure-string, inferred).
    4-figure AABB  → centre of 1km square = AA5 BB5  (grid_inferred=True)
    6-figure AAABBB → returned as-is                 (grid_inferred=False)
    8-figure AAAABBBB → truncate to AAA BBB           (grid_inferred=False)
    """
    s = re.sub(r'[^0-9]', '', raw)
    if len(s) == 4:
        # e.g. "8450" → easting 84, northing 50 → centre 845, 505
        e = s[0:2] + '5'
        n = s[2:4] + '5'
        return e + n, True
    if len(s) == 6:
        return s, False
    if len(s) == 8:
        # AAAABBBB: easting = AAAA (take first 3), northing = BBBB (take first 3)
        # e.g. 15618050 → E=1561 → 156, N=8050 → 805 → "156805"
        e = s[0:3]
        n = s[4:7]
        return e + n, False
    return s, False


_GRID_RE = re.compile(
    r'\b'
    r'(?:MR\s*|GR\s*)?'          # optional prefix
    r'([0-9]{3,4}\s*[0-9]{3,4})'  # 6 or 4+4 digit grid
    r'\b',
    re.IGNORECASE
)

def find_grids(text: str):
    """Return list of (raw_match, cleaned_digits)."""
    found = []
    for m in _GRID_RE.finditer(text):
        raw = m.group(1)
        digits = re.sub(r'\s', '', raw)
        if len(digits) in (4, 6, 8):
            found.append((m.group(0).strip(), digits))
    # also pick up standalone 6-digit runs not already caught
    for m in re.finditer(r'\b([0-9]{6})\b', text):
        digits = m.group(1)
        already = any(d == digits for _, d in found)
        if not already:
            found.append((digits, digits))
    return found


# ── HTML strip ────────────────────────────────────────────────────────────────

class _Stripper(HTMLParser):
    def __init__(self):
        super().__init__()
        self.parts = []
    def handle_data(self, data):
        self.parts.append(data)
    def get_text(self):
        return ' '.join(p for p in self.parts if p.strip())

def strip_html(html_str: str) -> str:
    s = _Stripper()
    # replace <br> with spaces for readability
    html_str = re.sub(r'<br\s*/?>', ' ', html_str, flags=re.IGNORECASE)
    s.feed(html_str)
    return s.get_text()


# ── categorise ───────────────────────────────────────────────────────────────

_SUBUNIT_RE = re.compile(
    r'\b(Able\s+Coy|Baker\s+Coy|Charlie\s+Coy|Dog\s+Coy|'
    r'"A"\s+Coy|"B"\s+Coy|"C"\s+Coy|"D"\s+Coy|'
    r"'A'\s+Coy|'B'\s+Coy|'C'\s+Coy|'D'\s+Coy|'Able'\s+Coy|'Baker'\s+Coy|'Charlie'\s+Coy|'Dog'\s+Coy|"
    r'Carrier\s+Platoon|carriers|Pioneer\s+Platoon|pioneers|scouts?|Scout\s+Platoon|'
    r'Support\s+Coy|Anti[-\s]?[Tt]ank\s+Pl(?:atoon)?|mortar\s+platoon|'
    r'\d+\s*Pl(?:atoon)?|'
    r'\d+\s*Sec(?:tion)?)\b',
    re.IGNORECASE
)
_ENEMY_RE = re.compile(
    r'\b(enemy|Jerry|Hun|Boche|MG\s*pos|MMG\s*pos|sniper|'
    r'counter.attack|strongpoint|mortar\s*pos|machine\s*gun|'
    r'block\s*house|pill\s*box|88mm|S\.S\.|SS troop)\b',
    re.IGNORECASE
)
_PATROL_RE = re.compile(
    r'\b(patrol|recce\s*patrol|fighting\s*patrol|'
    r'standing\s*patrol|OP|O\.P\.|observation\s*post)\b',
    re.IGNORECASE
)
_FRIENDLY_UNITS = re.compile(
    r'\b(R\.H\.C\.|Black\s*Watch|RHC|R\s*de\s*Mais|'
    r'Fus\s*M\.R\.|F\.M\.R\.|R\.R\.C\.|'
    r'4\s*S\.S\.|Royal\s*Regt|S\.Sask|'
    r'Tor\s*Scots|Toronto\s*Scots|'
    r'White\s*Brig(?:ade)?|F\.F\.I\.|Maquis)\b',
    re.IGNORECASE
)
# Tac HQ and Bn HQ in narrative
_TAC_HQ_RE = re.compile(
    r'\bTac\s*H(?:Q)?\b',
    re.IGNORECASE
)
_BN_HQ_RE = re.compile(
    r'\b(?:Bn\.?\s*H\.?Q\.?|Battalion\s*H\.?Q\.?|'
    r'Battle\s*H\.?Q\.?|Command\s*Post|'
    r'moved\s+(?:his\s+)?H\.?Q\.|'
    r'set\s+up\s+H\.?Q\.|'
    r'took\s+up\s+(?:a\s+)?H\.?Q\.)\b',
    re.IGNORECASE
)

def categorise(sentence: str, place_name: str | None, subunit: str | None,
               friendly: str | None, is_place_col: bool = False) -> str:
    """
    Categories (in priority order):
      TAC_HQ      – Tactical HQ position
      BN_HQ       – Battalion HQ position (incl. all place-column entries)
      FRIENDLY    – another friendly unit's position
      ENEMY       – enemy position / feature
      PATROL      – patrol route/endpoint
      SUBUNIT     – company or platoon position
      UNIT_MOVEMENT – general battalion move/position
      MISC        – catch-all
    """
    if is_place_col:
        # Place column always records where Bn HQ was
        if _TAC_HQ_RE.search(sentence):
            return "TAC_HQ"
        return "BN_HQ"
    if _TAC_HQ_RE.search(sentence):
        return "TAC_HQ"
    if _BN_HQ_RE.search(sentence):
        return "BN_HQ"
    if friendly:
        return "FRIENDLY"
    if _ENEMY_RE.search(sentence):
        return "ENEMY"
    if _PATROL_RE.search(sentence):
        return "PATROL"
    if subunit:
        return "SUBUNIT"
    return "UNIT_MOVEMENT"


def extract_subunit(sentence: str) -> str | None:
    m = _SUBUNIT_RE.search(sentence)
    if m:
        return m.group(0).strip()
    return None

def extract_friendly(sentence: str) -> str | None:
    m = _FRIENDLY_UNITS.search(sentence)
    if m:
        return m.group(0).strip()
    return None


# ── split sentences ──────────────────────────────────────────────────────────

def split_sentences(text: str):
    """Rough sentence splitter – split on '. ' or '<br>'."""
    # normalise
    text = re.sub(r'\s+', ' ', text).strip()
    parts = re.split(r'(?<=[.!?])\s+(?=[A-Z"\'(])', text)
    return [p.strip() for p in parts if p.strip()]


# ── page extractor ────────────────────────────────────────────────────────────

_PAGE_RE = re.compile(r'^## Page (\d+)\s*$', re.MULTILINE)

def extract_pages(text: str, first: int, last: int) -> str:
    pages = list(_PAGE_RE.finditer(text))
    start_idx = None
    end_idx = len(text)
    for i, m in enumerate(pages):
        n = int(m.group(1))
        if n == first and start_idx is None:
            start_idx = m.start()
        if n == last + 1 and start_idx is not None:
            end_idx = m.start()
            break
    if start_idx is None:
        return ""
    return text[start_idx:end_idx]


# ── table row parser ──────────────────────────────────────────────────────────

_ROW_RE = re.compile(r'<tr>(.*?)</tr>', re.DOTALL | re.IGNORECASE)
_CELL_RE = re.compile(r'<t[dh][^>]*>(.*?)</t[dh]>', re.DOTALL | re.IGNORECASE)

def parse_table_rows(table_html: str) -> list[dict]:
    """Parse a single HTML table into list of {place, date, hour, summary} dicts."""
    rows = []
    for row_m in _ROW_RE.finditer(table_html):
        cells = [strip_html(c.group(1)).strip()
                 for c in _CELL_RE.finditer(row_m.group(1))]
        if len(cells) < 4:
            continue
        # skip header rows
        if re.match(r'place|date|hour|summary|no\.', cells[0], re.IGNORECASE):
            continue
        place = cells[0] if cells[0] else None
        date  = cells[1] if len(cells) > 1 else None
        hour  = cells[2] if len(cells) > 2 else None
        summary = cells[3] if len(cells) > 3 else ""
        rows.append(dict(place=place, date=date, hour=hour, summary=summary))
    return rows


# ── sheet ref extractor ───────────────────────────────────────────────────────
_SHEET_RE = re.compile(
    # must start with a digit to avoid matching "Sheet Ste. Foy" etc.
    r'Sheet\s+(\d[A-Z0-9]*(?:\s*[&\-]\s*\d[A-Z0-9]*)?)',
    re.IGNORECASE
)

def extract_sheet(place_text: str) -> str | None:
    if not place_text:
        return None
    # find the LAST occurrence (most specific sheet reference in column)
    matches = list(_SHEET_RE.finditer(place_text))
    if matches:
        return "Sheet " + matches[-1].group(1).strip()
    return None

# also pull MR grid from place column
_PLACE_MR_RE = re.compile(r'MR\s*([0-9]{4,8}|\d{2,3}\s*\d{2,3})', re.IGNORECASE)

def extract_place_name(place_text: str) -> str | None:
    """
    Extract the primary place name from the Place column.
    Strip country, bare MR tokens, Sheet refs, standalone digits,
    and Tac H / Fort coord suffixes.
    """
    if not place_text:
        return None
    s = place_text
    # remove country names
    s = re.sub(r'\b(France|Belgium|Holland|Netherlands)\b', '', s, flags=re.IGNORECASE)
    # remove Sheet + value (digit-led)
    s = re.sub(r'\bSheet\s+\d[\w\s&\-]*', ' ', s, flags=re.IGNORECASE)
    # remove bare "Sheet" not followed by a digit
    s = re.sub(r'\bSheet\b', ' ', s, flags=re.IGNORECASE)
    # remove MR + optional digits/spaces
    s = re.sub(r'\bMR\b\s*[\d\s]*', ' ', s, flags=re.IGNORECASE)
    # remove Tac H + coords
    s = re.sub(r'\bTac\s*H\s*[\d]+\b', ' ', s, flags=re.IGNORECASE)
    # remove bare 4-8 digit grid refs
    s = re.sub(r'\b\d{4,8}\b', ' ', s)
    # remove "X Pub" and similar codes
    s = re.sub(r'\bX[\s\-]?Pub\b', ' ', s, flags=re.IGNORECASE)
    # remove orphan punctuation / collapse whitespace
    s = re.sub(r'[/\\|]', ' ', s)
    s = re.sub(r'\s+', ' ', s).strip().strip('.,-()')
    # If multiple tokens, take first meaningful phrase (up to the first double space separator)
    parts = [p.strip() for p in re.split(r'\s{2,}', s) if p.strip()]
    result = parts[0] if parts else s.strip()
    # Trim trailing stray words like "(outskirts...)" parenthetical noise
    result = re.sub(r'\s*\(.*\)\s*$', '', result).strip()
    return result if result else None


def clean_date(raw: str) -> str | None:
    if not raw:
        return None
    d = re.sub(r'\s+', ' ', raw).strip()
    # Remove parenthetical notes like "(Cont)"
    d = re.sub(r'\(.*?\)', '', d).strip()
    # Extract DD Mon [YY] — ignore anything else in the cell (e.g. embedded grid refs)
    m = re.search(
        r'(\d{1,2})\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s*(44|1944)?',
        d, re.IGNORECASE
    )
    if m:
        day = m.group(1)
        mon = m.group(2).capitalize()
        # always use "44" as year for this diary
        return f"{day} {mon} 44"
    return None   # unrecognisable — don't inherit garbage


def clean_hour(raw: str) -> str | None:
    if not raw:
        return None
    raw = raw.strip()
    # Reject empty, the bare year "44", and plain digit-strings ≥4 chars (grids)
    if not raw or raw in ('0', '44', '1944'):
        return None
    if re.match(r'^\d{4,6}$', raw):
        return None   # grid ref leaked into hour column, not a time
    # Extract a valid HHMM time
    m = re.search(r'\b([012]\d[0-5]\d)\b', raw)
    if m:
        return m.group(1)
    return None


def hour_col_grid(raw: str) -> str | None:
    """
    If the Hour column contains a plain 4-digit grid ref (not a time),
    return the digit string so it can be added as a BN_HQ entry.
    """
    if not raw:
        return None
    raw = raw.strip()
    if re.match(r'^\d{4}$', raw):
        # Confirm it can't be a time (hours > 23 or minutes > 59)
        h, m2 = int(raw[:2]), int(raw[2:])
        if h > 23 or m2 > 59:
            return raw
    return None


# ── place-column MR grid ──────────────────────────────────────────────────────

def place_col_positions(place_text: str, date: str, sheet: str) -> list[dict]:
    """
    Extract grids from the Place column.
    Handles:
      (a) NAME + MR + GRID  (e.g. 'Ste. Foy MR 2553')
      (b) NAME + bare 6-digit grid (e.g. 'Chateau Helleputte 769969')
    Returns one entry per unique grid found.
    """
    if not place_text:
        return []

    results = []
    seen_digits = set()

    def _clean_label(raw: str) -> str | None:
        s = raw
        s = re.sub(r'\b(France|Belgium|Holland|Netherlands)\b', '', s, flags=re.IGNORECASE)
        s = re.sub(r'\bSheet\b[\w\s&\-]*', '', s, flags=re.IGNORECASE)
        s = re.sub(r'\bMR\b', '', s, flags=re.IGNORECASE)
        s = re.sub(r'\b\d{4,8}\b', '', s)
        s = re.sub(r'\s+', ' ', s).strip().strip('.,-()')
        # take rightmost meaningful word group
        parts = [p.strip() for p in re.split(r'\s{2,}', s) if p.strip()]
        r = parts[-1] if parts else s.strip()
        return r if r else None

    # ── (a) NAME MR GRID ─────────────────────────────────────────────────────
    # Require digits immediately after MR (captures 4-digit and 6-digit grids)
    mr_pattern = re.compile(
        r'([A-Z][A-Za-zÀ-ÿ\s\.\-\']*?)\s+MR\s*(\d[\d\s]+\d)',
        re.IGNORECASE
    )
    for m in mr_pattern.finditer(place_text):
        label_raw = m.group(1)
        raw_digits = re.sub(r'\s', '', m.group(2))
        if len(raw_digits) not in (4, 6, 8):
            continue
        if raw_digits in seen_digits:
            continue
        seen_digits.add(raw_digits)

        grid, inferred = expand_grid(raw_digits)
        if len(grid) != 6:
            grid = None

        label = _clean_label(label_raw)
        # Determine category: place column = BN_HQ unless Tac H explicit
        cat = categorise(place_text, label, None, None, is_place_col=True)
        results.append(dict(
            date=date, hour=None,
            grid=grid, grid_inferred=inferred,
            place_name=label,
            sheet_ref=sheet,
            category=cat,
            subunit=None, friendly_unit=None, is_end_of_day=False,
            context=f"Place column: {label or '?'} MR {raw_digits}"
        ))

    # ── (b) NAME followed directly by bare 6-digit grid ──────────────────────
    bare_pattern = re.compile(
        r'([A-Z][A-Za-zÀ-ÿ\s\.\-\']+?)\s+(\d{6})\b'
    )
    for m in bare_pattern.finditer(place_text):
        label_raw = m.group(1)
        raw_digits = m.group(2)
        if raw_digits in seen_digits:
            continue
        # Skip if label contains only noise words
        clean = re.sub(r'\b(France|Belgium|Holland|Netherlands|Sheet|MR)\b', '',
                       label_raw, flags=re.IGNORECASE).strip()
        if not clean:
            continue
        seen_digits.add(raw_digits)

        grid, inferred = expand_grid(raw_digits)
        label = _clean_label(label_raw)
        cat = categorise(place_text, label, None, None, is_place_col=True)
        results.append(dict(
            date=date, hour=None,
            grid=grid, grid_inferred=inferred,
            place_name=label,
            sheet_ref=sheet,
            category=cat,
            subunit=None, friendly_unit=None, is_end_of_day=False,
            context=f"Place column: {label or '?'} {raw_digits}"
        ))

    # ── (c) NAME followed by bare 4-digit grid (no MR prefix) ────────────────
    bare_4_pattern = re.compile(
        r'([A-Z][A-Za-zÀ-ÿ\s\.\-\']+?)\s+(\d{4})\b'
    )
    for m in bare_4_pattern.finditer(place_text):
        label_raw = m.group(1)
        raw_digits = m.group(2)
        if raw_digits in seen_digits:
            continue
        # Only treat as grid if NOT a valid time (i.e. HH>23 or MM>59)
        h_val, mn_val = int(raw_digits[:2]), int(raw_digits[2:])
        if h_val <= 23 and mn_val <= 59:
            continue
        clean = re.sub(r'\b(France|Belgium|Holland|Netherlands|Sheet|MR)\b', '',
                       label_raw, flags=re.IGNORECASE).strip()
        if not clean:
            continue
        seen_digits.add(raw_digits)

        grid, inferred = expand_grid(raw_digits)
        label = _clean_label(label_raw)
        cat = categorise(label_raw, label, None, None, is_place_col=True)
        results.append(dict(
            date=date, hour=None,
            grid=grid if len(grid) == 6 else None,
            grid_inferred=inferred,
            place_name=label,
            sheet_ref=sheet,
            category=cat,
            subunit=None, friendly_unit=None, is_end_of_day=False,
            context=f"Place column: {label or '?'} {raw_digits}"
        ))

    return results


# ── main extraction ───────────────────────────────────────────────────────────

def extract_positions(md_text: str) -> list[dict]:
    section = extract_pages(md_text, 7, 57)

    # find all tables
    table_re = re.compile(r'<table>(.*?)</table>', re.DOTALL | re.IGNORECASE)

    all_positions = []
    last_date = None
    last_sheet = None

    for tbl_m in table_re.finditer(section):
        tbl_html = tbl_m.group(0)
        rows = parse_table_rows(tbl_html)
        if not rows:
            continue

        for row in rows:
            raw_date = clean_date(row.get('date', '') or '')
            raw_hour = clean_hour(row.get('hour', '') or '')
            place_text = row.get('place', '') or ''
            summary = row.get('summary', '') or ''

            # update tracking state
            if raw_date:
                last_date = raw_date
            cur_date = last_date

            sheet = extract_sheet(place_text) or last_sheet
            if sheet and re.search(r'\d', sheet):
                last_sheet = sheet

            # ── place-column positions ──
            for pos in place_col_positions(place_text, cur_date, sheet):
                pos['hour'] = raw_hour
                all_positions.append(pos)

            # ── hour-column grid (OCR sometimes puts MR ref here) ──
            hr_raw = row.get('hour', '') or ''
            hcg = hour_col_grid(hr_raw)
            if hcg:
                grid, inferred = expand_grid(hcg)
                col_place = extract_place_name(place_text)
                all_positions.append(dict(
                    date=cur_date, hour=None,
                    grid=grid if len(grid) == 6 else None,
                    grid_inferred=inferred,
                    place_name=col_place,
                    sheet_ref=last_sheet,
                    category="BN_HQ",
                    subunit=None, friendly_unit=None, is_end_of_day=False,
                    context=f"Place column (hour field): {col_place or '?'} {hcg}"
                ))

            # ── summary / narrative positions ──
            sentences = split_sentences(summary)
            # track whether this is the last sentence in the entry
            last_sentence_idx = len(sentences) - 1

            # collect all (sentence_idx, grid_str, raw_context) tuples for this row
            row_matches = []
            for s_idx, sent in enumerate(sentences):
                grids = find_grids(sent)
                if grids:
                    for raw_match, digits in grids:
                        row_matches.append((s_idx, sent, digits, raw_match))
                else:
                    # named positions without grids: look for capitalised place names
                    named = re.findall(
                        r'\b(Fme\s+\w+|Chateau\s+\w+|Fort\s+\w+|Casino|'
                        r'Distillery\s+\w+|Brickworks|blockhouse|'
                        r'Mardick|Dunkerque|Dunkirk|Brecht|Loon\s*Plage|'
                        r'Bourbourgville|St\.\s*Folquin|Nordamsques|Montreuil|'
                        r'Wommelg[ea]h?m|Antwerp|Ypres|Pasch[aeo]nd[ae]?le|'
                        r'Gravelines|Le\s*Clipon|Coppenaxfort|'
                        r'Lochtenberg|Eindhoven|Sternhoven|Ryckevorsel|'
                        r'St\.\s*Leonard|Bindhoven|Schilde|Schelde)\b',
                        sent, re.IGNORECASE
                    )
                    for name in named:
                        row_matches.append((s_idx, sent, None, name))

            # determine is_end_of_day: last grid-bearing sentence in last row of date?
            for i, (s_idx, sent, digits, raw_match) in enumerate(row_matches):
                is_last = (i == len(row_matches) - 1)

                # extract hour from sentence if not in column
                hour = raw_hour
                if not hour:
                    h_m = re.search(r'\b([012]\d[0-5]\d)\s*h(?:r|our|s)?', sent, re.IGNORECASE)
                    if h_m:
                        hour = h_m.group(1)
                    else:
                        h_m2 = re.search(r'\b([012]\d[0-5]\d)[A-Z]?\b', sent)
                        if h_m2:
                            candidate = h_m2.group(1)
                            # make sure it looks like a time not a grid
                            if int(candidate[:2]) <= 23 and int(candidate[2:]) <= 59:
                                hour = candidate

                # grid
                if digits:
                    grid, inferred = expand_grid(digits)
                    if len(grid) != 6:
                        grid = None
                        inferred = False
                else:
                    grid = None
                    inferred = False

                # place name from sentence context
                pn_m = re.search(
                    r'\b(Fme\s+\w+[\w\s]+?(?=\s+\d|\s+MR|\.|,|$)|'
                    r'Chateau\s+\w+|Fort\s+\d+|Casino|Distillery\s+\w+|'
                    r'Brickworks|blockhouse|moated\s+farm|'
                    r'Mardick|Dunkerque|Dunkirk|Brecht|Loon\s*Plage|'
                    r'Bourbourgville|St\.\s*Folquin|Nordamsques|Montreuil|'
                    r'Wommelg[ea]h?m|Antwerp|Ypres|Pasch[aeo]nd[ae]?le|'
                    r'Gravelines|Le\s*Clipon|Coppenaxfort|'
                    r'Lochtenberg|Eindhoven|Sternhoven|Ryckevorsel|'
                    r'St\.\s*Leonard|Bindhoven|Schilde|Schelde|'
                    r'cross.?roads?|road\s+junction|road\s+junc\.?|'
                    r'start\s+line|bridge|lock\s+gates|railway\s+st[na]|'
                    r'windmill|windpump|pier|beach|canal)\b',
                    sent, re.IGNORECASE)
                place_name = pn_m.group(0).strip() if pn_m else None

                # also check the place column label
                col_pn = extract_place_name(place_text)
                if not place_name and col_pn:
                    place_name = col_pn

                subunit = extract_subunit(sent)
                friendly = extract_friendly(sent)
                category = categorise(sent, place_name, subunit, friendly, is_place_col=False)

                # shorten context to 2 sentences max
                context = sent.strip()
                if len(context) > 300:
                    context = context[:297] + "..."

                all_positions.append(dict(
                    date=cur_date,
                    hour=hour,
                    grid=grid,
                    grid_inferred=inferred,
                    place_name=place_name,
                    sheet_ref=last_sheet,
                    category=category,
                    subunit=subunit,
                    friendly_unit=friendly,
                    is_end_of_day=False,   # will be set in post-processing
                    context=context
                ))

    return all_positions


# ── EOD post-processing ───────────────────────────────────────────────────────

def assign_end_of_day(positions: list[dict]) -> list[dict]:
    """
    For each date, set is_end_of_day=True on exactly ONE entry — the last
    recorded HQ position for that date (in document order).

    Priority (highest first):
      1. Last place-column TAC_HQ with grid   (context starts "Place column:")
      2. Last place-column BN_HQ with grid
      3. Last place-column BN_HQ or TAC_HQ without grid
      4. Last narrative TAC_HQ with grid
      5. Last narrative BN_HQ with grid
      6. Last UNIT_MOVEMENT with grid
      7. Last any entry with grid
    """
    from collections import defaultdict

    for p in positions:
        p['is_end_of_day'] = False

    date_indices: dict[str, list[int]] = defaultdict(list)
    for i, p in enumerate(positions):
        if p['date']:
            date_indices[p['date']].append(i)

    def _is_place_col(p):
        return (p.get('context') or '').startswith('Place column')

    for date, indices in date_indices.items():
        def _last(cats, require_grid=True, place_col_only=False):
            matches = [
                i for i in indices
                if positions[i]['category'] in cats
                and (not require_grid or positions[i]['grid'])
                and (not place_col_only or _is_place_col(positions[i]))
            ]
            return matches[-1] if matches else None

        winner = (
            _last({'TAC_HQ'}, require_grid=True,  place_col_only=True)  or
            _last({'BN_HQ'},  require_grid=True,  place_col_only=True)  or
            _last({'TAC_HQ', 'BN_HQ'}, require_grid=False, place_col_only=True) or
            _last({'TAC_HQ'}, require_grid=True,  place_col_only=False) or
            _last({'BN_HQ'},  require_grid=True,  place_col_only=False) or
            _last({'UNIT_MOVEMENT'}, require_grid=True) or
            next((i for i in reversed(indices) if positions[i]['grid']), None)
        )

        if winner is not None:
            positions[winner]['is_end_of_day'] = True

    return positions


# ── run ───────────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    import pathlib

    print("Reading source file …")
    text = pathlib.Path(INPUT_FILE).read_text(encoding="utf-8")

    print("Extracting positions …")
    positions = extract_positions(text)

    # deduplicate identical (date + grid + context[:60]) entries
    seen = set()
    unique = []
    for p in positions:
        key = (p["date"], p["grid"], p["context"][:60])
        if key not in seen:
            seen.add(key)
            unique.append(p)

    print("Assigning end-of-day flags …")
    unique = assign_end_of_day(unique)

    pathlib.Path(OUTPUT_FILE).parent.mkdir(parents=True, exist_ok=True)
    pathlib.Path(OUTPUT_FILE).write_text(
        json.dumps(unique, indent=2, ensure_ascii=False),
        encoding="utf-8"
    )
    print(f"Done. {len(unique)} positions written to {OUTPUT_FILE}")