diff --git a/.gitignore b/.gitignore index d05328d..c8ea75b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ _site/ pass-deployment deployment.json src/css/styles.min.css +inputs/ \ No newline at end of file diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..97809c3 --- /dev/null +++ b/README.MD @@ -0,0 +1,36 @@ +# AI-Prototype + +Prototype workspace for Project '44 / Frame 44 descendant report pipeline. +First end-to-end test: producing a Tier 3 report for a hypothetical +descendant of a Calgary Highlanders soldier, using the regimental war +diary as the primary source. + +## Conventions + +- **Test customer name:** Bill Bloggins. Used as a placeholder across all + prototype runs so test data is immediately distinguishable from real + customer data. Never use a real name in prototype work. +- **File naming:** `_.` where applicable. +- **Isolation:** This sandbox works only with copies of source data. + Never connect this prototype to the master Postgres database. + +## Folder structure + +- `inputs/` — copies of source data (OCR .docx, original PDFs, position + data). Read-only in spirit; never edited here. +- `prompts/` — prompts used to drive the AI runs, versioned (v1, v2…). +- `outputs/` — raw AI output from each run, saved verbatim. +- `reports/` — polished, human-reviewed deliverables built from outputs. +- `notes/` — project notes: unit selection, evaluation, follower + feedback, decisions. + +## Current test + +- Unit: Calgary Highlanders (1600221) +- Window: mid-Sep 1944 through early Nov 1944 (the Scheldt) +- Tier: 3 (regiment known, dates fuzzy, one event known) +- See `notes/unit-selection.md` for the full scenario. + +## Infrastructure + +- https://deepinfra.com/dash used for running OCR via olmocr \ No newline at end of file diff --git a/launch_viewer.py b/launch_viewer.py new file mode 100644 index 0000000..5ff1c79 --- /dev/null +++ b/launch_viewer.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +""" +P44 OCR Viewer — launcher +Starts a local HTTP server at port 5044 and opens the viewer in the default browser. +Run from any directory; the server root is always the project folder (this file's location). +""" +import http.server +import webbrowser +import threading +import os +import sys + +PORT = 5044 +ROOT = os.path.dirname(os.path.abspath(__file__)) + +class QuietHandler(http.server.SimpleHTTPRequestHandler): + """Serve files from ROOT, suppress request logs.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=ROOT, **kwargs) + def log_message(self, fmt, *args): + pass # silence per-request noise + +def start_server(): + server = http.server.HTTPServer(('localhost', PORT), QuietHandler) + server.serve_forever() + +if __name__ == '__main__': + # Start the server in a background daemon thread + t = threading.Thread(target=start_server, daemon=True) + t.start() + + url = f'http://localhost:{PORT}/p44-ocr-viewer.html' + print(f"\n ⚜ P44 OCR Viewer") + print(f" Serving: {ROOT}") + print(f" URL: {url}\n") + webbrowser.open(url) + + print("Press Enter (or Ctrl-C) to stop the server and exit.\n") + try: + input() + except KeyboardInterrupt: + pass + print("Server stopped.") + sys.exit(0) + diff --git a/notes/ocr-viewer-prompt.md b/notes/ocr-viewer-prompt.md new file mode 100644 index 0000000..01cb057 --- /dev/null +++ b/notes/ocr-viewer-prompt.md @@ -0,0 +1,46 @@ +# P44 OCR Viewer — Copilot Build Prompt + +## Context +A desktop OCR review tool for verifying war diary OCR output page by page. +Single HTML file, no build step, runs by double-clicking. + +## If Copilot times out — build in this order: + +### Step 1: Shell + layout +> Build the shell of the P44 OCR Viewer. Two panels side by side using CSS Grid. +> Dark background (#1a1a2e), gold accent (#c8a84b). Left panel: image display with +> scroll. Right panel: scrollable text area. Top toolbar with three file inputs: +> (1) page image, (2) OCR text file, (3) word-position JSON sidecar. No interactivity +> yet — just layout and file loading that populates each panel. + +### Step 2: Word overlay on image +> Add a word bounding box overlay layer to the left panel. The JSON sidecar format is: +> [{ "word": "string", "id": "w001", "bbox": { "x": 0, "y": 0, "w": 50, "h": 20 } }] +> Render transparent absolutely-positioned divs over the image for each word, +> using the bbox coordinates scaled to the rendered image dimensions. +> Each div gets a data-word-id attribute matching the JSON id. + +### Step 3: Word spans in text panel +> Parse the OCR text in the right panel and wrap each whitespace-delimited token +> in a with a data-word-id attribute. IDs are assigned sequentially (w001, +> w002...) matching the order in the JSON sidecar. + +### Step 4: Bidirectional hover highlight +> On hover over any word span (right) or bounding box div (left), highlight the +> matching element on the other side with a yellow background/border. Use the +> data-word-id to find the match. Clear highlight on mouseout. + +### Step 5: Load example button +> Add a "Load example" button to the toolbar that pre-populates all three panels +> with bundled inline sample data (a small image placeholder, 2-3 lines of sample +> OCR text, and a matching JSON sidecar with ~10 words). Tool should be fully +> demonstrable without real files. + +## Full one-shot prompt (use if Copilot has capacity) +Build a desktop OCR review tool called P44 OCR Viewer for reviewing war diary OCR output. The interface has two panels side by side: +Left panel — the source document image. Load a JPEG or PNG page scan. Render it at full panel height, scrollable. Each word in the image should be represented by a transparent, hoverable bounding box overlay (coordinates come from a JSON sidecar file, format: [{ "word": "string", "id": "w001", "bbox": { "x": 0, "y": 0, "w": 50, "h": 20 } }]). +Right panel — the OCR text output. Load the corresponding plain text or Markdown file. Each word in the text should be individually wrapped in a with a matching data-word-id attribute that maps to the same ID in the JSON sidecar. +Hover behaviour: Hovering over a word on either side highlights the corresponding word on the other side — a yellow highlight on the image bounding box overlay, and a yellow background on the text span. Bidirectional. Highlight clears on mouse-out. +File loading: A toolbar at the top has three file inputs: (1) Page image, (2) OCR text file, (3) Word-position JSON sidecar. Files are loaded locally, no server required. A "Load example" button pre-populates with bundled sample data so the tool works without real data. +Stack: Single HTML file, vanilla JS, no build step required. Designed to run by double-clicking — no localhost server, no npm. All layout via CSS Grid. Colour scheme: dark background (#1a1a2e), muted gold accent (#c8a84b) consistent with a WWII archival aesthetic. +The word-alignment JSON sidecar format is designed to be produced by olmOCR (Allen Institute), which outputs bounding box data alongside transcribed text. The viewer is the QA interface a historian uses to verify OCR accuracy page by page. \ No newline at end of file diff --git a/notes/pilot-pipeline-tasklist.md b/notes/pilot-pipeline-tasklist.md new file mode 100644 index 0000000..f455f5a --- /dev/null +++ b/notes/pilot-pipeline-tasklist.md @@ -0,0 +1,10 @@ +1. Pilot scope: 5 CIB + 4 battalions, Sept–Oct 44 (~10–12 PDFs) +2. Write 30–50 evaluation questions and answer them by hand from the source documents — this is your ground truth +3. omOCR the pilot corpus +4. Human-verify the OCR — full review on this set, since it's also benchmark data +5. Design the inventory schema (page-range based, with document-level granularity where pages are uniform) +6. Build the inventory using LLM-assisted prompts, refined against your hand-built inventories of CH/5CIB/BW Sept 44 +7. Design and test chunking strategy — chunk size, metadata attachment, special handling for tables/orders +8. RAG-chunk and embed +9. Run evaluation questions against the system, score, iterate +10. Once accuracy targets are hit on the pilot, scale to the full 1,200-doc corpus with automated OCR + inventory + chunking, with human spot-checks rather than full verification \ No newline at end of file diff --git a/notes/scoring-schema.md b/notes/scoring-schema.md new file mode 100644 index 0000000..02578fa --- /dev/null +++ b/notes/scoring-schema.md @@ -0,0 +1,63 @@ +Tier 1 — Direct factual lookup (10 questions) +Tests basic retrieval: does the right chunk come back for a clear factual query? + +What unit was the Calgary Highlanders' Dog Coy attacking on 15 September 1944, and what was the location? +Who signed the 5 CIB war diary for September 1944? +On what date did the Calgary Highlanders force the Albert Canal crossing? +What was the grid reference of the lock gates the Calgary Highlanders crossed at on 22 September 1944? +Who was singled out for distinction during the Albert Canal crossing, and what coy/platoon were they in? +What was the date of the Dieppe march past attended by the Calgary Highlanders? +What was the Black Watch's location at the start of September 1944 (per the cover sheet)? +How many German prisoners did the Calgary Highlanders take on 14 September 1944 in the action toward Fme Geersen? +What was the H-hour given in the Black Watch field message of 10 September 1944, 0025 hrs? +What route was issued under 5 CIB Mov Order No 5 on 1 September 1944? + + +Tier 2 — Cross-document synthesis (10 questions) +Tests whether the system can combine information from multiple sources to answer a single question. + +Compare the Calgary Highlanders and Black Watch officer strength at the start and end of September 1944. What does the trend suggest? +On 22 September 1944, the Calgary Highlanders were ordered to take over a crossing the Black Watch had failed to make. What did the Black Watch war diary say about that failed attempt? +What does the 5 CIB war diary narrative say about the Calgary Highlanders' actions on 7–10 September 1944, and how does it compare to the Calgary Highlanders' own narrative for those dates? +Which battalion in 5 CIB had the highest other-rank strength in the week ending 9 September 1944? +Trace the movement of 5 CIB from Totes to the Dunkerque outpost line using the Move Orders (No 4, 5, 6, 7) and the brigade narrative. +What enemy defences were the Calgary Highlanders likely to encounter at Bourbourgville based on the SHAEF and APIS traces in the Black Watch and brigade files? +On 5 September 1944, what was 5 CIB's planned route at the divisional level (per the 2 Cdn Inf Div Route Card) and how does it compare to the brigade's own Mov Order No 6? +Identify all the locations mentioned in any 5 CIB document for the date 1 September 1944. What movement is implied? +Which sister battalion was closest to the Calgary Highlanders' position on 9 September 1944? +What casualties did 5 CIB units sustain in the first ten days of September 1944, and where would that information be found? + + +Tier 3 — Operational and tactical reasoning (10 questions) +Tests whether the system can answer questions that require understanding military context, not just text retrieval. + +What was 5 CIB's tactical role between 1 and 10 September 1944 — exploitation, set-piece attack, defensive holding, or something else? Cite evidence. +Where was the Calgary Highlanders' advance party on 16–18 September 1944, and what does that suggest about brigade movement plans? +What was the significance of the Calgary Highlanders crossing at Fort 1 on the Albert Canal rather than at the original Black Watch crossing point? +Based on the patrol activity recorded around Loon Plage 9–13 September 1944, what was the tactical situation? +What artillery support was available to the Calgary Highlanders during the Bourbourgville–Loon Plage operations? Where would that be documented? +The 5 CIB war diary for September mentions "Dunkerque outpost line." What does that mean operationally, and which 5 CIB documents add detail to that picture? +Why did the Calgary Highlanders move from France to Belgium between 18–19 September 1944, and what was the operational objective? +What evidence exists in the brigade and battalion war diaries for the strength and quality of the German garrison the Calgary Highlanders were facing in early September 1944? +Was the Calgary Highlanders' advance from Bourbourgville to Loon Plage opposed, lightly opposed, or heavily opposed? Cite specific document evidence. +What changes in unit identifications from PW interrogations (per the GO-1 ISUM) might have affected 5 CIB's understanding of the enemy in early September 1944? + + +Tier 4 — Reinforcement / personnel queries (5 questions) +Tests the "His War" use case directly — reconstructing what an individual soldier would have experienced. + +A reinforcement arriving at the Calgary Highlanders on or about 1 September 1944 would have been part of which company actions in the following ten days? Walk through what they would have experienced day by day. +What named reinforcements arrived at the Calgary Highlanders during 16–17 September 1944? +If a soldier joined the Calgary Highlanders as a reinforcement in late August 1944, what was the strength of the battalion when they arrived (per the field returns), and how did it change over September? +What activities did the Calgary Highlanders conduct that were not direct combat operations during 1–10 September 1944 (parades, baths, religious services, etc.)? +A Calgary Highlanders soldier killed or wounded on 8 September 1944 at Bourbourgville would have their casualty recorded in which document(s), and on what dates would those documents have been filed? + + +Tier 5 — Stress tests and adversarial queries (5 questions) +Tests failure modes — does the system hallucinate when the answer isn't in the corpus, or does it correctly say "I don't know"? + +What were the Régiment de Maisonneuve's casualties on 8 September 1944? (R de Mais inventory not provided in pilot — system should say it doesn't have that source, not invent an answer.) +What did General Crerar say in his speech at the Dieppe march past? (Speech text is not in the inventory; the march past is mentioned but the speech content isn't. Correct answer: "not in available sources.") +What was the weather on 7 September 1944 at Bourbourgville? (May or may not be in the war diaries; system should answer from source if present, otherwise say so — not pull from external knowledge.) +Who commanded the German garrison at Dunkerque in September 1944? (Likely not in 5 CIB sources; tests whether system stays within corpus or hallucinates from training data.) +What does the war diary say about Calgary Highlanders operations on 5 October 1944? (October is in scope for the pilot but not yet inventoried in our session — tests whether system handles dates within scope but in un-verified portions of the corpus.) \ No newline at end of file diff --git a/notes/v1-evaluation-accuracy.md b/notes/v1-evaluation-accuracy.md new file mode 100644 index 0000000..c246958 --- /dev/null +++ b/notes/v1-evaluation-accuracy.md @@ -0,0 +1,10 @@ +Line 15 - Talks about Basingstoke, which I assume is where the Calgary highlanders were based in England before they went to France. This is a detail that I would expect to be in the report, and it is not. The report should be grounded in the diary, and this is a detail that is in the diary but not in the report. This is a miss, and it should be included in the report to give a sense of where the soldiers were before they went to France. +line 15 - it says the co who know his officers by their first names. Is this in the war diaries or an assumption? +line 17 - great detail about the first world war actions of the 10th battalion +line 19 - seems accurate to the war diary. It talks about GR Crockett recommended for the victoria cross,but I know crockett did not get the VC. Be careful to word it so that it is clear that he was recommended but did not receive it. +line 27 - unless we are showing these position on the map, giving grid references in the report is not helpful to the family. It is more important to give a sense of where they were in relation to known places (e.g. "the town of X" or "the river Y") rather than giving grid references that the family will not understand. If we are showing these positions on the map, then it is fine to include the grid references, but if not, then it is better to omit them from the report. +line 48 - spot checked. seems accurate to the war diary +Line 50 - spot checked. seems accurate to the war diary. +Line 52 - spot checked. seems accurate to the war diary. +Line 55 - spot checked. seems accurate to the war diary. +Line 61 - spot checked. seems accurate to the war diary. \ No newline at end of file diff --git a/notes/v1-evaluation-customer.md b/notes/v1-evaluation-customer.md new file mode 100644 index 0000000..d721c95 --- /dev/null +++ b/notes/v1-evaluation-customer.md @@ -0,0 +1,19 @@ +Did I learn what my grandfather lived through, in a way I'll remember? + +I think the average person will get a good sense of where their grandfather was, and roughly wha they are doing. I do think that the because the war diary is very officer centric, and based around the command post and tactical headquarters, it is hard to get a sense of what the average private would have experienced. The report does a good job of trying to include the human texture of the experience, but it is still very much focused on the operational events and the perspective of the officers. I think that is a limitation of the source material, and it is something we will have to be mindful of as we produce these reports. We may want to consider supplementing the war diary with other sources that give a more ground level perspective, such as letters, diaries, or oral histories from soldiers who were there. That would help us to give a more well rounded picture of what the average soldier experienced during those battles. + +Are there moments here I'll quote to my family at Christmas? + +I think there are some sections people would want to share, but again very officer centric. + +Did the report respect my intelligence, or did it talk down to me / over my head? + +The report is written in a way that is accessible to a general audience, but it does not talk down to the reader. It assumes a certain level of knowledge about the war and the battles, but it also explains things in a way that is easy to understand. I think it strikes a good balance between being informative and being engaging, without being condescending or overly simplistic. + +What do I now want to ask, that the report didn't answer? + +More on what the average soldier experienced, and what the fighting was like from their perspective. I would also want to know more about the specific actions that Bill was involved in, if we can find that out. I would also want to know more about the human texture of the experience, such as what the rations were like, what the billets were like, what the weather was like, and what the civilian encounters were like. Those are the things that really bring the experience to life for me, and I think they would be important for other people as well. + +Would I pay for this? If yes, roughly what? + +I would pay for this, but how much I dont know. I think $20 at a minimum. If there was more depth to what it was like for an average soldier it would be worth more. As well, if we could include photos and maps, and see where Bill was day by day, and where the most important events happened, that would add a lot of value. \ No newline at end of file diff --git a/notes/v1-evaluation-operational.md b/notes/v1-evaluation-operational.md new file mode 100644 index 0000000..53625ca --- /dev/null +++ b/notes/v1-evaluation-operational.md @@ -0,0 +1,8 @@ +Is this actually a mappable event? (Some movements/billets aren't really points; some are.) +Most of it is very mappable. Especially if we map the events to the units location ie we know where the unit was, so you are already near it, use the gsgs base maps to find the location, and then use the grid reference to find the exact location. Very very useful + +Does the location text contain enough information to geocode? (Grid refs are gold; "Hinkelenoord" alone is harder.) +Grids are perfect. Locations are harder, but I think over time as we map a place with a name, it needs to be captured into like a gazetter or toponomy record so that we can find it again. I think the more we do this, the better we will get at geocoding these locations. We will also get better at recognizing when a location is not specific enough to geocode, and when we need to do some additional research to find out where it is. + +Would Jonesy accept it without changes, edit it, or reject it? Estimate the proportions. +I think Jonesy would accept most of it without changes, but there would be some edits for clarity and to make sure that we are not including any information that is not supported by the war diary. I would estimate that about 80% of the events would be accepted without changes, about 15% would be edited for clarity or to remove unsupported information, and about 5% would be rejected because they are not supported by the war diary or because they are not mappable. \ No newline at end of file diff --git a/notes/v1-evaluation-voice.md b/notes/v1-evaluation-voice.md new file mode 100644 index 0000000..97eb48d --- /dev/null +++ b/notes/v1-evaluation-voice.md @@ -0,0 +1,3 @@ +The voice and tone is good but I feel it does not provide enough gravity to the heavy fightinng near hoogerheide, and Waclheren. I would like to see more of the emotional weight of the fighting in that area, and the toll it took on the soldiers. The report should not shy away from the brutality of the fighting, and should give a sense of the fear and uncertainty that the soldiers must have felt during those battles. Though, if it is missing from the war diary, it is not a problem to omit it from the report. I just want to make sure that we are not sanitizing the experience for the family, and that we are giving them a sense of what their ancestor went through during those battles. + +We can expand on the voice when we use other texts in the future such as CP Staceys The Victory Campaign. \ No newline at end of file diff --git a/notes/v2-source-selection.md b/notes/v2-source-selection.md new file mode 100644 index 0000000..13aa8ed --- /dev/null +++ b/notes/v2-source-selection.md @@ -0,0 +1,110 @@ +# Supplementary Source Inventory — Pte. Bill Bloggins (placeholder) +## Calgary Highlanders, 1–10 September 1944 + +**Purpose:** Curated list of supplementary documents to accompany the Calgary Highlanders main war diary narrative for the v2 prototype descendant report. +**Focus window:** 1–10 September 1944 +**Source PDFs covered:** Calgary Highlanders, 5 Cdn Inf Bde, RHC Black Watch (R de Mais inventory not provided) +**Inventory coverage:** CH pp. 1–45 of 343 · 5 CIB pp. 1–56 of 193 · BW pp. 1–51 of 144 +**Document count:** 24 items +**Status:** Ready for use; gaps flagged at end + +--- + +## Selection criteria (priority order) + +1. Documents covering specific events the Calgary Highlanders were involved in during 1–10 Sep 1944 +2. Brigade-level signals, sitreps, or orders covering the same dates and actions +3. Sister-battalion entries that mention or coordinate with the Calgary Highlanders during the focus window +4. Patrol programmes, patrol reports, casualty returns, or other operational documents adding detail beyond the diary narrative + +--- + +## Source 1 — Calgary Highlanders War Diary +**File:** `Calgary-Highlanders_War-Diary_Sep44.pdf` (343 pp; inventoried 1–45) + +| Page(s) | Document type | Date covered | Note | +|---|---|---|---| +| 6 | Appendix index | Sep 44 | Master key to the file — locates Appx 12 (Memorial Service 3 Sep), Appx 13 (Dieppe march past), Appx 10 (Daily News sheets) referenced in narrative for focus window | +| 21–26 | War diary narrative pp. 15–20 | 11–13 Sep 44 | **Just outside window** — direct continuation of Loon Plage fighting Bloggins entered 7–10 Sep; drop if hard 10 Sep cut required | + +--- + +## Source 2 — 5 Cdn Inf Bde War Diary +**File:** `5CIB_War-Diary_Sep44.pdf` (193 pp; inventoried 1–56) + +| Page(s) | Document type | Date covered | Note | +|---|---|---|---| +| 5–20 | Bde war diary narrative (Brig. Megill) | 1–30 Sep 44 | Brigade-level command perspective on the same operations; only 1–10 Sep entries in scope | +| 22–26 | Part I Orders (Appx 1) | 15 Sep 44 | Marginal — dated outside window but routine orders often cover preceding fortnight (dress, censorship, discipline) | +| 28–34 | AFW 3008 Field Returns of Officers (Appx 3) | wks ending 2 & 9 Sep 44 | Officer strength returns spanning focus window — establishes who was commanding what when Bloggins arrived | +| 35–43 | AFW 3009 Field Returns of Other Ranks (Appx 4) | wks ending 2 & 9 Sep 44 | OR strength returns — directly relevant to Bloggins as reinforcement; bde-level intake picture in his first days | +| 44 | Mov Order No 4 | 1 Sep 44 | Bde move order on Day 1 of window — order under which CH moved from Totes area | +| 45 | Trace "T" — unit dispositions | 1 Sep 44 | Companion trace to Mov Order 4 — shows CH placement within bde at start of window | +| 46 | Mov Order No 5 | 1 Sep 44 | Second 1 Sep move order — sequencing of bde group movement | +| 47 | Trace "R" — route sketch | 1 Sep 44 | Route sketch for Mov Order 5 | +| 48–49 | Mov Order No 6 | 5 Sep 44 | Mid-window move order covering advance toward Dunkerque outpost line | +| 50 | Route Card, 2 Cdn Inf Div | 5 Sep 44 | Div-level route card companion to Mov Order 6 — divisional context for bde move | +| 51–52 | Mov Order No 7 | 6 Sep 44 | Move order issued day before CH narrative records Bourbourgville advance — likely the order that put CH into contact | + +--- + +## Source 3 — RHC Black Watch War Diary +**File:** `RHC-Blackwatch_War-Diary_Sep44.pdf` (144 pp; inventoried 1–51) + +| Page(s) | Document type | Date covered | Note | +|---|---|---|---| +| 6–15 | Bn war diary narrative (Lt-Col Ritchie) | 1–30 Sep 44 | Sister-bn narrative; only 1–10 Sep entries in scope. BW alongside CH through Dieppe parade (3 Sep) and advance to Dunkerque approaches | +| 18 | SHAEF "Enemy Defences" trace | 23 Aug 44 | Pre-window but describes enemy defences CH would encounter at Bourbourgville/Loon Plage from 7 Sep — operational intel context | +| 19 | Dunkerque Plan of Port & Town | n/d | Terrain CH was advancing toward through focus window | +| 20 | Dunkerque defence overprint map | Ed. 11 Sep 44 | Compiled from air photo info as at 1 Sep and other sources as at 10 Sep — situation across Bloggins's first ten days | +| 21 | Defence overprint legend | n/d | Companion key to the overprint — needed to read p. 20 | +| 24–25 | RHC Field Return of Officers | wk ending 2 Sep 44 | Sister-bn officer strength at start of window | +| 26–27 | RHC Field Return of Officers | wk ending 9 Sep 44 | Sister-bn officer strength at close of window | +| 33 | Field Message Form (Lt-Col Ritchie to coys) | 10 Sep 44, 0025 hrs | Only timed BW signal in window — H-hour change to 0755, RV at 853093 at 0700; bde-sector operational tempo on closing night | +| 34–35 | RHC Field Return of Other Ranks | wk ending 3 Sep 44 | Sister-bn OR strength — comparison of reinforcement intake across CH/RHC/R de M | +| 36–37 | RHC Field Return of Other Ranks | wk ending 10 Sep 44 | Same, end of window | +| 50 | APIS 2 Cdn Inf Div trace, Part I | 1 Sep 44 | Divisional intel trace from first day of window — beach defences, MGs, minefields in bde sector | +| 51 | APIS 2 Cdn Inf Div trace, Part II | ~1 Sep 44 | Continuation of p. 50 trace | + +--- + +## Items considered and excluded + +| Source | Page(s) | Reason for exclusion | +|---|---|---| +| BW | 22–23, 28–29, 30–31 | Officer returns wks ending 16, 23, 30 Sep — outside window | +| BW | 38–39, 40–41, 42–44 | OR returns wks ending 16, 23, 30 Sep — outside window | +| BW | 46–49 | Undated coy messages and faded carbon — cannot place in window with confidence | +| 5 CIB | 53–56 | Mov Orders 9 (17 Sep) and later — outside window | + +--- + +## Gaps — material expected but not in catalogued ranges + +1. **R de Mais war diary inventory** — not provided. Largest gap. R de Mais was the third bn in 5 CIB, operating alongside CH throughout focus window. Without it, criterion 3 (sister-bn coordination) is half-answered. + +2. **Calgary Highlanders Appendices 3–15** — none catalogued. The CH Move Orders (Appx 4), Messages (Appx 5), Patrol Programme (Appx 6), Memorial Service 3 Sep (Appx 12), and March Past Dieppe (Appx 13) are all directly in scope and almost certainly the highest-value supplementary documents in the entire source set. **Inventorying CH pp. 46–343 should be the next priority.** + +3. **5 CIB Ops Log (Appx 13, sheets 109–161)** — un-catalogued in pp. 57–193. Bde signal log for the whole month; would be the single richest source of bde-level granularity for 1–10 Sep. + +4. **5 CIB Patrol Programmes and Patrol Reports (Appx 6, 7)** — un-catalogued. Patrol activity at Loon Plage from 9–10 Sep described in CH narrative; patrol reports would corroborate. + +5. **5 CIB Int Report on Dunkerque, 13 Sep** (Appx 10) — outside date but retrospective on focus-window area; un-catalogued. + +6. **BW Appx 6 (Warning Orders, movement tables)** and **Appx 8 (Sitreps)** — un-catalogued in pp. 52–144. Sitreps would map well onto focus window. + +7. **No casualty returns** in any catalogued range. CH narrative refers to casualties 7–10 Sep at Bourbourgville/Loon Plage; actual casualty returns (Part II Orders, AFW 3010-series, or unit casualty lists) aren't in catalogued portions of any file. Likely in CH Appx 2 (Part II Orders) or BW Appx 2 ("to follow"). + +8. **No artillery task tables** in scope. CH Appx 3 (Arty Task Tables) and BW Appx 7 (RCA Conc tables, tasks) both un-catalogued and would directly cover fire support for actions Bloggins was in. + +--- + +## Recommended next inventory pass + +To meaningfully increase supplementary-source density: + +- **CH pp. 46–100** — capture early appendices likely covering 1–10 Sep (Move Orders, Messages, Patrol Programme, Memorial Service, Dieppe March Past) +- **5 CIB pp. 57–100** — find start of Ops Log (Appx 13) and patrol material (Appx 6, 7) +- **R de Mais Sep 44 war diary** — full inventory pp. 1–~50 to recover sister-bn coordination evidence + +These three passes would roughly double the supplementary-source density of this list. diff --git a/outputs/v1-output.md b/outputs/v1-output.md new file mode 100644 index 0000000..4779c04 --- /dev/null +++ b/outputs/v1-output.md @@ -0,0 +1,96 @@ +# Pte. Bill Bloggins — Descendant Report (Tier 3) + +**Unit:** The Calgary Highlanders, 5th Canadian Infantry Brigade, 2nd Canadian Infantry Division +**Service window covered:** mid-September 1944 through early November 1944 +**Source:** Calgary Highlanders War Diary, Sep 1944 – May 1945 + +A note before you read this: we don't know Bill's exact arrival date, his company, his platoon, or the precise day and place he was wounded. What follows is an account of what his battalion was doing and where it was during the weeks he most likely served. Where the diary is specific, we've tried to be specific. Where it isn't, we've said so. + +--- + +## 1. Narrative + +When Bill joined the Calgary Highlanders in mid-September 1944, the battalion was on a stretch of front near Loon Plage, just west of Dunkirk. Dieppe had fallen on 1 September with almost no fighting, and the 2nd Canadian Division had paraded through it for General Crerar two days later, but Dunkirk was still held by a German garrison, and the Calgary Highlanders had been drawing patrols and fighting recces around farms named Fme Geersen and Fme Charlemagne. On 15 September, Major Wynn Lasher's "Baker" Company had been pinned down by machine guns trying to assist "Dog" Company near the Geersen farm; after the fight, one man's webbing was found on the bridge and the diary records that he was "assumed to have been taken prisoner." + +The next day, 16 September, was what the diary calls "a banner day" for reinforcements: thirty other ranks plus Major Stott, Major Baker, and Lts. Maguire, Mageli, and Holmgren joined the battalion. If Bill was not in this draft he came in with one of the smaller parties over the next ten days. The senior officers in the 16 September group had been "Blighted" — sent back to England wounded earlier in the campaign and now returned — and the men crowded around them to ask about Basingstoke. Whatever draft Bill came in on, this was the atmosphere he stepped into: a battalion that had been fighting since Normandy, run by a CO (Lt.-Col. D.G. MacLauchlan) who knew his officers by their first names and ran his orders groups in whatever kitchen or study the battalion happened to be in. + +On 17 September the Calgary Highlanders were pulled out of the Dunkirk perimeter and sent east. The convoy crossed into Belgium on 18 September on roads "the best that had been seen for a long time," and the battalion's diarist — clearly looking out the side of the carrier — noted the famous First World War names along the route: Ypres, St. Julien, Passchendaele, Poelcappelle. The diary entry for that day says that as they passed St. Julien "Calgarians began to experience a certain well-known lump in their throats," because the battalion's predecessor unit, the 10th Battalion, had made its stand there on 22 April 1915 and earned the regiment the right to wear the oak-leaf shoulder titles. The men were showered with fruit, bread, and "at times with liquers" along the way, and they reached their concentration area at Wommelghem, on the eastern outskirts of Antwerp, that evening. + +The next operation was the crossing of the Albert Canal. On the night of 21/22 September, after the Royal Highlanders of Canada (the Black Watch) had failed in their attempt the previous night, "Charlie" Company crossed the damaged lock gates at MR 769968 in single file as a fighting patrol, established a small bridgehead, and the rest of the battalion followed across the wrecked locks. The diarist describes the operation as "a complete success" and "a gratifying experience to watch the plan of Lt.-Col. D.G. MacLauchlan unfold in all its brilliant timing and co-ordination." Sgt. G.R. Crockett of 15 Platoon, Charlie Company, was eventually recommended for the Victoria Cross for the action. By 23 September, Bn HQ was at Chateau Helleputte at MR 769969, prisoners were "practically pouring in," and the bridgehead had forced a German withdrawal across an 18,000-metre front. For a reinforcement, this was the kind of action that would have introduced him to how the battalion actually worked: the careful planning, the use of artillery ("Shelldrake") to soften positions, the interrogation of prisoners through Belgian White Brigade volunteers, and the casual sniping of Germans in trees by Major Ellis's and Major Baker's companies on 24 September. + +The pace then quickened. Between 27 September and 1 October the Calgary Highlanders were committed to the Brigade thrust toward Brecht. The fighting around Brecht and St. Leonard, particularly Major Kearns's Able Company on the night of 30 September/1 October, was intense — Able Company was at one point with the enemy on three sides, and a German tank "milling about trying to pin-point Able Coy" had to be engaged with PIATs at point-blank range. The diary records Lt. Casey killed and Lt. Doug Craddock wounded in the knee that night. By 3 October the battalion was at Lochtenberg, where Major Ellis's Baker Company was billeted in bungalows around a small lake. On the night of 3/4 October the battalion seized Fort de Schooten — Major Kearns's Able Company crossing the moat in a punt that Capt. Mark Tennant and the I.O. had patched up with rags and wax when the recce boats promised by Brigade failed to arrive. + +This is the run-up to the Scheldt. On 6 October the battalion crossed the Dutch border for the first time, and on 7 October — "Ross Ellis Day," as it became known in the battalion — the Calgary Highlanders fought into Hoogerheide, just south of the South Beveland isthmus, taking 62 prisoners. The next several days, 8 through 10 October, were among the worst the battalion had: at the Jansen Farm at MR 626185, Able Company was overrun by a counter-attack on the afternoon of 9 October, Major Kearns was wounded by shrapnel while sitting in a chair at Bn HQ, Major Bruce MacKenzie of Dog Company was wounded, and the diary records the CO "almost in tears" when Capt. Mark Tennant — the battalion's much-loved scout/recce officer — was hit and evacuated. Major Ellis took over as 2 i/c, and on 10 October he led Baker Company forward to retake the lost ground around junction 622195. The intersection at 624198 became known in the battalion as "Hell's Cross Roads of Hoogerheide" and the Calgary Highlanders accounted for a reported 580 of the 600 German casualties the brigade inflicted in that sector. + +After a 24-hour rest on 11 October — bath parade, exchange parade for Dutch guilders organised by Capt. Rolly Higgins, a "Geen Bier" sign hung at the bar — the battalion moved to a holding role around Hinkelenoord on the south bank facing Woensdrecht. Through 14 to 22 October they patrolled, fired on enemy positions across the dykes, and watched Lt.-Col. Stott's S.S.R. (themselves former Calgary Highlanders) attack to the north. Lt.-Col. MacLauchlan's DSO was announced on 16 October. On 21 October a fire broke out in the upstairs of Wolfert's Farm at 599189, the battalion's tac HQ, and the building burned to the ground; Pte. Sapinsky, the CO's carrier driver, was badly burned trying to fight it. + +On 23 October the battalion attacked again to seal off the Beveland isthmus, with Capt. Pearson hit early and Lt. Wilkins taking over Dog Company. Fighting continued through 24, 25, and 26 October at Wynn 1, 2, and 3 around grid 6122, with hand-to-hand fighting in the area of road and rail 612225. By 27 October the enemy had withdrawn and the battalion handed over to the 8th Canadian Reconnaissance Regiment. This was the moment, for the first time in weeks, when the Calgary Highlanders had a meal that was not interrupted, distributed 50 cigarettes per man, and the GOC told Brigadier McGill that "the Calgary Highlanders have done a damn fine job for the Division." + +The hardest fighting was still to come. On 28 October the battalion moved across South Beveland, through Schore and Goes to Heer Arendskerke. On 29 October — and this is one of the saddest entries in the diary — Lt.-Col. MacLauchlan was suddenly seen "almost in tears, bidding farewell to the officers and men." He had been removed from command without warning. Major Ross Ellis took over the battalion. By 30 October the Calgary Highlanders were on the eastern shore of the Sloe Channel, looking at the Walcheren Causeway: a narrow, raised, exposed dyke road about 2,000 metres long, the only land approach to Walcheren Island. The Royal Regiment of Canada had reached the eastern end. The Black Watch had attempted to push across that night and been stopped halfway. + +If Bill was wounded "in early November during the Scheldt fighting," it was almost certainly during the Calgary Highlanders' own attempt on the Causeway between midnight 31 October and the morning of 2 November. Baker Company under Capt. Clarke set foot on the Causeway at midnight on 1 November and was halted about midway by machine-gun and mortar fire; Capt. Clarke obtained permission to withdraw. Dog Company tried again at 0605, made it past the road block, and reached the western end at 0933. Able and Charlie Companies crossed in turn. Capt. Wynn Lasher of Able Company was wounded that afternoon; Brigade Major George Hees crossed to take command of Able Company in his place, with Capt. Newman, the artillery FOO, volunteering as 2 i/c. On the morning of 2 November the Régiment de Maisonneuve and then the Glasgow Highlanders relieved them. The battalion's officer casualties for the action were Lt. John Moffat killed and Capt. Lasher and Lts. Lefroy, Schoening, and Hoy wounded; the diarist says only that "fighting along the Causeway had been terrific for the last 40 hours and words are inadequate to express all the difficulties that had to be surmounted." + +Bill is most likely to have been wounded somewhere on that Causeway, on 1 or 2 November, though the diary does not name him and does not record other-rank casualties individually. He would have been evacuated back across the Causeway, through the Regimental Aid Post, and along the chain to a field ambulance and on to England. The battalion withdrew to Lierre, in Belgium, on the night of 2/3 November for a rest. They were billeted in private homes; Major Ellis was promoted Lieutenant-Colonel on 5 November; an officers' mess dinner was held at the Hôtel Commerce on the 6th, and on the 9th they began moving north to Groesbeek-Nijmegen, where they would spend the winter as the reserve battalion in a quiet sector. By the time Bill's battalion reached Groesbeek, he was already in a hospital in England. + +What his service looked like, in the most ordinary terms: about six to seven weeks at the front, almost all of it on the move; nights spent in barns, ditches, kindergartens (one Bn HQ on 24 September was set up in a private boarding school with the children still living in part of the building), schools, and slit trenches; rations supplemented by potatoes and chickens from Belgian and Dutch farmers; long marches with the company carrier somewhere ahead; the rum issue at last light; and a battalion that, despite the casualties, kept up its sense of itself — the diary keeps mentioning cribbage games, photographs taken by Major Stott, and the CO's habit of getting "the familiar blue envelope" from home and exclaiming "That's the one." Bill was in this for one of its hardest stretches. + +--- + +## 2. Proposed Events List + +For each event: date, location as the diary records it, one-line description, why it matters for Bill's story, page reference as recorded in the diary's comments column. + +| # | Date | Location (diary text) | Description | Why it matters for Bill | Page ref | +|---|------|------------------------|-------------|--------------------------|----------| +| 1 | 15 Sep 1944 | Fme Geersen area, vicinity Loon Plage; companies at MR 147834, 148837, 148842 | Night action by Dog, Baker, Charlie Companies; one man's webbing found on bridge, "assumed taken prisoner." | The kind of fighting going on the day or two before reinforcements arrived — this is what Bill walked into. | (none recorded) | +| 2 | 16 Sep 1944 | Bn area near Loon Plage, France | "Banner day" for reinforcements: 30 other ranks plus Major Stott, Major Baker, Lts. Maguire, Mageli, Holmgren joined the battalion. | The most likely day Bill joined the unit; if not this draft, one of the smaller drafts in the following two weeks. | 14, 2, 14 | +| 3 | 18 Sep 1944 | Convoy from France into Belgium via Ypres, St. Julien (H-622556652), Passchendaele (H-685650), Poelcappelle (H-643685); ended at Wommelghem MR 7694 Sheet 23–33 | The battalion crossed the WWI battlefields where its predecessor 10th Bn fought on 22 April 1915; civilians showered the convoy with fruit and bread. | A representative "movement day" — far more typical of his service than combat days. | 15, 15, 15, 15 | +| 4 | 21–22 Sep 1944 | Albert Canal crossing at damaged lock gates, MR 769968; Bn HQ at Chateau Helleputte MR 769969 | "Charlie" Company crossed silently as a fighting patrol; whole battalion followed; bridgehead held against German counter-attack at first light. | Sgt. G.R. Crockett (15 Pl, Charlie Coy) recommended for the VC; the battalion's first major successful set-piece in this campaign. | 15 | +| 5 | 27 Sep 1944 | American Chateau MR 861035, Belgium Sheet 24 & 34 | Battalion concentrated at a "hotel-like lodge" before the push to Brecht; CO described in diary as "tired, overworked, worried and on edge." | Captures the tempo of a battalion under strain between actions. | 4, 15, 15 | +| 6 | 29–30 Sep 1944 | Tac H 873098; advance toward Brecht/St. Leonard | Calgary Highlanders attacked through R.H.C. positions toward grid 856089; Lt. Casey killed, Lt. Craddock wounded. | First major casualty day in his window; gives the family a sense of how exposed even a successful attack was. | (30 Sep: 15, 2; 29 Sep: none) | +| 7 | 3–4 Oct 1944 | Fort de Schooten, MR 7603; Bn HQ at Lochtenberg MR 788051 | Night assault on Fort de Schooten; Capt. Tennant patched up an abandoned punt with rags and wax to cross the moat. | A vivid example of how the battalion improvised under pressure. | 10, 3 | +| 8 | 7 Oct 1944 | Jansen Farm MR 626185, Sheet 23 NE; Hoogerheide objectives | "Ross Ellis Day" — battalion fought into Hoogerheide, took 62 prisoners. | First of the battalion's actions on Dutch soil; major engagement Bill was very likely present for. | 10 | +| 9 | 8–9 Oct 1944 | Jansen Farm 626185; junction 624198 ("Hell's Cross Roads") | Heavy German counter-attacks on Able Company; Capt. Tennant wounded, Major Kearns wounded by shrapnel at Bn HQ, Major MacKenzie wounded. | One of the worst days in the battalion's history during this period. | 8 Oct: 2; 9 Oct: 10, 1, 2 | +| 10 | 10 Oct 1944 | Junction 622195 ("Ross"), 61851995 ("Ellis") | Major Ellis led Baker Coy to retake lost ground; Capt. Bob Porter wounded; Bn moved to De Geest MR 622178 for rest. | The end of the worst phase of Hoogerheide. | 10 | +| 11 | 14 Oct 1944 | Bn HQ at Van den Maegdenbergh's farm, 600180; Hinkelenoord, Sheet 23 NE | Battalion took over from R.R.C. in static positions facing Woensdrecht across dykes. Scout Bill Alexander believed killed on patrol that night near 597206. | A representative "static" period of patrolling and OPs — very different from the set-piece days. | 3 | +| 12 | 16 Oct 1944 | Wolfert's Farm 599189 | DSO award to Lt.-Col. MacLauchlan announced; Major Baker reported in 8th General Hospital with peritonitis. | A morale-changing day for the unit. | 10, 3, 3, 2 | +| 13 | 21 Oct 1944 | Wolfert's Farm 599189 → P. van Gijn's home, Ossendrecht, MR 628166 | Fire destroyed the upstairs of the Bn HQ farmhouse; Pte. Sapinsky badly burned; battalion moved into Ossendrecht for a (curtailed) 48-hour rest. | A non-combat event the diary records in detail. | 10, 1, 2 | +| 14 | 23 Oct 1944 | Wolfert's Farm 599189 (re-occupied); attack to seal isthmus, objectives north of railway including 569203, 569206, 601213, 598206 | Capt. Pearson wounded; Lt. Wilkins took over Dog Company; battalion fought into the start of the South Beveland isthmus. | Major action Bill was likely involved in. | 10, 2 | +| 15 | 24–26 Oct 1944 | Woensdrecht; Bn HQ at home of van Liere 611201; objectives "Wynn 1, 2, 3" near grid 6122 | Multi-day push to clear the high ground; 13 prisoners on 24 Oct alone; close-range fighting — diary entry of 26 Oct describes a section throwing grenades into a German pillbox at very close range. | The action that finally sealed off the isthmus. | 24 Oct: 10, 1, 6; 25 Oct: 1, 10; 26 Oct: 10 | +| 16 | 28 Oct 1944 | Move from Woensdrecht across the Scheldt to Kruiningen 4223, Sheet 15 SW | Battalion crossed onto South Beveland; vehicles ferried over canal bridge; Bn HQ in a church at #36 Niewstraat. | Movement day across the captured isthmus. | 1, 10 | +| 17 | 29 Oct 1944 | Schore 4024 → Heer Hendrikskinderen, house at 311296 | Lt.-Col. MacLauchlan removed from command "almost in tears"; Major Ross Ellis took over the battalion. | Major leadership change just before the Causeway action. | 10, 8 | +| 18 | 31 Oct – 1 Nov 1944 | Walcheren Causeway, MR 2030 (Composite Sheet 14 NW NE SW SE); Tac H 217298, Rear Tac H 223293 | Calgary Highlanders attacked across the Causeway; Baker Coy halted midway; Dog Coy crossed at 0605 on 1 Nov; Able and Charlie followed. | **Most likely date and place Bill was wounded.** | 10 | +| 19 | 1–2 Nov 1944 | Causeway and dyke 208303 | Capt. Lasher wounded; Lt. Moffat killed; Lts. Lefroy, Schoening, Hoy wounded; battalion held east end of Causeway until R. de Mais and Glasgow Highlanders relieved. | The 40-hour engagement during which Bill was almost certainly hit. | 1 Nov: 10, 1, 2; 2 Nov: 10, 5 | +| 20 | 3 Nov 1944 | Lierre, Belgium, Sheet 24 34, MR 786861 | Battalion arrived for rest at the Technical School; men billeted in private homes; first quiet night since mid-October. | The day Bill's battalion went out of the line — he was already gone by this point. | 1 | + +--- + +## 3. Documents and Passages of Interest + +These are diary entries the family may want to read in full. Where a date has more than one passage of interest, we've separated them. + +| # | Date | Page ref | Why it's worth reading | +|---|------|----------|--------------------------| +| 1 | 16 Sep 1944 | 14, 2, 14 | The "banner day" reinforcement entry. Names the officers who came in, captures the ribbing of Major Baker about the "Basingstoke rumour," and gives a sense of what a reinforcement day actually felt like. The most likely day Bill joined. | +| 2 | 18 Sep 1944 | 15 (×4) | The drive across the Belgian frontier and the WWI battlefields. Vivid, unusual content — the diarist clearly moved by passing St. Julien Wood, where the regiment's predecessor unit fought in April 1915. The entry also has incidental detail (the "raw recruits in the White Brigade drilling near a detour sign" who collapsed in confusion when their squad was halted) that gives a sense of the diarist's voice. | +| 3 | 21 Sep 1944 | 15 | The Albert Canal crossing — the operation that established the battalion's reputation in this campaign. Includes the line that captures MacLauchlan's manner: "with a prophetic note declared 'We have made it.'" | +| 4 | 7 Oct 1944 | 10 | "Ross Ellis Day" — the fight into Hoogerheide. The diarist's account of the C.O. "almost in tears" when Capt. Tennant was hit ("There goes a stout fellow! Worth three men to us!") is one of the most human moments in the diary. | +| 5 | 9 Oct 1944 | 10, 1, 2 | The Able Company crisis at Jansen Farm — the day the company was overrun and Lt. Munro fought to bring out survivors, with Major Kearns wounded by shrapnel at Battalion HQ. A long, dense entry that gives the truest picture of how confused front-line fighting actually was. | +| 6 | 16 Oct 1944 | 10, 3, 3, 2 | The DSO award to Lt.-Col. MacLauchlan. Includes the touching detail that "I" Section celebrated by opening cognac donated by Major Ellis. | +| 7 | 21 Oct 1944 | 10, 1, 2 | The fire at Wolfert's Farm. An unusual non-combat entry — the battalion's own tac HQ burned down while the CO held an open-air orders group, and the diary describes the men salvaging the farmer's furniture. | +| 8 | 27 Oct 1944 | 10, 10, 11 | Long entry on the day the Calgary Highlanders came out of the line at Woensdrecht. Includes the GOC's praise ("The Calgary Highlanders have done a damn fine job for the Division"), the rare quiet humour around the Coy Commanders' meeting, and the German note left on Dog Company's objective: "Dear Tommy. If you get this room, thought to that, that you after a short time, go to the heaven. With the best heartfully wishes. Yours Fritz." | +| 9 | 29 Oct 1944 | 10, 8 | Lt.-Col. MacLauchlan's removal from command — "almost in tears, bidding farewell to the officers and men" — and Major Ellis taking over. The pivot point of this whole period. | +| 10 | 1 Nov 1944 | 10, 1, 2 | The Walcheren Causeway action. The most likely day Bill was wounded. The entry includes the Royal Canadian Engineer bulldozer operator's remark that he could fill in the crater "Well, the Calgary Highlanders are out there to protect us." | + +--- + +## Notes on uncertainty + +What we don't know, and chose not to invent: + +- **Bill's company.** The diary names the rifle companies (Able, Baker, Charlie, Dog) and Support Company throughout, and almost every entry describes one company doing one thing and another doing another. With no information on Bill's company we can't say which fights he was in personally — only which fights his battalion was in. +- **His exact arrival date.** "Mid-September" is consistent with the 16 September draft, but smaller drafts came in over the following weeks; he could have been any of those. +- **Where on the Causeway he was hit.** The diary records officer casualties by name and gives total numbers, but does not list other-rank wounded individually. "Early November during the Scheldt fighting" is most consistent with 1 November, possibly 2 November, on the Causeway. +- **The wound itself.** Diary describes the engagement, not individual injuries. His evacuation to England aligns with the standard chain (RAP → field ambulance → casualty clearing station → general hospital → cross-Channel). + +If his service file is available through Library and Archives Canada, it should resolve company, date of wounding, and nature of wound — the diary alone cannot. diff --git a/p44-ocr-viewer.html b/p44-ocr-viewer.html new file mode 100644 index 0000000..5bfa8c5 --- /dev/null +++ b/p44-ocr-viewer.html @@ -0,0 +1,1054 @@ + + + + + +P44 OCR Viewer — Calgary Highlanders Sep 44 + + + + + +
+ + +
+

⚜ P44 OCR VIEWER

+
+ + +
+ +
+ PDF + +
+
+ OCR + +
+
+ JSON + +
+
+ +
+ SCALE + +
+
+ +
+ + +
+ + +
+
+ Source Document — Page Scan +
+ + + + / — +
+
+
+
+ +
+ War diary scan loads automatically.
+ Run launch_viewer.py — or click the button above +
+
+ +
+
+ + +
+
+ OCR Text Output + +
+
+
+ +
+ OCR transcript loads automatically.
+ Run launch_viewer.py — or click the button above +
+
+ +
+
+
+ + + +
+ + + + + + diff --git a/prompts/unit-selection.md b/prompts/unit-selection.md index 5c1da27..68fa062 100644 --- a/prompts/unit-selection.md +++ b/prompts/unit-selection.md @@ -4,7 +4,8 @@ Calgary highlanders, part of 5th Canadian Infantry Brigade, 2nd Canadian Infantry Division. ## Why this unit -Good OCR coverage, high quality PDFs with lots of narrative. Content spans Sep 44 to Nov 44, though the OCR text extends to May 45, nothign is needed past Dec 44. +Good OCR coverage, high quality PDFs with lots of narrative. Content spans Sep 44 to Nov 44, +though the OCR text extends to May 45, nothing is needed past Dec 44. ## Hypothetical customer scenario Pte. Bloggins, reinforcement who joined the battalion mid September 1944 @@ -15,4 +16,6 @@ Family knows: the regiment, that he was wounded somewhere in "Holland or Germany in late 1944, the rough timeframe. ## What I expect the report to cover -Fighting in France and Belgium, the advance towards Antwerp, and the Scheldt Battles. Should feel like a personal narrative, outlining the experience of a private during the battles. Try and find any personal narratives that might give the family a sense of what the battles were like, or life was like. \ No newline at end of file +Fighting in France and Belgium, the advance towards Antwerp, and the Scheldt Battles. +Should feel like a personal narrative, outlining the experience of a private during the battles. +Try and find any personal narratives that might give the family a sense of what the battles were like, or life was like. \ No newline at end of file diff --git a/prompts/v1-tier3-report.md b/prompts/v1-tier3-report.md new file mode 100644 index 0000000..8078ef9 --- /dev/null +++ b/prompts/v1-tier3-report.md @@ -0,0 +1,97 @@ +# Prompt: Tier 3 Descendant Report — Calgary Highlanders + +## Your role +You are helping produce a report for a descendant of a Canadian WWII soldier. +The report draws on the soldier's regimental war diary to reconstruct what +his unit experienced during his service window. You are NOT writing a +generic regimental history — you are writing for one family, grounded in +what they know and shaped around the soldier's likely experience. + +## Input +You will be given the war diary OCR for the Calgary Highlanders covering +September 1944 through May 1945. The diary is in .docx form with a three- +column structure: +- Column 1: Date +- Column 2: Narrative entry (the diarist's account of the day) +- Column 3: Comments / page reference back to the original diary + +Each row is a day's entry. Some entries are operational (movements, attacks, +casualties), some are administrative (parades, pay, training), some are +human texture (weather, food, civilian encounters, rest). All of these +matter for the report — the human texture is what makes a descendant report +emotionally resonant, not just the operational events. + +## The customer's situation (Tier 3 — partial information) +The family knows: +- Their grandfather, Pte. Bill Bloggins, served with the Calgary Highlanders. +- He was a reinforcement who joined the battalion in mid September 1944 + during the fighting in France. +- He was wounded in action in early November 1944, somewhere in + "Holland or Germany," during what the family calls "the Scheldt." +- He was evacuated to England and did not return to the unit. + +The family does NOT know: +- The exact date he joined or was wounded. +- Specific actions he was personally involved in. +- His company, platoon, or section. + +## What to produce + +### 1. Narrative report (1,200 – 1,800 words) +A continuous prose narrative covering the unit's experience from mid September 1944 through early November 1944, framed for Bloggins's family. +Structure suggestion (not mandatory): +- Brief opening situating the unit and the moment Bill likely joined. +- The Scheldt fighting as his first weeks with the battalion. +- The static winter on the Maas. +- The Rhineland operations and the period during which Bill was wounded. +- A short closing reflecting on what his service likely looked like. + +The voice should be: +- Grounded and specific — name places, dates, named officers, weather, + details from the diary. Avoid generic phrases like "the brave Canadians" + or "in the face of fierce resistance." +- Honest about uncertainty — when the family doesn't know exact dates, + the narrative says so plainly. ("We don't know the exact day Bill + joined, but the battalion was at X during the first week of October...") +- Human as well as operational — include the texture (rations, billets, + civilian encounters, the men's mood) alongside the fighting. + +### 2. Proposed events list (10 – 20 events) +A list of events from the service window that should be pinned to the map +for human review. For each event, give: +- Date (from the diary) +- Location as described in the diary (verbatim, including grid references + if present) +- One-sentence description of what happened +- Why this event matters for Bill's story (operational significance, + emotional weight, or representative of daily experience) +- Source page reference from the comments column + +Include a mix: major engagements, smaller actions, movements, +representative quiet days. Not every event needs to be a battle. + +### 3. Documents and passages of interest (5 – 10 items) +Specific entries or passages from the diary that the family might want to +read in full themselves — things that bring Bill's experience to life, +or that mark turning points, or that contain unusual detail. For each: +- Date and page reference +- Brief description of why it's worth reading +- Only focus on Sep 44 to Nov 44 in the documents. Skip any months after that. + +## What NOT to do + +- Do NOT invent specifics. If the diary doesn't say where the battalion + was on a given day, say so. If a name isn't in the diary, don't add one. +- Do NOT smooth over gaps in the record with plausible-sounding filler. +- Do NOT write in a generic war-documentary voice. Stay close to the + diary's actual content and tone. +- Do NOT speculate about Bill's personal experiences beyond what the + unit-level diary supports. We don't know if he was scared, brave, + homesick, etc. We know what his battalion did. +- Do NOT pad the narrative to hit the word count. If the diary is thin + for a period, the narrative is thin for that period. + +## Output format +Produce the three sections in order, clearly labeled. Use Markdown. +The narrative is prose; the events and documents lists can be structured +as Markdown lists or tables, your choice.**** \ No newline at end of file diff --git a/prompts/v2-tier3-report.md b/prompts/v2-tier3-report.md new file mode 100644 index 0000000..8078ef9 --- /dev/null +++ b/prompts/v2-tier3-report.md @@ -0,0 +1,97 @@ +# Prompt: Tier 3 Descendant Report — Calgary Highlanders + +## Your role +You are helping produce a report for a descendant of a Canadian WWII soldier. +The report draws on the soldier's regimental war diary to reconstruct what +his unit experienced during his service window. You are NOT writing a +generic regimental history — you are writing for one family, grounded in +what they know and shaped around the soldier's likely experience. + +## Input +You will be given the war diary OCR for the Calgary Highlanders covering +September 1944 through May 1945. The diary is in .docx form with a three- +column structure: +- Column 1: Date +- Column 2: Narrative entry (the diarist's account of the day) +- Column 3: Comments / page reference back to the original diary + +Each row is a day's entry. Some entries are operational (movements, attacks, +casualties), some are administrative (parades, pay, training), some are +human texture (weather, food, civilian encounters, rest). All of these +matter for the report — the human texture is what makes a descendant report +emotionally resonant, not just the operational events. + +## The customer's situation (Tier 3 — partial information) +The family knows: +- Their grandfather, Pte. Bill Bloggins, served with the Calgary Highlanders. +- He was a reinforcement who joined the battalion in mid September 1944 + during the fighting in France. +- He was wounded in action in early November 1944, somewhere in + "Holland or Germany," during what the family calls "the Scheldt." +- He was evacuated to England and did not return to the unit. + +The family does NOT know: +- The exact date he joined or was wounded. +- Specific actions he was personally involved in. +- His company, platoon, or section. + +## What to produce + +### 1. Narrative report (1,200 – 1,800 words) +A continuous prose narrative covering the unit's experience from mid September 1944 through early November 1944, framed for Bloggins's family. +Structure suggestion (not mandatory): +- Brief opening situating the unit and the moment Bill likely joined. +- The Scheldt fighting as his first weeks with the battalion. +- The static winter on the Maas. +- The Rhineland operations and the period during which Bill was wounded. +- A short closing reflecting on what his service likely looked like. + +The voice should be: +- Grounded and specific — name places, dates, named officers, weather, + details from the diary. Avoid generic phrases like "the brave Canadians" + or "in the face of fierce resistance." +- Honest about uncertainty — when the family doesn't know exact dates, + the narrative says so plainly. ("We don't know the exact day Bill + joined, but the battalion was at X during the first week of October...") +- Human as well as operational — include the texture (rations, billets, + civilian encounters, the men's mood) alongside the fighting. + +### 2. Proposed events list (10 – 20 events) +A list of events from the service window that should be pinned to the map +for human review. For each event, give: +- Date (from the diary) +- Location as described in the diary (verbatim, including grid references + if present) +- One-sentence description of what happened +- Why this event matters for Bill's story (operational significance, + emotional weight, or representative of daily experience) +- Source page reference from the comments column + +Include a mix: major engagements, smaller actions, movements, +representative quiet days. Not every event needs to be a battle. + +### 3. Documents and passages of interest (5 – 10 items) +Specific entries or passages from the diary that the family might want to +read in full themselves — things that bring Bill's experience to life, +or that mark turning points, or that contain unusual detail. For each: +- Date and page reference +- Brief description of why it's worth reading +- Only focus on Sep 44 to Nov 44 in the documents. Skip any months after that. + +## What NOT to do + +- Do NOT invent specifics. If the diary doesn't say where the battalion + was on a given day, say so. If a name isn't in the diary, don't add one. +- Do NOT smooth over gaps in the record with plausible-sounding filler. +- Do NOT write in a generic war-documentary voice. Stay close to the + diary's actual content and tone. +- Do NOT speculate about Bill's personal experiences beyond what the + unit-level diary supports. We don't know if he was scared, brave, + homesick, etc. We know what his battalion did. +- Do NOT pad the narrative to hit the word count. If the diary is thin + for a period, the narrative is thin for that period. + +## Output format +Produce the three sections in order, clearly labeled. Use Markdown. +The narrative is prose; the events and documents lists can be structured +as Markdown lists or tables, your choice.**** \ No newline at end of file diff --git a/scripts/check_eod.py b/scripts/check_eod.py new file mode 100644 index 0000000..92ae1e0 --- /dev/null +++ b/scripts/check_eod.py @@ -0,0 +1,33 @@ +import json, pathlib +from collections import Counter, defaultdict + +data = json.loads(pathlib.Path('outputs/Calgary-Highlanders_Sep44_positions.json').read_text()) + +cats = Counter(p['category'] for p in data) +print('=== CATEGORIES ===') +for k, v in sorted(cats.items()): + print(f' {k}: {v}') + +eod = [p for p in data if p['is_end_of_day']] +print(f'\n=== EOD entries: {len(eod)} ===') +for p in eod: + print(f" {str(p['date']):<16} cat={p['category']:<14} grid={str(p['grid']):<8} place={p['place_name']}") + +eod_by_date = defaultdict(list) +for p in eod: + eod_by_date[p['date']].append(p) +multi = {d: ps for d, ps in eod_by_date.items() if len(ps) > 1} +if multi: + print('\nWARNING - multiple EOD on same date:') + for d, ps in multi.items(): + print(f' {d}: {len(ps)} entries') +else: + print('\nOK: exactly one EOD per date') + +# List dates with no EOD +all_dates = sorted({p['date'] for p in data if p['date']}) +eod_dates = set(eod_by_date.keys()) +missing = [d for d in all_dates if d not in eod_dates] +if missing: + print(f'\nDates with NO EOD: {missing}') + diff --git a/scripts/extract_positions.py b/scripts/extract_positions.py new file mode 100644 index 0000000..910f63a --- /dev/null +++ b/scripts/extract_positions.py @@ -0,0 +1,721 @@ +""" +Extract positional data from Calgary Highlanders War Diary Sep 44 (pages 7-57). +Outputs a JSON array of position objects. +""" + +import re +import json +from html.parser import HTMLParser + +INPUT_FILE = r"C:\Users\natha\IdeaProjects\AI-Prototype\Inputs\ocr-output\Calgary-Highlanders_War-Diary_Sep44_olmocr.md" +OUTPUT_FILE = r"C:\Users\natha\IdeaProjects\AI-Prototype\outputs\Calgary-Highlanders_Sep44_positions.json" + +# ── helpers ────────────────────────────────────────────────────────────────── + +def expand_grid(raw: str) -> tuple[str, bool]: + """ + Return (6-figure-string, inferred). + 4-figure AABB → centre of 1km square = AA5 BB5 (grid_inferred=True) + 6-figure AAABBB → returned as-is (grid_inferred=False) + 8-figure AAAABBBB → truncate to AAA BBB (grid_inferred=False) + """ + s = re.sub(r'[^0-9]', '', raw) + if len(s) == 4: + # e.g. "8450" → easting 84, northing 50 → centre 845, 505 + e = s[0:2] + '5' + n = s[2:4] + '5' + return e + n, True + if len(s) == 6: + return s, False + if len(s) == 8: + # AAAABBBB: easting = AAAA (take first 3), northing = BBBB (take first 3) + # e.g. 15618050 → E=1561 → 156, N=8050 → 805 → "156805" + e = s[0:3] + n = s[4:7] + return e + n, False + return s, False + + +_GRID_RE = re.compile( + r'\b' + r'(?:MR\s*|GR\s*)?' # optional prefix + r'([0-9]{3,4}\s*[0-9]{3,4})' # 6 or 4+4 digit grid + r'\b', + re.IGNORECASE +) + +def find_grids(text: str): + """Return list of (raw_match, cleaned_digits).""" + found = [] + for m in _GRID_RE.finditer(text): + raw = m.group(1) + digits = re.sub(r'\s', '', raw) + if len(digits) in (4, 6, 8): + found.append((m.group(0).strip(), digits)) + # also pick up standalone 6-digit runs not already caught + for m in re.finditer(r'\b([0-9]{6})\b', text): + digits = m.group(1) + already = any(d == digits for _, d in found) + if not already: + found.append((digits, digits)) + return found + + +# ── HTML strip ──────────────────────────────────────────────────────────────── + +class _Stripper(HTMLParser): + def __init__(self): + super().__init__() + self.parts = [] + def handle_data(self, data): + self.parts.append(data) + def get_text(self): + return ' '.join(p for p in self.parts if p.strip()) + +def strip_html(html_str: str) -> str: + s = _Stripper() + # replace
with spaces for readability + html_str = re.sub(r'', ' ', html_str, flags=re.IGNORECASE) + s.feed(html_str) + return s.get_text() + + +# ── categorise ─────────────────────────────────────────────────────────────── + +_SUBUNIT_RE = re.compile( + r'\b(Able\s+Coy|Baker\s+Coy|Charlie\s+Coy|Dog\s+Coy|' + r'"A"\s+Coy|"B"\s+Coy|"C"\s+Coy|"D"\s+Coy|' + r"'A'\s+Coy|'B'\s+Coy|'C'\s+Coy|'D'\s+Coy|'Able'\s+Coy|'Baker'\s+Coy|'Charlie'\s+Coy|'Dog'\s+Coy|" + r'Carrier\s+Platoon|carriers|Pioneer\s+Platoon|pioneers|scouts?|Scout\s+Platoon|' + r'Support\s+Coy|Anti[-\s]?[Tt]ank\s+Pl(?:atoon)?|mortar\s+platoon|' + r'\d+\s*Pl(?:atoon)?|' + r'\d+\s*Sec(?:tion)?)\b', + re.IGNORECASE +) +_ENEMY_RE = re.compile( + r'\b(enemy|Jerry|Hun|Boche|MG\s*pos|MMG\s*pos|sniper|' + r'counter.attack|strongpoint|mortar\s*pos|machine\s*gun|' + r'block\s*house|pill\s*box|88mm|S\.S\.|SS troop)\b', + re.IGNORECASE +) +_PATROL_RE = re.compile( + r'\b(patrol|recce\s*patrol|fighting\s*patrol|' + r'standing\s*patrol|OP|O\.P\.|observation\s*post)\b', + re.IGNORECASE +) +_FRIENDLY_UNITS = re.compile( + r'\b(R\.H\.C\.|Black\s*Watch|RHC|R\s*de\s*Mais|' + r'Fus\s*M\.R\.|F\.M\.R\.|R\.R\.C\.|' + r'4\s*S\.S\.|Royal\s*Regt|S\.Sask|' + r'Tor\s*Scots|Toronto\s*Scots|' + r'White\s*Brig(?:ade)?|F\.F\.I\.|Maquis)\b', + re.IGNORECASE +) +# Tac HQ and Bn HQ in narrative +_TAC_HQ_RE = re.compile( + r'\bTac\s*H(?:Q)?\b', + re.IGNORECASE +) +_BN_HQ_RE = re.compile( + r'\b(?:Bn\.?\s*H\.?Q\.?|Battalion\s*H\.?Q\.?|' + r'Battle\s*H\.?Q\.?|Command\s*Post|' + r'moved\s+(?:his\s+)?H\.?Q\.|' + r'set\s+up\s+H\.?Q\.|' + r'took\s+up\s+(?:a\s+)?H\.?Q\.)\b', + re.IGNORECASE +) + +def categorise(sentence: str, place_name: str | None, subunit: str | None, + friendly: str | None, is_place_col: bool = False) -> str: + """ + Categories (in priority order): + TAC_HQ – Tactical HQ position + BN_HQ – Battalion HQ position (incl. all place-column entries) + FRIENDLY – another friendly unit's position + ENEMY – enemy position / feature + PATROL – patrol route/endpoint + SUBUNIT – company or platoon position + UNIT_MOVEMENT – general battalion move/position + MISC – catch-all + """ + if is_place_col: + # Place column always records where Bn HQ was + if _TAC_HQ_RE.search(sentence): + return "TAC_HQ" + return "BN_HQ" + if _TAC_HQ_RE.search(sentence): + return "TAC_HQ" + if _BN_HQ_RE.search(sentence): + return "BN_HQ" + if friendly: + return "FRIENDLY" + if _ENEMY_RE.search(sentence): + return "ENEMY" + if _PATROL_RE.search(sentence): + return "PATROL" + if subunit: + return "SUBUNIT" + return "UNIT_MOVEMENT" + + +def extract_subunit(sentence: str) -> str | None: + m = _SUBUNIT_RE.search(sentence) + if m: + return m.group(0).strip() + return None + +def extract_friendly(sentence: str) -> str | None: + m = _FRIENDLY_UNITS.search(sentence) + if m: + return m.group(0).strip() + return None + + +# ── split sentences ────────────────────────────────────────────────────────── + +def split_sentences(text: str): + """Rough sentence splitter – split on '. ' or '
'.""" + # normalise + text = re.sub(r'\s+', ' ', text).strip() + parts = re.split(r'(?<=[.!?])\s+(?=[A-Z"\'(])', text) + return [p.strip() for p in parts if p.strip()] + + +# ── page extractor ──────────────────────────────────────────────────────────── + +_PAGE_RE = re.compile(r'^## Page (\d+)\s*$', re.MULTILINE) + +def extract_pages(text: str, first: int, last: int) -> str: + pages = list(_PAGE_RE.finditer(text)) + start_idx = None + end_idx = len(text) + for i, m in enumerate(pages): + n = int(m.group(1)) + if n == first and start_idx is None: + start_idx = m.start() + if n == last + 1 and start_idx is not None: + end_idx = m.start() + break + if start_idx is None: + return "" + return text[start_idx:end_idx] + + +# ── table row parser ────────────────────────────────────────────────────────── + +_ROW_RE = re.compile(r'(.*?)', re.DOTALL | re.IGNORECASE) +_CELL_RE = re.compile(r']*>(.*?)', re.DOTALL | re.IGNORECASE) + +def parse_table_rows(table_html: str) -> list[dict]: + """Parse a single HTML table into list of {place, date, hour, summary} dicts.""" + rows = [] + for row_m in _ROW_RE.finditer(table_html): + cells = [strip_html(c.group(1)).strip() + for c in _CELL_RE.finditer(row_m.group(1))] + if len(cells) < 4: + continue + # skip header rows + if re.match(r'place|date|hour|summary|no\.', cells[0], re.IGNORECASE): + continue + place = cells[0] if cells[0] else None + date = cells[1] if len(cells) > 1 else None + hour = cells[2] if len(cells) > 2 else None + summary = cells[3] if len(cells) > 3 else "" + rows.append(dict(place=place, date=date, hour=hour, summary=summary)) + return rows + + +# ── sheet ref extractor ─────────────────────────────────────────────────────── +_SHEET_RE = re.compile( + # must start with a digit to avoid matching "Sheet Ste. Foy" etc. + r'Sheet\s+(\d[A-Z0-9]*(?:\s*[&\-]\s*\d[A-Z0-9]*)?)', + re.IGNORECASE +) + +def extract_sheet(place_text: str) -> str | None: + if not place_text: + return None + # find the LAST occurrence (most specific sheet reference in column) + matches = list(_SHEET_RE.finditer(place_text)) + if matches: + return "Sheet " + matches[-1].group(1).strip() + return None + +# also pull MR grid from place column +_PLACE_MR_RE = re.compile(r'MR\s*([0-9]{4,8}|\d{2,3}\s*\d{2,3})', re.IGNORECASE) + +def extract_place_name(place_text: str) -> str | None: + """ + Extract the primary place name from the Place column. + Strip country, bare MR tokens, Sheet refs, standalone digits, + and Tac H / Fort coord suffixes. + """ + if not place_text: + return None + s = place_text + # remove country names + s = re.sub(r'\b(France|Belgium|Holland|Netherlands)\b', '', s, flags=re.IGNORECASE) + # remove Sheet + value (digit-led) + s = re.sub(r'\bSheet\s+\d[\w\s&\-]*', ' ', s, flags=re.IGNORECASE) + # remove bare "Sheet" not followed by a digit + s = re.sub(r'\bSheet\b', ' ', s, flags=re.IGNORECASE) + # remove MR + optional digits/spaces + s = re.sub(r'\bMR\b\s*[\d\s]*', ' ', s, flags=re.IGNORECASE) + # remove Tac H + coords + s = re.sub(r'\bTac\s*H\s*[\d]+\b', ' ', s, flags=re.IGNORECASE) + # remove bare 4-8 digit grid refs + s = re.sub(r'\b\d{4,8}\b', ' ', s) + # remove "X Pub" and similar codes + s = re.sub(r'\bX[\s\-]?Pub\b', ' ', s, flags=re.IGNORECASE) + # remove orphan punctuation / collapse whitespace + s = re.sub(r'[/\\|]', ' ', s) + s = re.sub(r'\s+', ' ', s).strip().strip('.,-()') + # If multiple tokens, take first meaningful phrase (up to the first double space separator) + parts = [p.strip() for p in re.split(r'\s{2,}', s) if p.strip()] + result = parts[0] if parts else s.strip() + # Trim trailing stray words like "(outskirts...)" parenthetical noise + result = re.sub(r'\s*\(.*\)\s*$', '', result).strip() + return result if result else None + + +def clean_date(raw: str) -> str | None: + if not raw: + return None + d = re.sub(r'\s+', ' ', raw).strip() + # Remove parenthetical notes like "(Cont)" + d = re.sub(r'\(.*?\)', '', d).strip() + # Extract DD Mon [YY] — ignore anything else in the cell (e.g. embedded grid refs) + m = re.search( + r'(\d{1,2})\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s*(44|1944)?', + d, re.IGNORECASE + ) + if m: + day = m.group(1) + mon = m.group(2).capitalize() + # always use "44" as year for this diary + return f"{day} {mon} 44" + return None # unrecognisable — don't inherit garbage + + +def clean_hour(raw: str) -> str | None: + if not raw: + return None + raw = raw.strip() + # Reject empty, the bare year "44", and plain digit-strings ≥4 chars (grids) + if not raw or raw in ('0', '44', '1944'): + return None + if re.match(r'^\d{4,6}$', raw): + return None # grid ref leaked into hour column, not a time + # Extract a valid HHMM time + m = re.search(r'\b([012]\d[0-5]\d)\b', raw) + if m: + return m.group(1) + return None + + +def hour_col_grid(raw: str) -> str | None: + """ + If the Hour column contains a plain 4-digit grid ref (not a time), + return the digit string so it can be added as a BN_HQ entry. + """ + if not raw: + return None + raw = raw.strip() + if re.match(r'^\d{4}$', raw): + # Confirm it can't be a time (hours > 23 or minutes > 59) + h, m2 = int(raw[:2]), int(raw[2:]) + if h > 23 or m2 > 59: + return raw + return None + + +# ── place-column MR grid ────────────────────────────────────────────────────── + +def place_col_positions(place_text: str, date: str, sheet: str) -> list[dict]: + """ + Extract grids from the Place column. + Handles: + (a) NAME + MR + GRID (e.g. 'Ste. Foy MR 2553') + (b) NAME + bare 6-digit grid (e.g. 'Chateau Helleputte 769969') + Returns one entry per unique grid found. + """ + if not place_text: + return [] + + results = [] + seen_digits = set() + + def _clean_label(raw: str) -> str | None: + s = raw + s = re.sub(r'\b(France|Belgium|Holland|Netherlands)\b', '', s, flags=re.IGNORECASE) + s = re.sub(r'\bSheet\b[\w\s&\-]*', '', s, flags=re.IGNORECASE) + s = re.sub(r'\bMR\b', '', s, flags=re.IGNORECASE) + s = re.sub(r'\b\d{4,8}\b', '', s) + s = re.sub(r'\s+', ' ', s).strip().strip('.,-()') + # take rightmost meaningful word group + parts = [p.strip() for p in re.split(r'\s{2,}', s) if p.strip()] + r = parts[-1] if parts else s.strip() + return r if r else None + + # ── (a) NAME MR GRID ───────────────────────────────────────────────────── + # Require digits immediately after MR (captures 4-digit and 6-digit grids) + mr_pattern = re.compile( + r'([A-Z][A-Za-zÀ-ÿ\s\.\-\']*?)\s+MR\s*(\d[\d\s]+\d)', + re.IGNORECASE + ) + for m in mr_pattern.finditer(place_text): + label_raw = m.group(1) + raw_digits = re.sub(r'\s', '', m.group(2)) + if len(raw_digits) not in (4, 6, 8): + continue + if raw_digits in seen_digits: + continue + seen_digits.add(raw_digits) + + grid, inferred = expand_grid(raw_digits) + if len(grid) != 6: + grid = None + + label = _clean_label(label_raw) + # Determine category: place column = BN_HQ unless Tac H explicit + cat = categorise(place_text, label, None, None, is_place_col=True) + results.append(dict( + date=date, hour=None, + grid=grid, grid_inferred=inferred, + place_name=label, + sheet_ref=sheet, + category=cat, + subunit=None, friendly_unit=None, is_end_of_day=False, + context=f"Place column: {label or '?'} MR {raw_digits}" + )) + + # ── (b) NAME followed directly by bare 6-digit grid ────────────────────── + bare_pattern = re.compile( + r'([A-Z][A-Za-zÀ-ÿ\s\.\-\']+?)\s+(\d{6})\b' + ) + for m in bare_pattern.finditer(place_text): + label_raw = m.group(1) + raw_digits = m.group(2) + if raw_digits in seen_digits: + continue + # Skip if label contains only noise words + clean = re.sub(r'\b(France|Belgium|Holland|Netherlands|Sheet|MR)\b', '', + label_raw, flags=re.IGNORECASE).strip() + if not clean: + continue + seen_digits.add(raw_digits) + + grid, inferred = expand_grid(raw_digits) + label = _clean_label(label_raw) + cat = categorise(place_text, label, None, None, is_place_col=True) + results.append(dict( + date=date, hour=None, + grid=grid, grid_inferred=inferred, + place_name=label, + sheet_ref=sheet, + category=cat, + subunit=None, friendly_unit=None, is_end_of_day=False, + context=f"Place column: {label or '?'} {raw_digits}" + )) + + # ── (c) NAME followed by bare 4-digit grid (no MR prefix) ──────────────── + bare_4_pattern = re.compile( + r'([A-Z][A-Za-zÀ-ÿ\s\.\-\']+?)\s+(\d{4})\b' + ) + for m in bare_4_pattern.finditer(place_text): + label_raw = m.group(1) + raw_digits = m.group(2) + if raw_digits in seen_digits: + continue + # Only treat as grid if NOT a valid time (i.e. HH>23 or MM>59) + h_val, mn_val = int(raw_digits[:2]), int(raw_digits[2:]) + if h_val <= 23 and mn_val <= 59: + continue + clean = re.sub(r'\b(France|Belgium|Holland|Netherlands|Sheet|MR)\b', '', + label_raw, flags=re.IGNORECASE).strip() + if not clean: + continue + seen_digits.add(raw_digits) + + grid, inferred = expand_grid(raw_digits) + label = _clean_label(label_raw) + cat = categorise(label_raw, label, None, None, is_place_col=True) + results.append(dict( + date=date, hour=None, + grid=grid if len(grid) == 6 else None, + grid_inferred=inferred, + place_name=label, + sheet_ref=sheet, + category=cat, + subunit=None, friendly_unit=None, is_end_of_day=False, + context=f"Place column: {label or '?'} {raw_digits}" + )) + + return results + + +# ── main extraction ─────────────────────────────────────────────────────────── + +def extract_positions(md_text: str) -> list[dict]: + section = extract_pages(md_text, 7, 57) + + # find all tables + table_re = re.compile(r'(.*?)
', re.DOTALL | re.IGNORECASE) + + all_positions = [] + last_date = None + last_sheet = None + + for tbl_m in table_re.finditer(section): + tbl_html = tbl_m.group(0) + rows = parse_table_rows(tbl_html) + if not rows: + continue + + for row in rows: + raw_date = clean_date(row.get('date', '') or '') + raw_hour = clean_hour(row.get('hour', '') or '') + place_text = row.get('place', '') or '' + summary = row.get('summary', '') or '' + + # update tracking state + if raw_date: + last_date = raw_date + cur_date = last_date + + sheet = extract_sheet(place_text) or last_sheet + if sheet and re.search(r'\d', sheet): + last_sheet = sheet + + # ── place-column positions ── + for pos in place_col_positions(place_text, cur_date, sheet): + pos['hour'] = raw_hour + all_positions.append(pos) + + # ── hour-column grid (OCR sometimes puts MR ref here) ── + hr_raw = row.get('hour', '') or '' + hcg = hour_col_grid(hr_raw) + if hcg: + grid, inferred = expand_grid(hcg) + col_place = extract_place_name(place_text) + all_positions.append(dict( + date=cur_date, hour=None, + grid=grid if len(grid) == 6 else None, + grid_inferred=inferred, + place_name=col_place, + sheet_ref=last_sheet, + category="BN_HQ", + subunit=None, friendly_unit=None, is_end_of_day=False, + context=f"Place column (hour field): {col_place or '?'} {hcg}" + )) + + # ── summary / narrative positions ── + sentences = split_sentences(summary) + # track whether this is the last sentence in the entry + last_sentence_idx = len(sentences) - 1 + + # collect all (sentence_idx, grid_str, raw_context) tuples for this row + row_matches = [] + for s_idx, sent in enumerate(sentences): + grids = find_grids(sent) + if grids: + for raw_match, digits in grids: + row_matches.append((s_idx, sent, digits, raw_match)) + else: + # named positions without grids: look for capitalised place names + named = re.findall( + r'\b(Fme\s+\w+|Chateau\s+\w+|Fort\s+\w+|Casino|' + r'Distillery\s+\w+|Brickworks|blockhouse|' + r'Mardick|Dunkerque|Dunkirk|Brecht|Loon\s*Plage|' + r'Bourbourgville|St\.\s*Folquin|Nordamsques|Montreuil|' + r'Wommelg[ea]h?m|Antwerp|Ypres|Pasch[aeo]nd[ae]?le|' + r'Gravelines|Le\s*Clipon|Coppenaxfort|' + r'Lochtenberg|Eindhoven|Sternhoven|Ryckevorsel|' + r'St\.\s*Leonard|Bindhoven|Schilde|Schelde)\b', + sent, re.IGNORECASE + ) + for name in named: + row_matches.append((s_idx, sent, None, name)) + + # determine is_end_of_day: last grid-bearing sentence in last row of date? + for i, (s_idx, sent, digits, raw_match) in enumerate(row_matches): + is_last = (i == len(row_matches) - 1) + + # extract hour from sentence if not in column + hour = raw_hour + if not hour: + h_m = re.search(r'\b([012]\d[0-5]\d)\s*h(?:r|our|s)?', sent, re.IGNORECASE) + if h_m: + hour = h_m.group(1) + else: + h_m2 = re.search(r'\b([012]\d[0-5]\d)[A-Z]?\b', sent) + if h_m2: + candidate = h_m2.group(1) + # make sure it looks like a time not a grid + if int(candidate[:2]) <= 23 and int(candidate[2:]) <= 59: + hour = candidate + + # grid + if digits: + grid, inferred = expand_grid(digits) + if len(grid) != 6: + grid = None + inferred = False + else: + grid = None + inferred = False + + # place name from sentence context + pn_m = re.search( + r'\b(Fme\s+\w+[\w\s]+?(?=\s+\d|\s+MR|\.|,|$)|' + r'Chateau\s+\w+|Fort\s+\d+|Casino|Distillery\s+\w+|' + r'Brickworks|blockhouse|moated\s+farm|' + r'Mardick|Dunkerque|Dunkirk|Brecht|Loon\s*Plage|' + r'Bourbourgville|St\.\s*Folquin|Nordamsques|Montreuil|' + r'Wommelg[ea]h?m|Antwerp|Ypres|Pasch[aeo]nd[ae]?le|' + r'Gravelines|Le\s*Clipon|Coppenaxfort|' + r'Lochtenberg|Eindhoven|Sternhoven|Ryckevorsel|' + r'St\.\s*Leonard|Bindhoven|Schilde|Schelde|' + r'cross.?roads?|road\s+junction|road\s+junc\.?|' + r'start\s+line|bridge|lock\s+gates|railway\s+st[na]|' + r'windmill|windpump|pier|beach|canal)\b', + sent, re.IGNORECASE) + place_name = pn_m.group(0).strip() if pn_m else None + + # also check the place column label + col_pn = extract_place_name(place_text) + if not place_name and col_pn: + place_name = col_pn + + subunit = extract_subunit(sent) + friendly = extract_friendly(sent) + category = categorise(sent, place_name, subunit, friendly, is_place_col=False) + + # shorten context to 2 sentences max + context = sent.strip() + if len(context) > 300: + context = context[:297] + "..." + + all_positions.append(dict( + date=cur_date, + hour=hour, + grid=grid, + grid_inferred=inferred, + place_name=place_name, + sheet_ref=last_sheet, + category=category, + subunit=subunit, + friendly_unit=friendly, + is_end_of_day=False, # will be set in post-processing + context=context + )) + + return all_positions + + +# ── EOD post-processing ─────────────────────────────────────────────────────── + +def assign_end_of_day(positions: list[dict]) -> list[dict]: + """ + For each date, set is_end_of_day=True on exactly ONE entry — the last + recorded HQ position for that date (in document order). + + Priority (highest first): + 1. Last place-column TAC_HQ with grid (context starts "Place column:") + 2. Last place-column BN_HQ with grid + 3. Last place-column BN_HQ or TAC_HQ without grid + 4. Last narrative TAC_HQ with grid + 5. Last narrative BN_HQ with grid + 6. Last UNIT_MOVEMENT with grid + 7. Last any entry with grid + """ + from collections import defaultdict + + for p in positions: + p['is_end_of_day'] = False + + date_indices: dict[str, list[int]] = defaultdict(list) + for i, p in enumerate(positions): + if p['date']: + date_indices[p['date']].append(i) + + def _is_place_col(p): + return (p.get('context') or '').startswith('Place column') + + for date, indices in date_indices.items(): + def _last(cats, require_grid=True, place_col_only=False): + matches = [ + i for i in indices + if positions[i]['category'] in cats + and (not require_grid or positions[i]['grid']) + and (not place_col_only or _is_place_col(positions[i])) + ] + return matches[-1] if matches else None + + winner = ( + _last({'TAC_HQ'}, require_grid=True, place_col_only=True) or + _last({'BN_HQ'}, require_grid=True, place_col_only=True) or + _last({'TAC_HQ', 'BN_HQ'}, require_grid=False, place_col_only=True) or + _last({'TAC_HQ'}, require_grid=True, place_col_only=False) or + _last({'BN_HQ'}, require_grid=True, place_col_only=False) or + _last({'UNIT_MOVEMENT'}, require_grid=True) or + next((i for i in reversed(indices) if positions[i]['grid']), None) + ) + + if winner is not None: + positions[winner]['is_end_of_day'] = True + + return positions + + +# ── run ─────────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + import pathlib + + print("Reading source file …") + text = pathlib.Path(INPUT_FILE).read_text(encoding="utf-8") + + print("Extracting positions …") + positions = extract_positions(text) + + # deduplicate identical (date + grid + context[:60]) entries + seen = set() + unique = [] + for p in positions: + key = (p["date"], p["grid"], p["context"][:60]) + if key not in seen: + seen.add(key) + unique.append(p) + + print("Assigning end-of-day flags …") + unique = assign_end_of_day(unique) + + pathlib.Path(OUTPUT_FILE).parent.mkdir(parents=True, exist_ok=True) + pathlib.Path(OUTPUT_FILE).write_text( + json.dumps(unique, indent=2, ensure_ascii=False), + encoding="utf-8" + ) + print(f"Done. {len(unique)} positions written to {OUTPUT_FILE}") + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/ocr_wardiaries.py b/scripts/ocr_wardiaries.py new file mode 100644 index 0000000..92a2d69 --- /dev/null +++ b/scripts/ocr_wardiaries.py @@ -0,0 +1,220 @@ +""" +ocr_wardiaries.py +----------------- +Converts war diary PDFs to Markdown using the olmOCR model hosted on DeepInfra. +Bypasses the olmOCR pipeline GPU check by calling the API directly. + +Requirements: + pip install requests pillow + +Poppler must be on PATH (pdftoppm command must work). + +Usage: + python ocr_wardiaries.py --input_dir "C:\path\to\pdfs" --output_dir "C:\path\to\output" --api_key "YOUR_KEY" +""" + +import argparse +import base64 +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import requests +from PIL import Image + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +DEEPINFRA_API_URL = "https://api.deepinfra.com/v1/openai/chat/completions" +MODEL = "allenai/olmOCR-2-7B-1025" +DPI = 150 # Higher = better quality but slower/more expensive. 150 is good for typewritten text. +MAX_RETRIES = 3 # Retries per page on API error +RETRY_DELAY = 5 # Seconds between retries + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def pdf_to_images(pdf_path: Path, output_dir: Path, dpi: int = DPI) -> list[Path]: + """Render each page of a PDF to a PNG image using pdftoppm (poppler).""" + prefix = output_dir / pdf_path.stem + cmd = [ + "pdftoppm", + "-r", str(dpi), + "-png", + str(pdf_path), + str(prefix), + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"pdftoppm failed for {pdf_path.name}:\n{result.stderr}") + + # pdftoppm names files like prefix-1.png, prefix-2.png etc (zero-padded) + images = sorted(output_dir.glob(f"{pdf_path.stem}-*.png")) + return images + + +def image_to_base64(image_path: Path) -> str: + """Convert an image file to a base64 string.""" + with open(image_path, "rb") as f: + return base64.b64encode(f.read()).decode("utf-8") + + +def ocr_page(image_path: Path, api_key: str, page_num: int) -> str: + """Send one page image to DeepInfra olmOCR and return the extracted text.""" + b64 = image_to_base64(image_path) + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + } + + payload = { + "model": MODEL, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{b64}" + }, + }, + { + "type": "text", + "text": ( + "Below is a scanned page from a Canadian Army WWII war diary or supplementary document, " + "circa 1944-1945. Extract all text exactly as it appears, preserving layout, " + "column structure, dates, grid references, unit names, and abbreviations. " + "Output clean Markdown. Do not add commentary or summaries." + ), + }, + ], + } + ], + "max_tokens": 4096, + "temperature": 0.0, + } + + for attempt in range(1, MAX_RETRIES + 1): + try: + response = requests.post(DEEPINFRA_API_URL, headers=headers, json=payload, timeout=120) + response.raise_for_status() + data = response.json() + return data["choices"][0]["message"]["content"] + except requests.exceptions.HTTPError as e: + print(f" HTTP error on page {page_num}, attempt {attempt}/{MAX_RETRIES}: {e}") + if attempt < MAX_RETRIES: + time.sleep(RETRY_DELAY) + else: + return f"[OCR FAILED - page {page_num} - HTTP error: {e}]" + except Exception as e: + print(f" Error on page {page_num}, attempt {attempt}/{MAX_RETRIES}: {e}") + if attempt < MAX_RETRIES: + time.sleep(RETRY_DELAY) + else: + return f"[OCR FAILED - page {page_num} - Error: {e}]" + + +def ocr_pdf(pdf_path: Path, output_dir: Path, api_key: str) -> Path: + """OCR a single PDF and write output to a Markdown file.""" + print(f"\n{'='*60}") + print(f"Processing: {pdf_path.name}") + print(f"{'='*60}") + + output_md = output_dir / f"{pdf_path.stem}_olmocr.md" + + # Skip if already done + if output_md.exists(): + print(f" Already processed — skipping. Delete {output_md.name} to reprocess.") + return output_md + + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) + + # Render PDF pages to images + print(f" Rendering PDF pages to images at {DPI} DPI...") + try: + images = pdf_to_images(pdf_path, tmp_path) + except RuntimeError as e: + print(f" ERROR: {e}") + return output_md + + total_pages = len(images) + print(f" Found {total_pages} pages.") + + all_text = [ + f"# {pdf_path.stem}\n", + f"*OCR'd by olmOCR ({MODEL}) via DeepInfra*\n", + f"*Source: {pdf_path.name} — {total_pages} pages*\n", + "---\n", + ] + + for i, image_path in enumerate(images, start=1): + print(f" Page {i}/{total_pages}...", end=" ", flush=True) + page_text = ocr_page(image_path, api_key, i) + all_text.append(f"\n---\n## Page {i}\n\n{page_text}\n") + print("done") + + # Small delay to avoid hammering the API + if i < total_pages: + time.sleep(0.5) + + # Write output + output_dir.mkdir(parents=True, exist_ok=True) + with open(output_md, "w", encoding="utf-8") as f: + f.write("\n".join(all_text)) + + print(f" Saved: {output_md}") + return output_md + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description="OCR war diary PDFs using olmOCR via DeepInfra.") + parser.add_argument("--input_dir", required=True, help="Folder containing PDF files to process.") + parser.add_argument("--output_dir", required=True, help="Folder to write Markdown output files.") + parser.add_argument("--api_key", required=True, help="Your DeepInfra API key.") + parser.add_argument("--single", default=None, help="Process a single PDF file instead of a folder.") + args = parser.parse_args() + + input_dir = Path(args.input_dir) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + if args.single: + pdfs = [Path(args.single)] + else: + pdfs = sorted(input_dir.glob("*.pdf")) + + if not pdfs: + print(f"No PDF files found in {input_dir}") + sys.exit(1) + + print(f"Found {len(pdfs)} PDF(s) to process.") + print(f"Output directory: {output_dir}") + + results = [] + for pdf_path in pdfs: + output_md = ocr_pdf(pdf_path, output_dir, args.api_key) + results.append(output_md) + + print(f"\n{'='*60}") + print(f"Complete. {len(results)} file(s) processed.") + print(f"Output files:") + for r in results: + print(f" {r}") + + +if __name__ == "__main__": + main()