GoogleDocumentOCR/db.py

@@ -0,0 +1,108 @@
+import os
+import psycopg2
+from psycopg2.extras import execute_values
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def get_conn():
+    return psycopg2.connect(
+        host=os.getenv("DB_HOST"),
+        port=os.getenv("DB_PORT"),
+        dbname=os.getenv("DB_NAME"),
+        user=os.getenv("DB_USER"),
+        password=os.getenv("DB_PASSWORD"),
+    )
+
+def init_db():
+    conn = get_conn()
+    cur = conn.cursor()
+    cur.execute("CREATE EXTENSION IF NOT EXISTS vector;")
+    cur.execute("""
+        CREATE TABLE IF NOT EXISTS document_chunks (
+            id          SERIAL PRIMARY KEY,
+            filename    TEXT NOT NULL,
+            page_num    INTEGER,
+            chunk_index INTEGER,
+            text        TEXT NOT NULL,
+            embedding   vector(768),
+            nationality TEXT DEFAULT 'unknown',
+            corpus      TEXT DEFAULT 'unknown',
+            created_at  TIMESTAMPTZ DEFAULT NOW()
+        );
+    """)
+    # Migrate existing tables that predate nationality/corpus columns
+    for col, default in [("nationality", "unknown"), ("corpus", "unknown")]:
+        cur.execute(f"""
+            ALTER TABLE document_chunks
+            ADD COLUMN IF NOT EXISTS {col} TEXT DEFAULT '{default}';
+        """)
+    cur.execute("""
+        CREATE INDEX IF NOT EXISTS document_chunks_embedding_idx
+        ON document_chunks
+        USING ivfflat (embedding vector_cosine_ops)
+        WITH (lists = 100);
+    """)
+    cur.execute("""
+        CREATE INDEX IF NOT EXISTS document_chunks_nationality_idx
+        ON document_chunks (nationality);
+    """)
+    cur.execute("""
+        CREATE INDEX IF NOT EXISTS document_chunks_corpus_idx
+        ON document_chunks (corpus);
+    """)
+    conn.commit()
+    cur.close()
+    conn.close()
+    print("DB initialized.")
+
+def upsert_chunks(rows: list[dict]):
+    conn = get_conn()
+    cur = conn.cursor()
+    execute_values(
+        cur,
+        """
+        INSERT INTO document_chunks (filename, page_num, chunk_index, text, embedding, nationality, corpus)
+        VALUES %s
+        """,
+        [
+            (
+                r["filename"], r["page_num"], r["chunk_index"], r["text"], r["embedding"],
+                r.get("nationality", "unknown"), r.get("corpus", "unknown"),
+            )
+            for r in rows
+        ],
+        template="(%s, %s, %s, %s, %s::vector, %s, %s)"
+    )
+    conn.commit()
+    cur.close()
+    conn.close()
+
+def semantic_search(query_embedding: list[float], top_k: int = 5,
+                    nationality: str = None, corpus: str = None):
+    conn = get_conn()
+    cur = conn.cursor()
+    filters = []
+    params  = []
+    if nationality:
+        filters.append("nationality = %s")
+        params.append(nationality)
+    if corpus:
+        filters.append("corpus = %s")
+        params.append(corpus)
+    where = ("WHERE " + " AND ".join(filters)) if filters else ""
+    cur.execute(
+        f"""
+        SELECT filename, page_num, chunk_index, text, nationality, corpus,
+               1 - (embedding <=> %s::vector) AS similarity
+        FROM document_chunks
+        {where}
+        ORDER BY embedding <=> %s::vector
+        LIMIT %s;
+        """,
+        [query_embedding] + params + [query_embedding, top_k]
+    )
+    rows = cur.fetchall()
+    cur.close()
+    conn.close()
+    return rows

GoogleDocumentOCR/embed_and_store.py

@@ -0,0 +1,73 @@
+import os
+from openai import OpenAI
+from db import upsert_chunks
+from dotenv import load_dotenv
+
+load_dotenv()
+
+client = OpenAI(
+    api_key=os.getenv("DEEPINFRA_API_KEY"),
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+
+CHUNK_SIZE    = 500
+CHUNK_OVERLAP = 50
+EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5"
+EMBEDDING_DIMS  = 768  # bge-base produces 768-dim vectors
+
+
+def chunk_text(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[str]:
+    words = text.split()
+    chunks = []
+    start = 0
+    while start < len(words):
+        end = start + chunk_size
+        chunks.append(" ".join(words[start:end]))
+        start += chunk_size - overlap
+    return [c for c in chunks if c.strip()]
+
+
+def embed_texts(texts: list[str]) -> list[list[float]]:
+    all_embeddings = []
+    batch_size = 100
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        response = client.embeddings.create(
+            input=batch,
+            model=EMBEDDING_MODEL,
+        )
+        all_embeddings.extend([item.embedding for item in response.data])
+    return all_embeddings
+
+
+def process_pages(filename: str, pages: list[dict],
+                  nationality: str = "unknown", corpus: str = "unknown"):
+    rows = []
+    all_chunks = []
+
+    for page in pages:
+        chunks = chunk_text(page["text"])
+        for idx, chunk in enumerate(chunks):
+            all_chunks.append({
+                "filename":    filename,
+                "page_num":    page["page_num"],
+                "chunk_index": idx,
+                "text":        chunk,
+                "nationality": nationality,
+                "corpus":      corpus,
+            })
+
+    if not all_chunks:
+        print(f"  No text extracted from {filename}, skipping.")
+        return
+
+    print(f"  Embedding {len(all_chunks)} chunks from {filename}...")
+    texts = [c["text"] for c in all_chunks]
+    embeddings = embed_texts(texts)
+
+    for chunk, embedding in zip(all_chunks, embeddings):
+        chunk["embedding"] = embedding
+        rows.append(chunk)
+
+    upsert_chunks(rows)
+    print(f"  Stored {len(rows)} chunks for {filename}.")

GoogleDocumentOCR/main.py

@@ -0,0 +1,149 @@
+import os
+import shutil
+from tqdm import tqdm
+from dotenv import load_dotenv
+
+from db import init_db
+from ocr import ocr_file
+from embed_and_store import process_pages
+
+load_dotenv()
+
+DOCUMENTS_DIR = "documents"
+PROCESSED_DIR = "processed"
+
+SUPPORTED_EXTENSIONS = {".pdf", ".png", ".jpg", ".jpeg", ".tiff", ".tif", ".gif", ".bmp", ".webp"}
+
+# ── Tag your documents here ──────────────────────────────────────────────────
+# Maps a filename substring → (nationality, corpus)
+# First matching rule wins. Falls back to ("unknown", "unknown").
+DOCUMENT_TAGS = {
+    # German records
+    "german":    ("german",   "german_records"),
+    "deutsch":   ("german",   "german_records"),
+    "Wehrmacht": ("german",   "german_records"),
+    # Canadian war diaries
+    "Calgary":   ("canadian", "war_diary_narrative"),
+    "Blackwatch": ("canadian","war_diary_narrative"),
+    "5CIB":      ("canadian", "war_diary_narrative"),
+}
+
+def tag_document(filename: str) -> tuple[str, str]:
+    for key, (nationality, corpus) in DOCUMENT_TAGS.items():
+        if key.lower() in filename.lower():
+            return nationality, corpus
+    return "unknown", "unknown"
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def main():
+    os.makedirs(DOCUMENTS_DIR, exist_ok=True)
+    os.makedirs(PROCESSED_DIR, exist_ok=True)
+    init_db()
+
+    files = [
+        f for f in os.listdir(DOCUMENTS_DIR)
+        if os.path.splitext(f)[1].lower() in SUPPORTED_EXTENSIONS
+    ]
+
+    if not files:
+        print(f"No supported files found in '{DOCUMENTS_DIR}'. Drop PDFs or images there and re-run.")
+        return
+
+    print(f"Found {len(files)} file(s) to process.\n")
+
+    for filename in tqdm(files, desc="Processing documents"):
+        file_path = os.path.join(DOCUMENTS_DIR, filename)
+        nationality, corpus = tag_document(filename)
+        print(f"\n--- {filename} --- [{nationality} / {corpus}]")
+
+        try:
+            pages = ocr_file(file_path)
+            print(f"  OCR complete: {len(pages)} page(s)")
+
+            process_pages(filename, pages, nationality=nationality, corpus=corpus)
+
+            shutil.move(file_path, os.path.join(PROCESSED_DIR, filename))
+            print(f"  Moved to {PROCESSED_DIR}/")
+
+        except Exception as e:
+            print(f"  ERROR processing {filename}: {e}")
+
+    print("\nDone. All files processed.")
+
+
+def search(query: str, top_k: int = 5, nationality: str = None, corpus: str = None):
+    from embed_and_store import embed_texts
+    from db import semantic_search
+
+    filters = []
+    if nationality: filters.append(f"nationality={nationality}")
+    if corpus:      filters.append(f"corpus={corpus}")
+    filter_str = f" [{', '.join(filters)}]" if filters else ""
+    print(f"\nSearching for: '{query}'{filter_str}\n")
+
+    embedding = embed_texts([query])[0]
+    results = semantic_search(embedding, top_k=top_k, nationality=nationality, corpus=corpus)
+
+    for rank, (filename, page_num, chunk_index, text, nat, corp, similarity) in enumerate(results, 1):
+        print(f"[{rank}] {filename} — page {page_num}, chunk {chunk_index} [{nat} / {corp}] (similarity: {similarity:.3f})")
+        print(f"    {text[:200]}...")
+        print()
+def ask(question: str, top_k: int = 5, nationality: str = None, corpus: str = None):
+    from embed_and_store import embed_texts
+    from db import semantic_search
+    import os
+    from openai import OpenAI
+
+    filters = []
+    if nationality: filters.append(f"nationality={nationality}")
+    if corpus:      filters.append(f"corpus={corpus}")
+    filter_str = f" [{', '.join(filters)}]" if filters else ""
+    print(f"\nAsking: '{question}'{filter_str}\n")
+
+    # Step 1 - find relevant chunks
+    embedding = embed_texts([question])[0]
+    results = semantic_search(embedding, top_k=top_k, nationality=nationality, corpus=corpus)
+
+    if not results:
+        print("No relevant chunks found.")
+        return
+
+    # Step 2 - build context from chunks
+    context = ""
+    for filename, page_num, chunk_index, text, nat, corp, similarity in results:
+        context += f"[Source: {filename}, page {page_num}]\n{text}\n\n"
+
+    # Step 3 - send to LLM
+    client = OpenAI(
+        api_key=os.getenv("DEEPINFRA_API_KEY"),
+        base_url="https://api.deepinfra.com/v1/openai",
+    )
+
+    prompt = f"""You are a WWII military historian. Answer the question below using only the provided source documents.
+Cite the source file and page number for each claim you make.
+
+QUESTION: {question}
+
+SOURCE DOCUMENTS:
+{context}
+
+Answer:"""
+
+    response = client.chat.completions.create(
+        model="meta-llama/Meta-Llama-3.1-8B-Instruct",
+        messages=[{"role": "user", "content": prompt}],
+        max_tokens=1000,
+    )
+
+    print("\nQUESTION:", question)
+    print("\nANSWER:")
+    print(response.choices[0].message.content)
+    print("\nSOURCES USED:")
+    for filename, page_num, chunk_index, text, nat, corp, similarity in results:
+        print(f"  - {filename} page {page_num} [{nat} / {corp}] (similarity: {similarity:.3f})")
+
+if __name__ == "__main__":
+    # main()
+    # search("is this document in german?", nationality="german")
+    ask("What was the Calgary Highlanders doing in October 1944?", nationality="canadian", corpus="war_diary_narrative")

GoogleDocumentOCR/ocr.py

@@ -0,0 +1,94 @@
+import os
+import io
+from google.api_core.client_options import ClientOptions
+from google.cloud import documentai
+from dotenv import load_dotenv
+
+load_dotenv()
+
+PROJECT_ID    = os.getenv("GOOGLE_PROJECT_ID")
+LOCATION      = os.getenv("GOOGLE_LOCATION")
+PROCESSOR_ID  = os.getenv("GOOGLE_PROCESSOR_ID")
+
+PAGE_LIMIT = 14  # stay under the 30 page limit
+
+
+def get_client():
+    opts = ClientOptions(api_endpoint=f"{LOCATION}-documentai.googleapis.com")
+    return documentai.DocumentProcessorServiceClient(client_options=opts)
+
+
+def split_pdf(content: bytes, page_limit: int = PAGE_LIMIT) -> list[bytes]:
+    """Split PDF bytes into chunks of page_limit pages."""
+    import pypdf
+    reader = pypdf.PdfReader(io.BytesIO(content))
+    total_pages = len(reader.pages)
+    chunks = []
+    for start in range(0, total_pages, page_limit):
+        writer = pypdf.PdfWriter()
+        for i in range(start, min(start + page_limit, total_pages)):
+            writer.add_page(reader.pages[i])
+        buf = io.BytesIO()
+        writer.write(buf)
+        chunks.append(buf.getvalue())
+    return chunks
+
+
+def ocr_chunk(client, name: str, content: bytes, mime_type: str, page_offset: int) -> list[dict]:
+    """OCR a single chunk and return pages with corrected page numbers."""
+    raw_doc = documentai.RawDocument(content=content, mime_type=mime_type)
+    request = documentai.ProcessRequest(name=name, raw_document=raw_doc)
+    result = client.process_document(request=request)
+    doc = result.document
+
+    pages = []
+    for i, page in enumerate(doc.pages):
+        page_text_parts = []
+        for block in page.blocks:
+            seg = block.layout.text_anchor.text_segments
+            for s in seg:
+                start = int(s.start_index) if s.start_index else 0
+                end   = int(s.end_index)
+                page_text_parts.append(doc.text[start:end])
+        page_text = "\n".join(page_text_parts).strip()
+        if page_text:
+            pages.append({"page_num": page_offset + i + 1, "text": page_text})
+    return pages
+
+
+def ocr_file(file_path: str) -> list[dict]:
+    client = get_client()
+    name = client.processor_path(PROJECT_ID, LOCATION, PROCESSOR_ID)
+
+    ext = os.path.splitext(file_path)[1].lower()
+    mime_map = {
+        ".pdf":  "application/pdf",
+        ".png":  "image/png",
+        ".jpg":  "image/jpeg",
+        ".jpeg": "image/jpeg",
+        ".tiff": "image/tiff",
+        ".tif":  "image/tiff",
+        ".gif":  "image/gif",
+        ".bmp":  "image/bmp",
+        ".webp": "image/webp",
+    }
+    mime_type = mime_map.get(ext, "application/pdf")
+
+    with open(file_path, "rb") as f:
+        content = f.read()
+
+    # Only PDFs need splitting
+    if mime_type == "application/pdf":
+        chunks = split_pdf(content)
+        print(f"  Split into {len(chunks)} chunk(s) of up to {PAGE_LIMIT} pages")
+    else:
+        chunks = [content]
+
+    all_pages = []
+    for chunk_idx, chunk in enumerate(chunks):
+        page_offset = chunk_idx * PAGE_LIMIT
+        print(f"  OCR chunk {chunk_idx + 1}/{len(chunks)}...")
+        pages = ocr_chunk(client, name, chunk, mime_type, page_offset)
+        all_pages.extend(pages)
+
+    return all_pages

GoogleDocumentOCR/web_app.py

@@ -0,0 +1,473 @@
+"""
+web_app.py  —  Terminal-style search UI
+Run:  python GoogleDocumentOCR/web_app.py
+Then open:  http://localhost:5000
+"""
+
+import os, sys
+from pathlib import Path
+from flask import Flask, request, jsonify, render_template_string
+from dotenv import load_dotenv
+
+sys.path.insert(0, str(Path(__file__).parent))
+load_dotenv()
+
+app = Flask(__name__)
+
+# ── HTML template ──────────────────────────────────────────────────────────────
+HTML = """<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>WAR DIARY SEARCH // HISTORICAL RECORDS SYSTEM</title>
+<style>
+  :root {
+    --green:   #00ff41;
+    --dimgreen: #00a82b;
+    --amber:   #ffb000;
+    --red:     #ff4444;
+    --bg:      #0a0a0a;
+    --bg2:     #0f0f0f;
+    --border:  #1a3a1a;
+  }
+
+  * { box-sizing: border-box; margin: 0; padding: 0; }
+
+  body {
+    background: var(--bg);
+    color: var(--green);
+    font-family: 'Courier New', Courier, monospace;
+    font-size: 14px;
+    height: 100vh;
+    display: flex;
+    flex-direction: column;
+    overflow: hidden;
+  }
+
+  /* ── Header ── */
+  #header {
+    border-bottom: 1px solid var(--border);
+    padding: 10px 16px;
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    flex-shrink: 0;
+    background: var(--bg2);
+  }
+  #header .title {
+    color: var(--amber);
+    font-size: 13px;
+    letter-spacing: 2px;
+    text-transform: uppercase;
+  }
+  #header .status {
+    font-size: 11px;
+    color: var(--dimgreen);
+  }
+  #header .status span { color: var(--green); }
+
+  /* ── Filters bar ── */
+  #filters {
+    padding: 6px 16px;
+    border-bottom: 1px solid var(--border);
+    display: flex;
+    gap: 20px;
+    align-items: center;
+    font-size: 12px;
+    color: var(--dimgreen);
+    flex-shrink: 0;
+    background: var(--bg2);
+  }
+  #filters label { color: var(--dimgreen); }
+  #filters select {
+    background: #000;
+    color: var(--green);
+    border: 1px solid var(--border);
+    font-family: inherit;
+    font-size: 12px;
+    padding: 2px 6px;
+    outline: none;
+    cursor: pointer;
+  }
+  #filters select:focus { border-color: var(--green); }
+
+  /* ── Chat window ── */
+  #chat {
+    flex: 1;
+    overflow-y: auto;
+    padding: 16px;
+    display: flex;
+    flex-direction: column;
+    gap: 18px;
+    scroll-behavior: smooth;
+  }
+  #chat::-webkit-scrollbar { width: 6px; }
+  #chat::-webkit-scrollbar-track { background: #000; }
+  #chat::-webkit-scrollbar-thumb { background: var(--border); }
+
+  /* ── Boot message ── */
+  .boot {
+    color: var(--dimgreen);
+    font-size: 12px;
+    line-height: 1.8;
+    border-bottom: 1px solid var(--border);
+    padding-bottom: 14px;
+  }
+  .boot .hi { color: var(--amber); }
+
+  /* ── Exchange (Q+A pair) ── */
+  .exchange {}
+
+  .query-line {
+    color: var(--amber);
+    margin-bottom: 8px;
+    word-break: break-word;
+  }
+  .query-line::before { content: '> '; color: var(--dimgreen); }
+
+  .answer-block {
+    color: var(--green);
+    line-height: 1.7;
+    white-space: pre-wrap;
+    word-break: break-word;
+    padding-left: 14px;
+    border-left: 2px solid var(--border);
+  }
+
+  .sources-block {
+    margin-top: 10px;
+    padding-left: 14px;
+    border-left: 2px solid var(--border);
+  }
+  .sources-label {
+    color: var(--dimgreen);
+    font-size: 11px;
+    letter-spacing: 1px;
+    margin-bottom: 4px;
+  }
+  .source-row {
+    font-size: 11px;
+    color: #336633;
+    line-height: 1.6;
+  }
+  .source-row .sim { color: var(--dimgreen); }
+
+  /* ── Thinking indicator ── */
+  .thinking {
+    color: var(--dimgreen);
+    font-size: 12px;
+  }
+  .dot-anim::after {
+    content: '';
+    animation: dots 1.2s steps(4, end) infinite;
+  }
+  @keyframes dots {
+    0%   { content: ''; }
+    25%  { content: '.'; }
+    50%  { content: '..'; }
+    75%  { content: '...'; }
+    100% { content: ''; }
+  }
+
+  .error-line { color: var(--red); padding-left: 14px; }
+
+  /* ── Input bar ── */
+  #inputbar {
+    border-top: 1px solid var(--border);
+    padding: 12px 16px;
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    flex-shrink: 0;
+    background: var(--bg2);
+  }
+  #prompt-symbol {
+    color: var(--amber);
+    font-size: 15px;
+    flex-shrink: 0;
+  }
+  #question {
+    flex: 1;
+    background: transparent;
+    border: none;
+    outline: none;
+    color: var(--green);
+    font-family: inherit;
+    font-size: 14px;
+    caret-color: var(--green);
+  }
+  #question::placeholder { color: #1a3a1a; }
+
+  #send-btn {
+    background: transparent;
+    border: 1px solid var(--border);
+    color: var(--dimgreen);
+    font-family: inherit;
+    font-size: 12px;
+    padding: 4px 10px;
+    cursor: pointer;
+    letter-spacing: 1px;
+    transition: color 0.15s, border-color 0.15s;
+  }
+  #send-btn:hover:not(:disabled) {
+    color: var(--green);
+    border-color: var(--green);
+  }
+  #send-btn:disabled { opacity: 0.3; cursor: not-allowed; }
+
+  /* ── Scanline overlay ── */
+  body::after {
+    content: '';
+    position: fixed;
+    inset: 0;
+    background: repeating-linear-gradient(
+      0deg,
+      transparent,
+      transparent 2px,
+      rgba(0,0,0,0.07) 2px,
+      rgba(0,0,0,0.07) 4px
+    );
+    pointer-events: none;
+    z-index: 999;
+  }
+</style>
+</head>
+<body>
+
+<div id="header">
+  <div class="title">&#9632; WAR DIARY SEARCH // HISTORICAL RECORDS SYSTEM</div>
+  <div class="status">DB STATUS: <span>ONLINE</span> &nbsp;|&nbsp; CORPUS: <span>CANADIAN WWII</span></div>
+</div>
+
+<div id="filters">
+  <span>FILTERS:</span>
+  <label>NATIONALITY
+    <select id="nationality">
+      <option value="">ALL</option>
+      <option value="canadian" selected>CANADIAN</option>
+      <option value="german">GERMAN</option>
+      <option value="unknown">UNKNOWN</option>
+    </select>
+  </label>
+  <label>CORPUS
+    <select id="corpus">
+      <option value="">ALL</option>
+      <option value="war_diary_narrative" selected>WAR DIARY NARRATIVE</option>
+      <option value="war_diary_appendix">WAR DIARY APPENDIX</option>
+      <option value="german_records">GERMAN RECORDS</option>
+    </select>
+  </label>
+  <label>RESULTS
+    <select id="top_k">
+      <option value="3">3</option>
+      <option value="5" selected>5</option>
+      <option value="10">10</option>
+    </select>
+  </label>
+</div>
+
+<div id="chat">
+  <div class="boot">
+    <div class="hi">HISTORICAL RECORDS RETRIEVAL SYSTEM v1.0</div>
+    <div>Corpus: Calgary Highlanders · 5 CIB · RHC Black Watch · Sep–Oct 1944</div>
+    <div>Embedding model: BAAI/bge-base-en-v1.5 &nbsp;|&nbsp; LLM: Meta-Llama-3.1-8B</div>
+    <div>&nbsp;</div>
+    <div>Type a question and press ENTER or [SEND].</div>
+    <div>Example: <span style="color:var(--green)">What was the Calgary Highlanders doing on October 23, 1944?</span></div>
+  </div>
+</div>
+
+<div id="inputbar">
+  <span id="prompt-symbol">&gt;</span>
+  <input id="question" type="text"
+    placeholder="Ask about the war diaries..."
+    autocomplete="off" spellcheck="false" autofocus />
+  <button id="send-btn">SEND</button>
+</div>
+
+<script>
+const chatEl    = document.getElementById('chat');
+const inputEl   = document.getElementById('question');
+const sendBtn   = document.getElementById('send-btn');
+
+function escHtml(s) {
+  return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+}
+
+function scrollBottom() {
+  chatEl.scrollTop = chatEl.scrollHeight;
+}
+
+function addExchange(question, answer, sources, isError) {
+  const div = document.createElement('div');
+  div.className = 'exchange';
+
+  const qLine = document.createElement('div');
+  qLine.className = 'query-line';
+  qLine.textContent = question;
+  div.appendChild(qLine);
+
+  if (isError) {
+    const err = document.createElement('div');
+    err.className = 'error-line';
+    err.textContent = 'ERROR: ' + answer;
+    div.appendChild(err);
+  } else {
+    const ans = document.createElement('div');
+    ans.className = 'answer-block';
+    ans.textContent = answer;
+    div.appendChild(ans);
+
+    if (sources && sources.length) {
+      const sb = document.createElement('div');
+      sb.className = 'sources-block';
+      const lbl = document.createElement('div');
+      lbl.className = 'sources-label';
+      lbl.textContent = '── SOURCES (' + sources.length + ') ──────────────';
+      sb.appendChild(lbl);
+      sources.forEach((s, i) => {
+        const row = document.createElement('div');
+        row.className = 'source-row';
+        const simPct = (s.similarity * 100).toFixed(1);
+        row.innerHTML =
+          '[' + (i+1) + '] ' + escHtml(s.filename) +
+          ' &nbsp;pg.' + s.page_num +
+          ' &nbsp;<span class="sim">sim:' + simPct + '%</span>' +
+          (s.corpus ? ' &nbsp;[' + escHtml(s.corpus) + ']' : '');
+        sb.appendChild(row);
+      });
+      div.appendChild(sb);
+    }
+  }
+
+  chatEl.appendChild(div);
+  scrollBottom();
+}
+
+async function ask() {
+  const question = inputEl.value.trim();
+  if (!question) return;
+
+  const nationality = document.getElementById('nationality').value;
+  const corpus      = document.getElementById('corpus').value;
+  const top_k       = parseInt(document.getElementById('top_k').value);
+
+  inputEl.value = '';
+  sendBtn.disabled = true;
+  inputEl.disabled = true;
+
+  // Thinking indicator
+  const thinking = document.createElement('div');
+  thinking.className = 'thinking dot-anim';
+  thinking.textContent = 'SEARCHING';
+  chatEl.appendChild(thinking);
+  scrollBottom();
+
+  try {
+    const res = await fetch('/ask', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ question, nationality, corpus, top_k }),
+    });
+    const data = await res.json();
+    chatEl.removeChild(thinking);
+
+    if (data.error) {
+      addExchange(question, data.error, null, true);
+    } else {
+      addExchange(question, data.answer, data.sources, false);
+    }
+  } catch (e) {
+    chatEl.removeChild(thinking);
+    addExchange(question, e.message, null, true);
+  }
+
+  sendBtn.disabled = false;
+  inputEl.disabled = false;
+  inputEl.focus();
+}
+
+sendBtn.addEventListener('click', ask);
+inputEl.addEventListener('keydown', e => { if (e.key === 'Enter') ask(); });
+</script>
+</body>
+</html>
+"""
+
+# ── API endpoint ───────────────────────────────────────────────────────────────
+@app.route("/")
+def index():
+    return render_template_string(HTML)
+
+
+@app.route("/ask", methods=["POST"])
+def ask():
+    from embed_and_store import embed_texts
+    from db import semantic_search
+    from openai import OpenAI
+
+    data       = request.get_json(force=True)
+    question   = (data.get("question") or "").strip()
+    nationality = data.get("nationality") or None
+    corpus      = data.get("corpus") or None
+    top_k       = int(data.get("top_k", 5))
+
+    if not question:
+        return jsonify({"error": "No question provided."}), 400
+
+    try:
+        # 1. Embed + search
+        embedding = embed_texts([question])[0]
+        rows = semantic_search(embedding, top_k=top_k,
+                               nationality=nationality, corpus=corpus)
+
+        if not rows:
+            return jsonify({
+                "answer": "No relevant records found for that query with the current filters.",
+                "sources": []
+            })
+
+        # 2. Build context
+        context = ""
+        sources = []
+        for filename, page_num, chunk_index, text, nat, corp, similarity in rows:
+            context += f"[Source: {filename}, page {page_num}]\n{text}\n\n"
+            sources.append({
+                "filename":   filename,
+                "page_num":   page_num,
+                "similarity": round(similarity, 4),
+                "corpus":     corp,
+            })
+
+        # 3. LLM answer
+        client = OpenAI(
+            api_key=os.getenv("DEEPINFRA_API_KEY"),
+            base_url="https://api.deepinfra.com/v1/openai",
+        )
+
+        prompt = (
+            "You are a WWII military historian. Answer the question using only "
+            "the provided source documents. Cite source file and page number "
+            "for each claim.\n\n"
+            f"QUESTION: {question}\n\n"
+            f"SOURCE DOCUMENTS:\n{context}\n\nAnswer:"
+        )
+
+        response = client.chat.completions.create(
+            model="meta-llama/Meta-Llama-3.1-8B-Instruct",
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=1200,
+        )
+
+        answer = response.choices[0].message.content.strip()
+        return jsonify({"answer": answer, "sources": sources})
+
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
+if __name__ == "__main__":
+    print("\n  WAR DIARY SEARCH  —  http://localhost:5000\n")
+    app.run(debug=False, port=5000)
+

check_db.py

@@ -0,0 +1,24 @@
+import sys
+sys.path.insert(0, 'GoogleDocumentOCR')
+from db import get_conn
+
+conn = get_conn()
+cur  = conn.cursor()
+
+cur.execute("""
+    SELECT nationality, corpus, chunk_type, COUNT(*)
+    FROM document_chunks
+    GROUP BY nationality, corpus, chunk_type
+    ORDER BY nationality, corpus, chunk_type
+""")
+print(f"{'Nationality':<12} {'Corpus':<26} {'Type':<14} {'Count':>6}")
+print("-" * 64)
+for nat, corpus, ctype, count in cur.fetchall():
+    print(f"{nat:<12} {corpus:<26} {ctype:<14} {count:>6}")
+
+cur.execute("SELECT COUNT(*) FROM document_chunks")
+print(f"\nTOTAL: {cur.fetchone()[0]} chunks")
+
+cur.close()
+conn.close()
+

p44_product-specification_war-diary-pipeline.md

@@ -0,0 +1,620 @@
+### Import War Diaries ###
+
+War diaries are uploaded into the system via the Frame 44 UI or Frame 44 plugin. Before uploading, the user is responsible for ensuring pages are in the correct order and that the diary has been separated by month. A warning is displayed at upload reminding the user of these requirements.
+
+**Inputs:**
+- PDF or JPEG of a war diary, separated by month
+
+**Process:**
+- User uploads the document via the Frame 44 UI or Frame 44 plugin
+- JPEG uploads are assembled into a single PDF; user confirms page order is correct before proceeding
+- User assigns the document to an existing unit or creates a new unit record
+- User records provenance metadata: archive source, unit ID, date range, and digitisation quality
+- User is presented with a thumbnail gallery of all pages and tags the narrative page range (e.g. pages 7–35)
+- Document and metadata are stored server-side
+
+**Outputs:**
+- PDF stored server-side with status set to "pending OCR"
+- Provenance metadata record linked to the document
+- Narrative page range tag stored as metadata, to be used downstream to split and classify document content
+
+**Desired Result:**
+- Every document entering the pipeline has a clean, confirmed page order, known provenance, and a human-verified narrative boundary before any automated processing begins
+
+### OCR War Diaries ###
+
+Once a document is imported and the narrative page range is confirmed, the system automatically processes the full document through an OCR pipeline. The OCR pipeline has one job: extract accurate text from every page.
+
+Human verification of OCR output is a critical quality gate for narrative pages. Raw OCR, however accurate, will contain errors that only a human reviewer with historical context can reliably identify and correct — misspelled place names, misread rank abbreviations, ambiguous handwriting. The OCR editor (described in the next step) is where this verification happens. Verified OCR is the foundation everything downstream depends on — RAG quality, position extraction accuracy, and report generation all degrade if the underlying text is wrong.
+
+Non-narrative pages follow a separate track. These pages — appendices, patrol reports, message logs, and similar documents — are useful as supporting context even without human verification and proceed directly to classification and RAG chunking without a human verification gate.
+
+**Inputs:**
+- PDF with confirmed page order
+- Narrative page range metadata
+
+**Outputs:**
+- Raw OCR text per page stored in the database
+- Narrative pages set to status "pending review" and queued for the OCR editor
+- Non-narrative pages set to status "pending classification" and queued for document classification and RAG chunking
+
+**Desired Result:**
+- Every page in the document has a raw OCR text record in the database
+- Narrative and non-narrative pages immediately split into their respective tracks and proceed in parallel
+- Narrative pages do not enter downstream processing until human verified
+- Non-narrative pages begin contributing to the RAG pipeline without delay
+
+### Document Classification ###
+
+Non-narrative pages — everything outside the narrative page range tagged at import — are automatically classified into document types after OCR. Narrative pages are already classified at import and do not pass through this step.
+
+Document type determines how a page is processed downstream. Patrol reports, message logs, and sitreps are highly relevant for mapping and semantic search. Nominal rolls and casualty returns are primary sources for personnel extraction. Appendices and traces provide supporting spatial context. Each document type follows a processing path appropriate to its content and relevance.
+
+Classification confidence is scored automatically. Low-confidence classifications are flagged for human review. Human-verified classifications feed back into the system to improve automated classification accuracy over time.
+
+**Inputs:**
+- Non-narrative pages as defined by the narrative page range tag from import
+- Page images for visual classification where OCR text alone is ambiguous
+
+**Outputs:**
+- Document type tag per page (message log, intel report, newsletter, Part 1 Orders, nominal roll, appendix, sitrep, patrol report, trace, etc.)
+- Confidence score per classification
+- Low-confidence pages flagged for human review
+- Verified classifications fed back into the system to improve future accuracy
+
+**Desired Result:**
+- Every non-narrative page has a document type tag before entering the RAG pipeline
+- Document type metadata travels with each page through all downstream processing, enabling filtered search and type-appropriate chunking
+- Classification accuracy improves continuously as human-verified examples accumulate
+
+### OCR Editor ###
+
+Verified narrative OCR is the quality gate that all downstream processing depends on. The OCR editor is a human-facing tool that allows reviewers to compare the original page image with the raw OCR text and make corrections before the page is released to downstream processing.
+
+The editor presents a work queue of narrative pages with status tracking — pending, in progress, and verified. Reviewers can see the overall progress of each war diary and pick up work from where they or another reviewer left off. Changes are tracked per user for basic version control.
+
+A page is not released to downstream processing until it is marked verified by a reviewer.
+
+**Inputs:**
+- Raw OCR text of narrative pages with status "pending review"
+- Original page images for side-by-side comparison
+
+**Outputs:**
+- Verified OCR text per page stored in the database
+- Narrative pages set to status "verified" and queued for entity extraction and RAG chunking
+- Edit history per page tracking which user made which changes
+
+**Desired Result:**
+- Every narrative page has been reviewed and corrected by a human before entering downstream processing
+- Reviewers have a clear view of what needs to be done and can collaborate without duplicating effort
+- Verified OCR text is the single source of truth for all downstream steps including entity extraction, RAG chunking, and report generation
+
+### Entity Extraction — Personnel ###
+
+After OCR is verified, the system extracts personnel names and ranks mentioned in the narrative text and stores them as structured records linked to the source diary entry.
+
+**Inputs:**
+- Verified OCR text of narrative pages
+
+**Outputs:**
+- Structured record per mention containing: name, rank (where present), context snippet, source diary entry, date, unit
+- Unresolved or ambiguous names flagged for human review
+
+**Desired Result:**
+- A descendant or researcher can search by surname and retrieve every diary mention of that person across the corpus, with date and unit context
+- Personnel records are linked to the source diary entry, enabling the full narrative passage to be surfaced alongside the structured record
+
+### Entity Extraction ###
+
+After narrative pages are verified in the OCR editor, and after non-narrative pages are classified, the system automatically extracts structured entities from both tracks. Entity extraction runs without a human gate on both tracks — non-narrative pages proceed automatically after classification, narrative pages proceed automatically after verification.
+
+Extracted entities are the structured layer that sits between raw text and downstream products. Every entity record is linked to its source page and RAG chunk, enabling any downstream product to surface the full source passage alongside the structured record.
+
+The following entity types are extracted:
+
+- **Personnel** — names and ranks mentioned in the text, linked to source diary entry, date, and unit
+- **Place names** — locations mentioned in the text, passed to the gazetteer for resolution to modern coordinates
+- **Grid references** — WWII map grid references extracted from text, passed to grid reference extraction and verification for coordinate resolution
+- **Unit mentions** — references to friendly, allied, and enemy units, linked to source diary entry and date
+- **Events** — operationally significant events such as attacks, patrols, reliefs, casualties, and objectives
+
+Low-confidence or ambiguous extractions are flagged for human review across all entity types.
+
+**Inputs:**
+- Verified OCR text of narrative pages
+- Classified non-narrative pages (nominal rolls, casualty returns, patrol reports, message logs, etc.)
+
+**Outputs:**
+- Structured entity records per extraction containing: entity type, value, context snippet, source page, source RAG chunk, date, and unit
+- Ambiguous or low-confidence extractions flagged for human review
+
+**Desired Result:**
+- Every entity mentioned across the full corpus — personnel, places, grids, units, and events — has a structured record in the database linked to its source
+- Entity records are the primary input for the gazetteer, grid reference verification, and all downstream search and report generation
+- Non-narrative pages contribute entities to the corpus immediately after classification without waiting for human verification
+
+### Gazetteer ###
+
+As place names are extracted from verified narrative text, the system builds a gazetteer — a controlled reference list of place names encountered in the diaries, resolved to modern coordinates and standardised spellings. The gazetteer grows and improves as more diaries are processed, and feeds back into improving future place name extraction across the corpus.
+
+Place name candidates are spatially constrained against known unit positions on the date of the diary entry, and corroborated across multiple diary entries from different units. Low-confidence matches are flagged for human review by volunteers.
+
+**Inputs:**
+- Extracted place name records from entity extraction
+- Known unit positions with date and confidence level
+- Modern GIS place name datasets (OpenStreetMap, GeoNames, national topographic data)
+
+**Outputs:**
+- A gazetteer record for each resolved place containing: all variant spellings encountered, resolved modern coordinate, confidence score, and source diary entries that corroborate the match
+- Unresolved place names flagged for human review
+
+**Desired Result:**
+- A living reference dataset that grows and improves as more diaries are processed
+- Variant and archaic spellings (phonetic, French, Flemish, German) are all linked to a single canonical modern place, so a user searching any variant surfaces the same results
+- Gazetteer feeds back into improving future place name extraction across the corpus
+
+### RAG Pipeline ###
+
+The RAG pipeline is the core retrieval infrastructure that powers all downstream products and search capabilities. It ingests verified narrative text and classified non-narrative pages as two separate tracks, chunks them into searchable segments, and stores them in a GIS-enabled Postgres database with vector search capabilities. All downstream products — descendant reports, semantic search, museum portals, battlefield tours — query this single system to retrieve relevant content.
+
+Both tracks arrive with metadata from earlier pipeline steps: document type, unit, date, confidence level, and entity extraction records. This metadata is stored alongside each chunk and enables precise filtered retrieval — by unit, date range, document type, personnel name, or any combination. Spatial capabilities are enriched further in the grid reference verification step described later in the pipeline.
+
+**Inputs:**
+- Verified narrative OCR text with entity extraction records
+- Classified non-narrative pages with document type tags
+
+**Outputs:**
+- Chunked and embedded text segments stored in Postgres with pgvector and PostGIS
+- Each chunk carries metadata: source document, page, unit, date, document type, confidence level, and linked entity records
+- Chunks are retrievable by semantic similarity, metadata filter, spatial query, or any combination in a single query
+
+**Desired Result:**
+- The full corpus — narrative and non-narrative, verified and classified — is queryable in a single system
+- Any question about a unit, date, location, person, or event can be answered by retrieving the most relevant chunks across the entire corpus
+- Semantic search, spatial search, and metadata filtering operate together in a single query without requiring separate systems
+- All downstream products query a single pipeline, ensuring consistency across reports, search, and mapping outputs
+
+### Grid Reference Extraction and Verification ###
+
+Grid references and place names identified during entity extraction are resolved to real world coordinates and plotted as positions. Positions are linked to the RAG chunk they came from via a foreign key relationship, enabling map and text to be navigated together — a position on the map surfaces the diary passage it came from, and a search result can surface its position on the map.
+
+**Extraction — Automated**
+
+Grid references extracted from verified narrative text and classified non-narrative pages are automatically resolved to coordinates using WWII map projections and the known 100,000m grid square the unit was operating in at the time.
+
+**Inputs:**
+- Grid reference and place name records from entity extraction
+- Known unit positions with date
+
+**Outputs:**
+- Candidate positions stored in Postgres with PostGIS, linked to source RAG chunk
+- Each position carries: coordinate, source diary entry, date, unit, position type, confidence level, and relevance category
+- Unverified positions set to status "pending verification"
+
+**Verification — Human**
+
+Positions are not always accurate at extraction and require human verification before they are considered reliable. A reviewer confirms or corrects each candidate position against reference materials. Verified positions are released for use on the web map and in downstream products.
+
+Positions carry a confidence tier:
+- **Level 1** — verified by a human
+- **Level 2** — high confidence, not yet human verified
+- **Level 3** — low confidence, requires human verification
+
+Positions are categorised by type — end of day position, patrol position, enemy position, objective, etc. — and by relevance — whether the position describes the unit's own activity or references another unit or location.
+
+**Outputs:**
+- Verified positions stored in Postgres with PostGIS, linked to source RAG chunk
+- Position type and relevance category tags
+- Confidence tier updated to Level 1 on verification
+
+**Desired Result:**
+- Every grid reference and place name in the corpus has a candidate position in the database
+- Verified positions are spatially accurate and linked back to their source diary entry
+- Positions are filterable by type, confidence, relevance, unit, and date
+- The map and the text are mutually navigable via the foreign key relationship between positions and RAG chunks 
+
+### Document Versioning and Corpus Expansion ###
+
+The corpus is never static. Better quality scans of existing documents are found over time, and new documents belonging to existing units and date ranges are discovered across different archives and digitisation projects. This section describes how both scenarios are handled.
+
+**Scenario 1 — Higher Quality Scan of an Existing Document**
+
+When a better quality scan of an existing war diary is found, the stored source image is replaced with the new scan. No downstream data is affected — OCR text, entity extraction records, RAG chunks, and positions are all untouched. The link between the text and the source image is updated to point to the new scan.
+
+Source image quality matters because users can read the original documents page by page alongside the verified narrative text, and may download the original scans for their own research. A higher quality scan improves that experience without requiring any re-processing.
+
+**Inputs:**
+- Higher quality scan of an existing document (PDF or JPEG)
+- Existing document record in the database
+
+**Outputs:**
+- Source image updated in the database
+- All downstream records remain intact and unchanged
+
+**Scenario 2 — New Documents Found**
+
+New documents belonging to an existing unit and date range — a separately filed patrol report, a message log from a different archive, a second copy of a war diary with different appendices — enter the pipeline via the standard import step. They are assigned to an existing unit and appended to the corpus. No replacement occurs and no existing records are affected.
+
+**Inputs:**
+- New document uploaded via the standard import pipeline
+- Assigned to an existing unit record
+
+**Outputs:**
+- New document processed through the full pipeline from import onwards
+- All outputs appended to existing unit records in the database
+
+**Desired Result:**
+- The corpus improves continuously as better scans and new documents are found
+- Users can always access the highest quality version of the original source document
+- Users can read original documents page by page alongside verified narrative text and download scans for their own research
+- New documents discovered across archives slot cleanly into the existing corpus without disrupting any existing data
+
+### Secondary Sources ###
+
+Secondary sources — published histories, regimental histories, and operational studies — are ingested as a distinct document class alongside primary war diary material. They provide authoritative contextual reference that enriches RAG retrieval and report generation, particularly for operational context that individual unit war diaries do not capture.
+
+Secondary sources are clearly attributed in every record so downstream products always distinguish between primary source material and secondary literature.
+
+Where a secondary source exists as clean digital text it proceeds directly to entity extraction and RAG chunking without OCR. Where only a scanned copy exists it passes through the OCR pipeline before proceeding, without a human verification gate.
+
+Secondary sources are assigned to the theatre and formations they cover rather than to a specific unit. For example, Stacey's *Victory Campaign* covers II Canadian Corps and all child formations for Northwest Europe, while the Italian theatre corpus covers I Canadian Corps exclusively. This assignment ensures RAG retrieval correctly surfaces secondary sources when querying any unit within the relevant formation and theatre.
+
+Entity extraction runs on all secondary sources, capturing place names, personnel, unit mentions, and events. Extracted place names and grid references feed into the gazetteer and grid reference verification steps, providing additional corroboration alongside primary source extractions.
+
+Unit relationship mapping — the parent/child hierarchy between formations — is handled by Project '44's existing unit infrastructure and is outside the scope of this pipeline.
+
+**Inputs:**
+- Clean digital text of published secondary sources, or scanned copies where digital text is unavailable
+- Theatre and formation assignment
+
+**Outputs:**
+- Secondary source records ingested into the RAG pipeline, attributed as secondary literature
+- Entity extraction records for personnel, place names, unit mentions, and events
+- Extracted place names and grid references passed to the gazetteer and grid reference verification steps
+
+**Desired Result:**
+- Every RAG query can draw on both primary war diary material and relevant secondary literature in a single retrieval pass
+- Secondary sources provide operational and contextual depth that individual unit war diaries cannot supply on their own
+- Attribution is always clear — users and downstream products always know whether a source is a primary diary or secondary literature
+
+### Semantic Search ###
+
+Semantic search is the capability layer that sits between the RAG pipeline and all customer-facing products. It powers every downstream use case — descendant reports, museum portals, battlefield tours, academic research, and more. It can also be exposed directly as a standalone product for researchers, institutions, and internal use.
+
+Semantic search combines natural language vector search with structured SQL filtering and spatial queries in a single pass against the Postgres database. A query is never purely semantic or purely structured — all filters operate together, returning results that are both contextually relevant and precisely constrained.
+
+Users can enter the corpus from multiple starting points:
+
+- **Natural language** — "Calgary Highlanders crossing the Leopold Canal October 1944"
+- **Unit-first** — browse or filter by unit, formation, or theatre
+- **Date-first** — filter by date range across any unit or theatre
+- **Location-first** — draw a bounding box on a map or search by place name to return everything that happened in a geographic area, regardless of unit
+- **Person-first** — search by surname to surface every mention of a specific individual across the corpus
+
+Results are returned as both narrative text and mapped outputs. A search result surfaces the relevant text chunk with full source attribution alongside mapped positions, period map overlays, referenced reports, and related records such as photographs. Users who know military history can search unit-first; users who only know a place can search spatially. Both entry points return the same richness of output.
+
+Access tiers control what each customer type can search, filter, and export. Public users, authenticated researchers, and institutional customers operate within different permission levels.
+
+**Inputs:**
+- Natural language query
+- Structured filters: unit, date range, theatre, document type, confidence level
+- Spatial input: bounding box or place name
+- Any combination of the above
+
+**Outputs:**
+- Relevant text chunks with source attribution: document, page, unit, date, document type
+- Mapped positions linked to returned chunks
+- Period map overlays applicable to the search area and date range
+- Referenced reports and related records surfaced alongside results
+
+**Desired Result:**
+- Any question about any unit, person, place, or event across the entire corpus can be answered in a single query
+- A researcher who knows a unit name and a civilian who only knows a place name both get equally rich results from different entry points
+- Text and map are always returned together, giving every result both narrative and spatial context
+- The same search infrastructure powers every downstream customer product consistently
+
+### Access Tiers ###
+
+Access to the corpus is controlled by subscription tier. Every customer product — museum portals, descendant reports, academic research, battlefield tours — operates within the corpus scope defined by the customer's tier. Higher tiers unlock broader search scope, richer outputs, and greater export capability.
+
+Report and output quality is never deliberately constrained by tier. The system always produces the best possible output given available data. Output richness is a function of input completeness — a customer who can provide unit, date range, and surname will always receive a richer result than one who can only provide a theatre and a year. Tiers control scope, not quality.
+
+**Tier 1 — Unit Scoped**
+Search and outputs are limited to a single unit and directly related formations. Designed for regimental associations, unit museums, and organisations focused on a specific unit's history.
+
+**Tier 2 — Theatre or Campaign Scoped**
+Search and outputs are limited to a defined theatre or campaign. Designed for general museums, regional heritage organisations, and customers with a broader but still geographically constrained focus.
+
+**Tier 3 — Full Corpus Access**
+Search and outputs span all units, all theatres, and all date ranges. Designed for academic institutions, government bodies, and high value commercial customers requiring unrestricted access to the full corpus.
+
+**Tier 4 — Full Corpus Plus Export and API Access**
+All Tier 3 capabilities plus bulk export, GeoJSON export, and API access for integration into third party products and platforms. Designed for institutional partners and Frame 44 commercial products.
+
+**Desired Result:**
+- Every customer operates within a corpus scope appropriate to their use case and subscription
+- No customer receives a deliberately degraded product — scope is constrained, quality is not
+- Tier 4 enables Frame 44 and institutional partners to build their own products on top of the platform
+
+### Token Cost Estimates by Customer Type ###
+
+All customer interactions with the platform consume computational resources in two forms: database compute (semantic search, spatial queries, filtering) and LLM token usage (retrieval context and report generation). Database compute costs are minimal and consistent regardless of query complexity. Token costs vary significantly based on how much corpus material is retrieved and how much is generated in response.
+
+**Cost per interaction type:**
+- Simple semantic search, no generation: ~$0.01–0.03
+- Search with short generated summary: ~$0.03–0.08
+- Full narrative report with sourced retrieval: ~$0.10–0.25
+- Full report with extensive cross-corpus retrieval: ~$0.25–0.50
+- Map and position queries: database compute only, negligible token cost
+
+---
+
+**Project '44 Community Users**
+High search frequency, low report generation. Community users browse, search, and follow threads of interest. They generate short summaries occasionally and full reports rarely.
+
+- Estimated interactions per month: 20–50 searches, 2–5 short summaries, 0–1 full reports
+- Estimated monthly token cost per user: $0.50–$2.00
+- Subscription viability: viable at $5–10/month with query caps on full report generation
+
+---
+
+**Relative Research**
+Low total interaction volume but report-heavy. A descendant typically conducts a focused research session, generates one or two full reports, and returns occasionally for follow-up searches. This is largely a one-time high-value interaction rather than a recurring high-volume use.
+
+- Estimated interactions per session: 5–10 searches, 1–2 full reports
+- Estimated cost per session: $0.50–$1.50
+- Estimated lifetime cost per customer: $1.00–$3.00
+- Subscription viability: viable as a one-time purchase or low monthly subscription with report generation limits
+
+---
+
+**Museum Staff Admin Panel**
+Moderate regular usage. Staff run searches frequently to support exhibit development and research enquiries but generate full reports less often. Usage is spread across the working week rather than concentrated in single sessions.
+
+- Estimated interactions per month: 50–100 searches, 5–10 short summaries, 2–5 full reports
+- Estimated monthly token cost per staff user: $2.00–$5.00
+- Subscription viability: viable at institutional subscription pricing, cost absorbed across staff team
+
+---
+
+**Museum Visitor Kiosk**
+High volume, low cost per interaction. Visitors run a single focused search and receive a short generated summary. No full reports are generated at the kiosk — the full report is saved to Project '44 for home access. Volume can be very high at busy museums.
+
+- Estimated cost per visitor interaction: $0.03–0.08
+- Estimated daily visitor interactions at a busy museum: 50–200
+- Estimated daily token cost: $1.50–$16.00
+- Subscription viability: viable as a flat monthly institutional fee with per-interaction cost absorbed into the subscription cap
+
+---
+
+**Academic Research**
+High volume, high complexity. Academic users run deep cross-corpus queries, retrieve large amounts of source material, and generate detailed attributed reports. This is the most token-intensive customer type.
+
+- Estimated interactions per month: 100–300 searches, 10–20 full reports, frequent bulk retrieval
+- Estimated monthly token cost per user: $10.00–$30.00
+- Subscription viability: viable at Tier 3/4 institutional pricing, with API rate limits and query caps for bulk access
+
+---
+
+**UXO Search**
+Low volume, spatially constrained, export-heavy. UXO professionals run focused geographic queries and export results. Interactions are infrequent but high value. Report generation is minimal — raw results and exports are the primary output.
+
+- Estimated interactions per project: 10–30 spatial queries, 2–5 short summaries, minimal full reports
+- Estimated cost per project: $0.50–$2.00
+- Subscription viability: viable as a per-project or annual institutional subscription
+
+---
+
+**Media Research**
+Moderate volume, report-heavy. Media researchers chase specific stories and generate narrative reports to support their work. Usage is project-based — intensive during production, dormant between projects.
+
+- Estimated interactions per project: 20–50 searches, 5–10 full reports
+- Estimated cost per project: $2.00–$8.00
+- Subscription viability: viable as a per-project subscription or annual access for organisations with ongoing production
+
+---
+
+**Historical Mystery and Artifact Research**
+Unpredictable volume, potentially very high retrieval cost. Cross-corpus aggregation queries pull large amounts of material from many sources. This is the hardest use case to cost-cap reliably.
+
+- Estimated interactions per investigation: 50–200 searches, extensive cross-corpus retrieval, 5–15 full reports
+- Estimated cost per investigation: $5.00–$25.00
+- Subscription viability: viable at Tier 3/4 with query complexity caps and rate limiting to prevent runaway costs
+
+---
+
+**Key Principle:**
+Subscription tiers control corpus scope. Query and report generation caps control token cost exposure. No customer type should be able to generate unbounded token costs within a fixed subscription — every tier includes defined limits on full report generation and bulk retrieval, with overage pricing for high volume users.
+
+### Project '44 Community ###
+
+**Actor:** Amateur historians, local heritage organisations, Dutch and Belgian community groups, Faces to Graves volunteers, students, and anyone with a general interest in WWII history who is not necessarily researching a specific relative.
+
+**Need:** To explore, research, and contribute to a living archive of WWII primary source material. Community users approach the corpus from many different angles — following a specific battle, researching a local area, verifying a grave identification, or simply browsing out of historical curiosity. Many are also the volunteer contributor base that verifies OCR, confirms positions, and improves data quality across the platform.
+
+**How They Use It:**
+- Community users run natural language searches, browse by unit, date range, theatre, or geographic area
+- Spatial search allows local community groups to find all documented military activity in their town, village, or region
+- Faces to Graves volunteers can search by personnel name to find diary mentions of specific individuals and corroborate grave identifications
+- Community users can contribute to the platform by verifying OCR text, confirming position classifications, and tagging document types — improving data quality for all users
+- Premium and Patreon subscribers unlock deeper report generation, broader cross-corpus search, and higher query limits
+- All community interactions are token-capped at the community tier to keep subscription costs viable
+
+**Outputs:**
+- Search results with source attribution across the corpus within community tier limits
+- Short generated summaries for community tier users
+- Full generated reports for premium and Patreon subscribers
+- Contributor actions — verified OCR, confirmed positions, document tags — recorded and attributed to the contributing user
+
+**Desired Result:**
+- Project '44 becomes the go-to public resource for anyone interested in the WWII history of Northwest Europe and Italy
+- Local communities in the Netherlands, Belgium, and Italy can explore what happened in their specific area during the war
+- The community contributor base continuously improves data quality across the platform — OCR accuracy, position confidence, and document classification all improve as volunteer engagement grows
+- Premium and Patreon tiers provide a sustainable revenue stream while keeping basic access open to the broader community
+- The community layer feeds the commercial products — a community user who finds a reference to their relative becomes a Relative Research customer.
+- 
+### Relative Research ###
+
+**Actor:** A family member or descendant researching a relative who served in a WWII unit covered by the corpus.
+
+**Need:** To understand what their relative experienced during the war — where they were, what they did, what happened around them, and what the broader context of their service was. The customer may know very little about their relative's service or quite a lot. The system works with whatever they can provide.
+
+**How They Use It:**
+- Customer provides whatever information they have about their relative: name, unit, date range, service number, theatre, or any combination
+- The system searches the corpus for mentions of the individual by name, and retrieves all relevant narrative, non-narrative, and secondary source material for the unit and date range provided
+- A report is generated drawing on all retrieved material, placing the relative's unit in operational context and surfacing any direct mentions of the individual by name
+- A map is generated alongside the report showing unit positions, patrol routes, objectives, and operational area for the relevant period
+- Report and map are saved to the customer's profile for future reference
+- Customer can order a printed version of the report and map through a drop shipping integration
+
+**Outputs:**
+- Generated narrative report with full source attribution
+- Generated map with unit positions, routes, and operational context for the relevant period
+- Applicable period map overlays
+- Direct name mentions surfaced and highlighted within the report where found
+- Report and map saved to customer profile
+- Printable report and map available for drop shipping
+
+**Desired Result:**
+- Any descendant regardless of how much or how little they know about their relative's service can receive a meaningful, accurate, and personal account of what their relative's unit experienced
+- Customers who provide more information receive richer, more personalised outputs
+- The experience feels personal and specific — not a generic unit history but a account centred on the customer's relative and their unit's experience
+- Customers have a permanent record of their research saved to their profile that they can return to, share, or have printed
+
+### Museum Research ###
+
+**Actor:** A military museum, regimental museum, or heritage organisation using the platform to support both internal staff research and visitor-facing experiences.
+
+**Need:** Staff need a research tool to support exhibit development, historical enquiries, and institutional knowledge. Visitors need an engaging, accessible experience that connects them personally to the museum's unit or theatre without requiring prior military history knowledge.
+
+**How They Use It — Staff Admin Panel:**
+- Staff access an admin panel scoped to the museum's subscription tier
+- Staff can run natural language queries, filter by unit, date range, and document type, and retrieve narrative and non-narrative material relevant to their research
+- Staff can generate reports drawing on all corpus material within their tier scope
+- Staff can save and organise research for internal reference and exhibit development
+
+**How They Use It — Visitor Touchscreen:**
+- Visitors interact with a touchscreen interface scoped to the museum's unit or theatre
+- Visitors enter whatever information they have about a relative — name, unit, approximate dates — and the system searches the corpus for relevant material
+- A report and map are generated and displayed in full on the touchscreen
+- Visitors can save their results by entering their email address
+- Saved results are stored on Project '44 and the visitor receives an email prompting them to create an account and access their report at home
+
+**Outputs:**
+- Staff: generated reports, search results, and saved research within tier scope
+- Visitors: full generated report and map displayed on the touchscreen
+- Visitors: results saved to Project '44 and emailed with an account creation prompt
+
+**Desired Result:**
+- Museum staff have a powerful internal research tool scoped to their collection focus
+- Visitors have a personal, engaging experience that connects them to the museum's unit history through their own family connection
+- Every visitor who saves their results becomes a potential Project '44 customer — the touchscreen experience is a direct acquisition funnel
+- The museum delivers a richer, more personalised visitor experience without building or maintaining the underlying platform themselves
+
+### Academic Research ###
+
+**Actor:** An academic researcher, historian, graduate student, or university institution conducting research using primary and secondary WWII source material.
+
+**Need:** Access to a deep, fully attributed corpus of primary and secondary source material that can be searched, queried, and cited with confidence. Academic users require complete provenance on every result — every chunk of text must be traceable to its source document, archive, unit, and date.
+
+**How They Use It:**
+- Researcher runs natural language queries or structured filtered searches across the full corpus
+- Results are returned with full source attribution — document, archive, unit, date, and document type — suitable for academic citation
+- Researcher can search spatially using a bounding box to find all activity in a geographic area across any unit or date range
+- Researcher can search by personnel name to surface all mentions of a specific individual across the corpus
+- Researcher can combine filters — unit, date range, theatre, document type — to narrow results to a precise research question
+- Results can be saved to a research profile and organised by project or topic
+- Tier 4 customers can export results and access the corpus via API for bulk queries and integration into their own research tools
+
+**Outputs:**
+- Search results with full source attribution suitable for academic citation
+- Generated reports drawing on retrieved material with attributed sources
+- Saved research organised by project or topic
+- Tier 4: bulk export and API access
+
+**Desired Result:**
+- Researchers can answer precise historical questions across the full corpus in a fraction of the time traditional archival research would require
+- Every result is fully attributed and citable — the platform is trustworthy as an academic research tool
+- Spatial, temporal, unit, and natural language search entry points allow researchers to approach the corpus from any angle their research requires
+- The platform surfaces connections across units, theatres, and date ranges that would be impossible to find manually across fragmented archival sources
+
+### UXO Search ###
+
+**Actor:** A UXO disposal company, military engineer, government body, or heritage organisation responsible for identifying, assessing, or clearing unexploded ordnance in formerly contested areas.
+
+**Need:** To quickly locate all documented military activity, bombardments, mine laying, and munitions references within a specific geographic area, supported by primary source evidence traceable to its origin.
+
+**How They Use It:**
+- User defines a geographic area of interest using a bounding box on the map
+- The system returns all positions, grid references, and diary passages referencing military activity within that area — artillery positions, mine laying, bombardments, defensive works, and similar activity
+- Results are displayed as mapped positions with attached narrative passages from the source diary entries
+- User can filter by activity type, date range, unit, and confidence level
+- Position confidence tiers are clearly displayed — verified Level 1 positions are distinguished from unverified Level 2 and Level 3 positions
+- Full source attribution on every result allows the user to trace findings back to the original primary source document
+- Results can be exported for use in field assessments and official reports
+
+**Outputs:**
+- Mapped positions of all relevant military activity within the defined area
+- Source diary passages attached to each position
+- Confidence tier clearly displayed per position
+- Full provenance on every result — document, archive, unit, date
+- Exportable results for field assessment and reporting
+
+**Desired Result:**
+- UXO professionals can rapidly identify areas of documented military activity that may present ordnance risk, without manually searching fragmented archival sources
+- Primary source evidence with full provenance supports official assessments and reporting requirements
+- Position confidence tiers allow professionals to distinguish between verified and unverified locations and prioritise accordingly
+- A tool that could directly contribute to public safety in formerly contested areas across Northwest Europe and Italy
+
+### Media Research ###
+
+**Actor:** A journalist, documentary filmmaker, author, or media organisation researching WWII events for publication, broadcast, or production.
+
+**Need:** To find specific, vivid, well-attributed primary source material that supports a narrative they are developing. Media researchers are often chasing a specific story, event, or person rather than conducting broad historical research. They need compelling source material quickly and need to be able to cite it accurately.
+
+**How They Use It:**
+- Researcher runs natural language queries describing the story, event, or person they are researching
+- Results surface relevant narrative passages, non-narrative documents, and secondary source material with full attribution
+- Researcher can narrow results by date range, theatre, unit, or geographic area as their research develops
+- Spatial search allows the researcher to find all documented activity around a specific location relevant to their story
+- Personnel search surfaces all mentions of specific individuals across the corpus
+- Generated reports can be produced drawing on retrieved material, providing a structured narrative overview with cited sources
+- Results and reports are saved to a research profile and organised by project
+
+**Outputs:**
+- Search results with full source attribution suitable for publication citation
+- Generated narrative reports with attributed primary and secondary sources
+- Mapped positions and period overlays relevant to the story or event being researched
+- Saved research organised by project
+
+**Desired Result:**
+- Media researchers can find compelling, specific, well-attributed primary source material in a fraction of the time traditional archival research would require
+- The platform surfaces stories, events, and individuals that would be impossible to find manually across fragmented archival sources
+- Every result is fully attributed — researchers can cite primary sources with confidence
+- The corpus becomes a go-to resource for any media organisation producing WWII content covering the Northwest Europe and Italian theatres
+
+### Historical Mystery and Artifact Research ###
+
+**Actor:** A researcher, investigator, historian, or organisation attempting to locate specific objects, resolve undocumented historical events, or trace activities that have no clear archival resolution.
+
+**Need:** To aggregate fragmentary references across a large corpus that individually appear insignificant but collectively point toward a resolution. No single diary entry solves the mystery — the value is in surfacing and connecting partial references across multiple units, dates, and locations that a manual researcher could never feasibly cross-reference.
+
+**How They Use It:**
+- Researcher runs natural language queries describing the subject of their investigation — an object, an event, a location, a unit action
+- Results surface all relevant references across the full corpus regardless of unit or date, including passing mentions in non-narrative documents that a traditional researcher would never find
+- Spatial search allows the researcher to constrain results to a geographic area of interest and map all documented activity within it
+- Cross-unit search surfaces what neighbouring units were doing at the same time and place, providing corroborating or contradicting context
+- Researcher can follow threads across the corpus — a reference in one unit's diary leads to a patrol report in another, leads to a message log in a third
+- Results and generated reports are saved to a research profile and organised by investigation
+
+**Outputs:**
+- Aggregated search results across the full corpus with full source attribution
+- Mapped positions of all relevant activity within a defined area
+- Generated reports drawing on retrieved material with attributed sources
+- Saved research organised by investigation topic
+
+**Desired Result:**
+- Researchers can surface and connect fragmentary references across the full corpus that would be impossible to find and aggregate manually
+- The platform transforms the corpus from a collection of isolated unit histories into a connected, queryable record of everything that happened across an entire theatre
+- Investigations that would take years of manual archival research can be conducted in days
+- Every finding is fully attributed to primary sources, giving investigations an evidential foundation

p44_use-case_step-by-step_cdn-war-diaries.md

@@ -0,0 +1,174 @@
+WD_1 - Import War Diaries
+=========================
+User: Admin
+Input: War diary (PDF or JPEG), separated by month
+Result: Document stored server-side, available for OCR
+
+1. Go to Frame 44 UI or plugin
+2. Upload PDF or JPEG
+3. Assemble JPEG uploads into single PDF, confirm page order
+4. Assign to existing unit or create new unit record
+5. Fill in provenance metadata: archive source, unit ID, date range, digitisation quality
+6. Tag narrative page range in thumbnail gallery
+7. Click submit
+8. Document stored server-side, status set to "pending OCR"
+
+---
+
+WD_2 - OCR War Diaries
+=======================
+User: System (automated)
+Input: Imported document, narrative page range metadata
+Result: Raw OCR text stored per page, narrative and non-narrative pages split into separate processing tracks
+
+1. System detects document with status "pending OCR"
+2. Full document is processed through OCR pipeline
+3. Raw OCR text stored per page in database
+4. Narrative pages set to status "pending review", queued for OCR editor
+5. Non-narrative pages set to status "pending classification", queued for document classification
+
+---
+
+WD_3 - Document Classification
+===============================
+User: System (automated), Volunteer (review)
+Input: Non-narrative pages from OCR
+Result: Every non-narrative page has a document type tag before entering the RAG pipeline
+
+1. System detects non-narrative pages with status "pending classification"
+2. System classifies each page by document type: message log, intel report, newsletter, Part 1 Orders, nominal roll, appendix, sitrep, patrol report, trace, etc.
+3. Confidence score assigned per classification
+4. High confidence classifications proceed automatically to RAG chunking
+5. Low confidence classifications flagged for human review
+6. Reviewer confirms or corrects classification
+7. Verified classifications fed back into system to improve future accuracy
+8. All classified pages proceed to RAG pipeline
+
+---
+
+WD_4 - OCR Editor
+=================
+User: Volunteer
+Input: Narrative pages with status "pending review"
+Result: Every narrative page verified by a human before downstream processing
+
+1. Volunteer logs into OCR editor
+2. Volunteer sees work queue of narrative pages with status "pending review"
+3. Volunteer selects a war diary to work on
+4. Editor displays original page image on left, raw OCR text on right
+5. Volunteer reads and corrects OCR text
+6. Changes tracked per user for version control
+7. Volunteer marks page as "verified"
+8. Verified pages queued for entity extraction and RAG chunking
+
+---
+
+WD_5 - Entity Extraction
+=========================
+User: System (automated), Volunteer (flagged items)
+Input: Verified narrative pages, classified non-narrative pages
+Result: All entities extracted and stored as structured records linked to source page and RAG chunk
+
+1. System detects verified narrative pages and classified non-narrative pages
+2. System extracts entities from all pages:
+    - Personnel: names and ranks
+    - Place names: locations mentioned in text
+    - Grid references: WWII map grid references
+    - Unit mentions: friendly, allied, and enemy units
+    - Events: attacks, patrols, reliefs, casualties, objectives
+3. Structured record created per entity: entity type, value, context snippet, source page, source RAG chunk, date, unit
+4. Low confidence or ambiguous extractions flagged for volunteer review
+5. Volunteer confirms or corrects flagged extractions
+6. All entity records stored in database linked to source page and RAG chunk
+
+---
+
+WD_6 - Gazetteer
+================
+User: System (automated), Volunteer (flagged items)
+Input: Extracted place name records, known unit positions with date, modern GIS datasets
+Result: All place names resolved to modern coordinates and standardised spellings
+
+1. System detects extracted place name records
+2. For each place name, system spatially constrains candidate matches to within a defined radius of the unit's known position on that date
+3. Candidate matches ranked by name similarity and spatial proximity
+4. High confidence matches resolved automatically
+5. Where multiple diary entries from different units independently resolve the same string to the same coordinate, confidence increases automatically
+6. Low confidence matches flagged for volunteer review
+7. Volunteer confirms or corrects flagged matches
+8. Gazetteer record created per resolved place: variant spellings, modern coordinate, confidence score, corroborating source entries
+9. Unresolved place names remain flagged for future review
+10. Gazetteer feeds back into improving future place name extraction
+
+---
+
+WD_7 - RAG Pipeline
+====================
+User: System (automated)
+Input: Verified narrative OCR text, classified non-narrative pages, entity extraction records
+Result: Full corpus chunked, embedded, and stored in Postgres with pgvector and PostGIS, queryable by semantic similarity, metadata filter, and spatial query
+
+1. System detects verified narrative pages and classified non-narrative pages
+2. Pages chunked into segments respecting natural document boundaries
+3. Each chunk embedded and stored in Postgres with pgvector
+4. Chunk metadata attached: source document, page, unit, date, document type, confidence level, linked entity records
+5. PostGIS spatial data attached to chunks where gazetteer-resolved coordinates are available
+6. Chunks queryable by semantic similarity, metadata filter, spatial query, or any combination in a single query
+
+---
+
+WD_8 - Grid Reference Extraction and Verification
+==================================================
+User: System (automated), Volunteer (verification)
+Input: Grid reference records from entity extraction, known unit positions with date
+Result: All grid references resolved to real world coordinates, verified positions linked to source RAG chunk
+
+1. System detects grid reference records from entity extraction
+2. System resolves each grid reference to coordinates using WWII map projections and known 100,000m grid square for the unit and date
+3. Candidate position created per grid reference: coordinate, source diary entry, date, unit, position type, confidence level, relevance category
+4. Candidate positions stored in Postgres with PostGIS, linked to source RAG chunk
+5. All candidate positions set to status "pending verification"
+6. Volunteer reviews candidate positions against reference materials
+7. Volunteer confirms or corrects each position
+8. Verified positions set to confidence Level 1
+9. Unverified high confidence positions set to Level 2
+10. Low confidence positions remain Level 3 and flagged for further review
+11. Position type tagged: end of day, patrol, enemy, objective, etc.
+12. Relevance category tagged: own unit activity or reference to another unit
+
+---
+
+WD_9 - Document Versioning and Corpus Expansion
+================================================
+User: Admin
+Input: Higher quality scan of existing document, or new document belonging to existing unit
+Result: Corpus updated without disrupting existing data
+
+Scenario 1 — Higher Quality Scan:
+1. Admin uploads higher quality scan via standard import
+2. Admin identifies existing document record in database
+3. Source image updated in database
+4. All downstream records remain intact and unchanged
+
+Scenario 2 — New Document Found:
+1. Admin uploads new document via standard import pipeline
+2. Admin assigns document to existing unit record
+3. Document processed through full pipeline from WD_1 onwards
+4. All outputs appended to existing unit records in database
+
+---
+
+WD_10 - Secondary Sources
+==========================
+User: Admin
+Input: Published secondary source as clean digital text or scanned copy
+Result: Secondary source ingested into RAG pipeline, attributed as secondary literature, entities extracted
+
+1. Admin uploads secondary source via import pipeline
+2. Admin assigns source to theatre and formation coverage (e.g. II Canadian Corps, Northwest Europe)
+3. If clean digital text: proceeds directly to entity extraction and RAG chunking
+4. If scanned copy: processed through OCR pipeline without human verification gate, then proceeds to entity extraction and RAG chunking
+5. Entity extraction runs on all secondary sources: personnel, place names, unit mentions, events
+6. Extracted place names and grid references passed to gazetteer and grid reference verification
+7. Secondary source chunks stored in Postgres, attributed as secondary literature in all records
+8. Secondary source queryable alongside primary sources in all downstream products

prompts/v2-tier3-report.md

@@ -7,26 +7,13 @@ his unit experienced during his service window. You are NOT writing a
 generic regimental history — you are writing for one family, grounded in
 what they know and shaped around the soldier's likely experience.
 
-## Input
-You will be given the war diary OCR for the Calgary Highlanders covering
-September 1944 through May 1945. The diary is in .docx form with a three-
-column structure:
-- Column 1: Date
-- Column 2: Narrative entry (the diarist's account of the day)
-- Column 3: Comments / page reference back to the original diary
-
-Each row is a day's entry. Some entries are operational (movements, attacks,
-casualties), some are administrative (parades, pay, training), some are
-human texture (weather, food, civilian encounters, rest). All of these
-matter for the report — the human texture is what makes a descendant report
-emotionally resonant, not just the operational events.
 
 ## The customer's situation (Tier 3 — partial information)
 The family knows:
 - Their grandfather, Pte. Bill Bloggins, served with the Calgary Highlanders.
 - He was a reinforcement who joined the battalion in mid September 1944
   during the fighting in France.
-- He was wounded in action in early November 1944, somewhere in
+- He was wounded in action in late October 1944, somewhere in
   "Holland or Germany," during what the family calls "the Scheldt."
 - He was evacuated to England and did not return to the unit.
 

scripts/step5_embed_wardiary.py

@@ -0,0 +1,297 @@
+"""
+step5_embed_wardiary.py
+───────────────────────
+Reads step3/step4 JSON output files from a TestRun (or any outputs folder) and
+embeds them into the pgvector document_chunks table.
+
+Sources ingested:
+  *_narratives.json     → chunk_type=narrative   (one chunk per diary entry)
+  *_appx_records.json   → chunk_type=appx_record (one chunk per document page)
+
+Positions JSON (*_positions.json, *_appx_positions.json) are NOT embedded here —
+they belong in a PostGIS geometry table, not a vector search table.
+
+Usage:
+  python scripts/step5_embed_wardiary.py --input_dir TestRun/outputs --nationality canadian
+  python scripts/step5_embed_wardiary.py --input_dir TestRun/outputs --nationality canadian --dry_run
+"""
+
+import os
+import sys
+import json
+import argparse
+from pathlib import Path
+from dotenv import load_dotenv
+from openai import OpenAI
+
+load_dotenv()
+
+# ── Embedding config (same as GoogleDocumentOCR/embed_and_store.py) ──────────
+EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5"
+EMBEDDING_DIMS  = 768
+BATCH_SIZE      = 100
+
+client = OpenAI(
+    api_key=os.getenv("DEEPINFRA_API_KEY"),
+    base_url="https://api.deepinfra.com/v1/openai",
+)
+
+# ── DB (reuse GoogleDocumentOCR/db.py) ───────────────────────────────────────
+sys.path.insert(0, str(Path(__file__).parent.parent / "GoogleDocumentOCR"))
+from db import init_db, get_conn
+from psycopg2.extras import execute_values
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Embedding
+# ─────────────────────────────────────────────────────────────────────────────
+
+def embed_texts(texts: list[str]) -> list[list[float]]:
+    all_embeddings = []
+    for i in range(0, len(texts), BATCH_SIZE):
+        batch = texts[i:i + BATCH_SIZE]
+        response = client.embeddings.create(input=batch, model=EMBEDDING_MODEL)
+        all_embeddings.extend([item.embedding for item in response.data])
+    return all_embeddings
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# DB insert (extended schema with chunk_type, date_entry, unit)
+# ─────────────────────────────────────────────────────────────────────────────
+
+def init_extended_schema():
+    """Add chunk_type, date_entry, unit columns if they don't exist yet."""
+    conn = get_conn()
+    cur  = conn.cursor()
+    for col, typedef in [
+        ("chunk_type",  "TEXT DEFAULT 'unknown'"),
+        ("date_entry",  "TEXT"),
+        ("unit",        "TEXT"),
+    ]:
+        cur.execute(f"ALTER TABLE document_chunks ADD COLUMN IF NOT EXISTS {col} {typedef};")
+    conn.commit()
+    cur.close()
+    conn.close()
+
+
+def upsert_rows(rows: list[dict]):
+    conn = get_conn()
+    cur  = conn.cursor()
+    execute_values(
+        cur,
+        """
+        INSERT INTO document_chunks
+            (filename, page_num, chunk_index, text, embedding,
+             nationality, corpus, chunk_type, date_entry, unit)
+        VALUES %s
+        """,
+        [
+            (
+                r["filename"], r["page_num"], r["chunk_index"], r["text"], r["embedding"],
+                r["nationality"], r["corpus"], r["chunk_type"],
+                r.get("date_entry"), r.get("unit"),
+            )
+            for r in rows
+        ],
+        template="(%s, %s, %s, %s, %s::vector, %s, %s, %s, %s, %s)"
+    )
+    conn.commit()
+    cur.close()
+    conn.close()
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Text builders — turn each JSON record into a single embeddable string
+# ─────────────────────────────────────────────────────────────────────────────
+
+def narrative_to_text(entry: dict) -> str:
+    parts = []
+    if entry.get("date"):
+        parts.append(f"Date: {entry['date']}")
+    if entry.get("place"):
+        parts.append(f"Location: {entry['place']}")
+    if entry.get("weather"):
+        parts.append(f"Weather: {entry['weather']}")
+    if entry.get("summary"):
+        parts.append(entry["summary"])
+    return "\n".join(parts).strip()
+
+
+def appx_record_to_text(record: dict) -> str:
+    parts = []
+    if record.get("page_type"):
+        parts.append(f"Document type: {record['page_type']}")
+    if record.get("document_id"):
+        parts.append(f"Document ID: {record['document_id']}")
+    if record.get("date"):
+        parts.append(f"Date: {record['date']}")
+    if record.get("time"):
+        parts.append(f"Time: {record['time']}")
+    if record.get("from_unit"):
+        parts.append(f"From: {record['from_unit']}")
+    if record.get("to_units"):
+        parts.append(f"To: {', '.join(record['to_units'])}")
+    if record.get("subject"):
+        parts.append(f"Subject: {record['subject']}")
+    if record.get("summary"):
+        parts.append(record["summary"])
+    if record.get("units_mentioned"):
+        parts.append(f"Units mentioned: {', '.join(record['units_mentioned'])}")
+    return "\n".join(parts).strip()
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Ingest functions
+# ─────────────────────────────────────────────────────────────────────────────
+
+def ingest_narratives(json_path: Path, nationality: str, unit: str, dry_run: bool):
+    data = json.loads(json_path.read_text(encoding="utf-8"))
+    print(f"  {json_path.name}: {len(data)} narrative entries")
+
+    chunks = []
+    for idx, entry in enumerate(data):
+        text = narrative_to_text(entry)
+        if not text:
+            continue
+        chunks.append({
+            "filename":    json_path.name,
+            "page_num":    entry.get("source_page"),
+            "chunk_index": idx,
+            "text":        text,
+            "nationality": nationality,
+            "corpus":      "war_diary_narrative",
+            "chunk_type":  "narrative",
+            "date_entry":  entry.get("date"),
+            "unit":        unit,
+        })
+
+    if not chunks:
+        print("    No embeddable entries.")
+        return
+
+    if dry_run:
+        print(f"    DRY RUN — would embed+store {len(chunks)} chunks")
+        print(f"    Sample: {chunks[0]['text'][:120]}...")
+        return
+
+    print(f"    Embedding {len(chunks)} chunks...")
+    embeddings = embed_texts([c["text"] for c in chunks])
+    for c, emb in zip(chunks, embeddings):
+        c["embedding"] = emb
+
+    upsert_rows(chunks)
+    print(f"    ✓ Stored {len(chunks)} narrative chunks")
+
+
+def ingest_appx_records(json_path: Path, nationality: str, unit: str, dry_run: bool):
+    data = json.loads(json_path.read_text(encoding="utf-8"))
+    print(f"  {json_path.name}: {len(data)} appendix records")
+
+    chunks = []
+    for idx, record in enumerate(data):
+        text = appx_record_to_text(record)
+        if not text:
+            continue
+        chunks.append({
+            "filename":    json_path.name,
+            "page_num":    record.get("source_page"),
+            "chunk_index": idx,
+            "text":        text,
+            "nationality": nationality,
+            "corpus":      "war_diary_appendix",
+            "chunk_type":  "appx_record",
+            "date_entry":  record.get("date"),
+            "unit":        unit,
+        })
+
+    if not chunks:
+        print("    No embeddable records.")
+        return
+
+    if dry_run:
+        print(f"    DRY RUN — would embed+store {len(chunks)} chunks")
+        print(f"    Sample: {chunks[0]['text'][:120]}...")
+        return
+
+    print(f"    Embedding {len(chunks)} chunks...")
+    embeddings = embed_texts([c["text"] for c in chunks])
+    for c, emb in zip(chunks, embeddings):
+        c["embedding"] = emb
+
+    upsert_rows(chunks)
+    print(f"    ✓ Stored {len(chunks)} appendix record chunks")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Unit name extraction from filename
+# ─────────────────────────────────────────────────────────────────────────────
+
+UNIT_PATTERNS = {
+    "calgary-highlanders":  "Calgary Highlanders",
+    "calgary_highlanders":  "Calgary Highlanders",
+    "blackwatch":           "Black Watch (RHC)",
+    "black-watch":          "Black Watch (RHC)",
+    "5cib":                 "5 Canadian Infantry Brigade",
+}
+
+def extract_unit(filename: str) -> str:
+    lower = filename.lower()
+    for key, name in UNIT_PATTERNS.items():
+        if key in lower:
+            return name
+    return "unknown"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Main
+# ─────────────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="Embed war diary JSON outputs into pgvector")
+    parser.add_argument("--input_dir",   required=True,  help="Folder containing step3/step4 JSON files")
+    parser.add_argument("--nationality", default="canadian", help="Nationality tag (default: canadian)")
+    parser.add_argument("--dry_run",     action="store_true", help="Print what would be done without writing to DB")
+    args = parser.parse_args()
+
+    input_dir = Path(args.input_dir)
+    if not input_dir.exists():
+        print(f"ERROR: input_dir not found: {input_dir}")
+        sys.exit(1)
+
+    if not args.dry_run:
+        init_db()
+        init_extended_schema()
+
+    # Find all JSON files that are narratives or appx_records (skip positions)
+    narrative_files    = sorted(input_dir.glob("*_narratives.json"))
+    appx_record_files  = sorted(input_dir.glob("*_appx_records.json"))
+
+    skipped = [f.name for f in input_dir.glob("*.json")
+               if f not in narrative_files + appx_record_files]
+
+    print(f"\n{'DRY RUN — ' if args.dry_run else ''}War Diary Embedding Ingest")
+    print(f"Input dir : {input_dir}")
+    print(f"Nationality: {args.nationality}")
+    print(f"Narratives : {len(narrative_files)} file(s)")
+    print(f"Appx records: {len(appx_record_files)} file(s)")
+    if skipped:
+        print(f"Skipping (positions/other): {', '.join(skipped)}\n")
+
+    for f in narrative_files:
+        unit = extract_unit(f.name)
+        print(f"\n[narrative] {f.name} -> unit: {unit}")
+        ingest_narratives(f, args.nationality, unit, args.dry_run)
+
+    for f in appx_record_files:
+        unit = extract_unit(f.name)
+        print(f"\n[appx_record] {f.name} -> unit: {unit}")
+        ingest_appx_records(f, args.nationality, unit, args.dry_run)
+
+    print("\nDone.")
+
+
+if __name__ == "__main__":
+    main()
+
+
+

scripts/step6_generate_report.py

@@ -0,0 +1,285 @@
+"""
+step6_generate_report.py
+─────────────────────────────────────────────────────────────────────────────
+Generates a Tier-3 descendant report by combining:
+  1. Semantic search against the pgvector DB (war diary chunks)
+  2. The prompt template from prompts/v2-tier3-report.md
+  3. Claude (Anthropic) for the final report generation
+
+Usage
+─────
+  # Default — uses the Bloggins placeholder soldier
+  python scripts/step6_generate_report.py
+
+  # Real soldier
+  python scripts/step6_generate_report.py \
+    --name "Pte. John Smith" \
+    --unit "Calgary Highlanders" \
+    --joined "mid September 1944" \
+    --wounded "late October 1944" \
+    --notes "Family knows he was in D Company. Was evacuated to England."
+
+Output
+──────
+  reports/<soldier-slug>_<timestamp>_report.md
+"""
+
+import os
+import sys
+import argparse
+import textwrap
+from datetime import datetime
+from pathlib import Path
+from dotenv import load_dotenv
+
+# ── Paths ─────────────────────────────────────────────────────────────────────
+REPO_ROOT   = Path(__file__).resolve().parent.parent
+PROMPT_FILE = REPO_ROOT / "prompts" / "v2-tier3-report.md"
+REPORTS_DIR = REPO_ROOT / "reports"
+
+# ── Add GoogleDocumentOCR to path so we can import db + embed_and_store ───────
+sys.path.insert(0, str(REPO_ROOT / "GoogleDocumentOCR"))
+
+load_dotenv()
+
+# ── Search queries to give broad coverage of the service window ────────────────
+SEARCH_QUERIES = [
+    "Calgary Highlanders September 1944 operations movements France",
+    "Calgary Highlanders October 1944 Scheldt fighting Holland",
+    "Calgary Highlanders November 1944 casualties wounded Maas",
+    "battalion advance attack objective company platoon",
+    "reinforcements joined unit billets rest weather rations",
+    "parades training administrative daily routine",
+    "casualties killed wounded evacuated",
+    "officers commanding lieutenant colonel major company commander",
+    "civilian population towns villages liberated",
+    "enemy German positions shelling mortars small arms fire",
+]
+
+CHUNKS_PER_QUERY = 5   # results per query
+MAX_CONTEXT_CHARS = 80_000  # ~20k tokens — safe Claude window
+
+
+def retrieve_diary_chunks(nationality: str = "canadian",
+                           corpus: str = "war_diary_narrative") -> list[dict]:
+    """
+    Run multiple semantic searches and return a deduplicated, sorted list
+    of diary chunks.
+    """
+    from embed_and_store import embed_texts
+    from db import semantic_search
+
+    seen_ids = set()
+    chunks = []
+
+    print(f"  Running {len(SEARCH_QUERIES)} semantic searches...")
+    for query in SEARCH_QUERIES:
+        embedding = embed_texts([query])[0]
+        rows = semantic_search(
+            embedding,
+            top_k=CHUNKS_PER_QUERY,
+            nationality=nationality,
+            corpus=corpus,
+        )
+        for row in rows:
+            filename, page_num, chunk_index, text, nat, corp, similarity = row
+            key = (filename, page_num, chunk_index)
+            if key not in seen_ids:
+                seen_ids.add(key)
+                chunks.append({
+                    "filename":    filename,
+                    "page_num":    page_num,
+                    "chunk_index": chunk_index,
+                    "text":        text,
+                    "similarity":  similarity,
+                })
+
+    # Sort by filename then page so the context reads chronologically
+    chunks.sort(key=lambda c: (c["filename"], c["page_num"], c["chunk_index"]))
+    print(f"  Retrieved {len(chunks)} unique chunks after deduplication.")
+    return chunks
+
+
+def build_context_block(chunks: list[dict], max_chars: int = MAX_CONTEXT_CHARS) -> str:
+    """
+    Format chunks as a readable diary-extract block, trimmed to max_chars.
+    """
+    lines = []
+    current_file = None
+    char_count = 0
+
+    for c in chunks:
+        if c["filename"] != current_file:
+            current_file = c["filename"]
+            header = f"\n--- Source: {current_file} ---\n"
+            lines.append(header)
+            char_count += len(header)
+
+        entry = f"[page {c['page_num']}] {c['text']}\n\n"
+        if char_count + len(entry) > max_chars:
+            lines.append("\n[Context truncated — additional sources available in DB]\n")
+            break
+        lines.append(entry)
+        char_count += len(entry)
+
+    return "".join(lines)
+
+
+def build_customer_situation(name: str, unit: str, joined: str,
+                              wounded: str, notes: str) -> str:
+    knows = textwrap.dedent(f"""\
+        The family knows:
+        - Their relative, {name}, served with the {unit}.
+        - They joined the battalion around {joined}.
+        - They were wounded in action around {wounded}.
+        - They were evacuated and did not return to the unit.
+    """)
+
+    doesnt_know = textwrap.dedent("""\
+        The family does NOT know:
+        - The exact date they joined or were wounded.
+        - Specific actions they were personally involved in.
+        - Their company, platoon, or section.
+    """)
+
+    if notes:
+        knows += f"\nAdditional context provided by the family:\n{notes}\n"
+
+    return f"## The customer's situation\n{knows}\n{doesnt_know}"
+
+
+def load_prompt_template() -> str:
+    return PROMPT_FILE.read_text(encoding="utf-8")
+
+
+def assemble_full_prompt(template: str, customer_block: str,
+                          context_block: str) -> str:
+    """
+    Replace the customer situation section in the template with the real
+    soldier info, then append the diary context before the output instructions.
+    """
+    # Strip the hardcoded customer section and inject the real one
+    start_marker = "## The customer's situation"
+    end_marker   = "## What to produce"
+
+    if start_marker in template and end_marker in template:
+        before = template[:template.index(start_marker)]
+        after  = template[template.index(end_marker):]
+        template = before + customer_block + "\n\n" + after
+
+    diary_section = (
+        "## War diary source material\n"
+        "The following excerpts are drawn from the pgvector database of "
+        "OCR-processed war diaries. Use them as your primary source.\n\n"
+        + context_block
+        + "\n\n"
+    )
+
+    # Insert diary context just before "## What to produce"
+    if end_marker in template:
+        idx = template.index(end_marker)
+        template = template[:idx] + diary_section + template[idx:]
+    else:
+        template += "\n\n" + diary_section
+
+    return template
+
+
+def generate_report_with_claude(prompt: str, soldier_name: str) -> str:
+    import anthropic
+
+    client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
+
+    print("  Sending to Claude... (this may take 30–60 seconds)")
+    message = client.messages.create(
+        model="claude-sonnet-4-5",
+        max_tokens=4096,
+        messages=[
+            {
+                "role": "user",
+                "content": prompt,
+            }
+        ],
+    )
+    return message.content[0].text
+
+
+def save_report(text: str, soldier_name: str) -> Path:
+    REPORTS_DIR.mkdir(exist_ok=True)
+    slug = soldier_name.lower().replace(" ", "_").replace(".", "").replace(",", "")
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    out_path = REPORTS_DIR / f"{slug}_{timestamp}_report.md"
+    out_path.write_text(text, encoding="utf-8")
+    return out_path
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate a Tier-3 descendant report from war diary pgvector DB"
+    )
+    parser.add_argument("--name",    default="Pte. Bill Bloggins",
+                        help="Soldier's name (default: placeholder Bloggins)")
+    parser.add_argument("--unit",    default="Calgary Highlanders",
+                        help="Unit name")
+    parser.add_argument("--joined",  default="mid September 1944",
+                        help="Approximate join date (plain English)")
+    parser.add_argument("--wounded", default="late October 1944",
+                        help="Approximate wound/end date (plain English)")
+    parser.add_argument("--notes",   default="",
+                        help="Any extra family context (optional)")
+    parser.add_argument("--nationality", default="canadian",
+                        help="DB nationality filter (default: canadian)")
+    parser.add_argument("--corpus",      default="war_diary_narrative",
+                        help="DB corpus filter (default: war_diary_narrative)")
+    parser.add_argument("--top-k", type=int, default=5,
+                        help="Results per search query (default: 5)")
+    args = parser.parse_args()
+
+    print(f"\n=== Generating report for: {args.name} ===")
+    print(f"    Unit    : {args.unit}")
+    print(f"    Joined  : {args.joined}")
+    print(f"    Wounded : {args.wounded}")
+    if args.notes:
+        print(f"    Notes   : {args.notes}")
+    print()
+
+    # 1. Retrieve diary chunks from pgvector
+    print("[1/4] Retrieving diary chunks from pgvector DB...")
+    chunks = retrieve_diary_chunks(
+        nationality=args.nationality,
+        corpus=args.corpus,
+    )
+    if not chunks:
+        print("ERROR: No chunks found. Check DB connection and filters.")
+        sys.exit(1)
+
+    # 2. Build context block
+    print("[2/4] Assembling context...")
+    context_block = build_context_block(chunks)
+    print(f"      Context: {len(context_block):,} characters from {len(chunks)} chunks")
+
+    # 3. Assemble final prompt
+    print("[3/4] Building prompt from v2-tier3-report.md...")
+    template = load_prompt_template()
+    customer_block = build_customer_situation(
+        name=args.name, unit=args.unit,
+        joined=args.joined, wounded=args.wounded,
+        notes=args.notes,
+    )
+    full_prompt = assemble_full_prompt(template, customer_block, context_block)
+    print(f"      Total prompt: {len(full_prompt):,} characters")
+
+    # 4. Generate with Claude
+    print("[4/4] Generating report with Claude...")
+    report_text = generate_report_with_claude(full_prompt, args.name)
+
+    # 5. Save
+    out_path = save_report(report_text, args.name)
+    print(f"\n✅ Report saved to: {out_path}")
+    print("-" * 60)
+    print(report_text[:500] + "...")
+
+
+if __name__ == "__main__":
+    main()
+