updated ocr viewer, built and tested ai workflows and OCR.

Built out use cases

Built out googledocumentOCR and a semantic search webpage
This commit is contained in:
nathan
2026-05-19 18:01:49 -04:00
parent 7e651953bc
commit 92071d6489
11 changed files with 2298 additions and 14 deletions

View File

@@ -0,0 +1,297 @@
"""
step5_embed_wardiary.py
───────────────────────
Reads step3/step4 JSON output files from a TestRun (or any outputs folder) and
embeds them into the pgvector document_chunks table.
Sources ingested:
*_narratives.json → chunk_type=narrative (one chunk per diary entry)
*_appx_records.json → chunk_type=appx_record (one chunk per document page)
Positions JSON (*_positions.json, *_appx_positions.json) are NOT embedded here —
they belong in a PostGIS geometry table, not a vector search table.
Usage:
python scripts/step5_embed_wardiary.py --input_dir TestRun/outputs --nationality canadian
python scripts/step5_embed_wardiary.py --input_dir TestRun/outputs --nationality canadian --dry_run
"""
import os
import sys
import json
import argparse
from pathlib import Path
from dotenv import load_dotenv
from openai import OpenAI
load_dotenv()
# ── Embedding config (same as GoogleDocumentOCR/embed_and_store.py) ──────────
EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5"
EMBEDDING_DIMS = 768
BATCH_SIZE = 100
client = OpenAI(
api_key=os.getenv("DEEPINFRA_API_KEY"),
base_url="https://api.deepinfra.com/v1/openai",
)
# ── DB (reuse GoogleDocumentOCR/db.py) ───────────────────────────────────────
sys.path.insert(0, str(Path(__file__).parent.parent / "GoogleDocumentOCR"))
from db import init_db, get_conn
from psycopg2.extras import execute_values
# ─────────────────────────────────────────────────────────────────────────────
# Embedding
# ─────────────────────────────────────────────────────────────────────────────
def embed_texts(texts: list[str]) -> list[list[float]]:
all_embeddings = []
for i in range(0, len(texts), BATCH_SIZE):
batch = texts[i:i + BATCH_SIZE]
response = client.embeddings.create(input=batch, model=EMBEDDING_MODEL)
all_embeddings.extend([item.embedding for item in response.data])
return all_embeddings
# ─────────────────────────────────────────────────────────────────────────────
# DB insert (extended schema with chunk_type, date_entry, unit)
# ─────────────────────────────────────────────────────────────────────────────
def init_extended_schema():
"""Add chunk_type, date_entry, unit columns if they don't exist yet."""
conn = get_conn()
cur = conn.cursor()
for col, typedef in [
("chunk_type", "TEXT DEFAULT 'unknown'"),
("date_entry", "TEXT"),
("unit", "TEXT"),
]:
cur.execute(f"ALTER TABLE document_chunks ADD COLUMN IF NOT EXISTS {col} {typedef};")
conn.commit()
cur.close()
conn.close()
def upsert_rows(rows: list[dict]):
conn = get_conn()
cur = conn.cursor()
execute_values(
cur,
"""
INSERT INTO document_chunks
(filename, page_num, chunk_index, text, embedding,
nationality, corpus, chunk_type, date_entry, unit)
VALUES %s
""",
[
(
r["filename"], r["page_num"], r["chunk_index"], r["text"], r["embedding"],
r["nationality"], r["corpus"], r["chunk_type"],
r.get("date_entry"), r.get("unit"),
)
for r in rows
],
template="(%s, %s, %s, %s, %s::vector, %s, %s, %s, %s, %s)"
)
conn.commit()
cur.close()
conn.close()
# ─────────────────────────────────────────────────────────────────────────────
# Text builders — turn each JSON record into a single embeddable string
# ─────────────────────────────────────────────────────────────────────────────
def narrative_to_text(entry: dict) -> str:
parts = []
if entry.get("date"):
parts.append(f"Date: {entry['date']}")
if entry.get("place"):
parts.append(f"Location: {entry['place']}")
if entry.get("weather"):
parts.append(f"Weather: {entry['weather']}")
if entry.get("summary"):
parts.append(entry["summary"])
return "\n".join(parts).strip()
def appx_record_to_text(record: dict) -> str:
parts = []
if record.get("page_type"):
parts.append(f"Document type: {record['page_type']}")
if record.get("document_id"):
parts.append(f"Document ID: {record['document_id']}")
if record.get("date"):
parts.append(f"Date: {record['date']}")
if record.get("time"):
parts.append(f"Time: {record['time']}")
if record.get("from_unit"):
parts.append(f"From: {record['from_unit']}")
if record.get("to_units"):
parts.append(f"To: {', '.join(record['to_units'])}")
if record.get("subject"):
parts.append(f"Subject: {record['subject']}")
if record.get("summary"):
parts.append(record["summary"])
if record.get("units_mentioned"):
parts.append(f"Units mentioned: {', '.join(record['units_mentioned'])}")
return "\n".join(parts).strip()
# ─────────────────────────────────────────────────────────────────────────────
# Ingest functions
# ─────────────────────────────────────────────────────────────────────────────
def ingest_narratives(json_path: Path, nationality: str, unit: str, dry_run: bool):
data = json.loads(json_path.read_text(encoding="utf-8"))
print(f" {json_path.name}: {len(data)} narrative entries")
chunks = []
for idx, entry in enumerate(data):
text = narrative_to_text(entry)
if not text:
continue
chunks.append({
"filename": json_path.name,
"page_num": entry.get("source_page"),
"chunk_index": idx,
"text": text,
"nationality": nationality,
"corpus": "war_diary_narrative",
"chunk_type": "narrative",
"date_entry": entry.get("date"),
"unit": unit,
})
if not chunks:
print(" No embeddable entries.")
return
if dry_run:
print(f" DRY RUN — would embed+store {len(chunks)} chunks")
print(f" Sample: {chunks[0]['text'][:120]}...")
return
print(f" Embedding {len(chunks)} chunks...")
embeddings = embed_texts([c["text"] for c in chunks])
for c, emb in zip(chunks, embeddings):
c["embedding"] = emb
upsert_rows(chunks)
print(f" ✓ Stored {len(chunks)} narrative chunks")
def ingest_appx_records(json_path: Path, nationality: str, unit: str, dry_run: bool):
data = json.loads(json_path.read_text(encoding="utf-8"))
print(f" {json_path.name}: {len(data)} appendix records")
chunks = []
for idx, record in enumerate(data):
text = appx_record_to_text(record)
if not text:
continue
chunks.append({
"filename": json_path.name,
"page_num": record.get("source_page"),
"chunk_index": idx,
"text": text,
"nationality": nationality,
"corpus": "war_diary_appendix",
"chunk_type": "appx_record",
"date_entry": record.get("date"),
"unit": unit,
})
if not chunks:
print(" No embeddable records.")
return
if dry_run:
print(f" DRY RUN — would embed+store {len(chunks)} chunks")
print(f" Sample: {chunks[0]['text'][:120]}...")
return
print(f" Embedding {len(chunks)} chunks...")
embeddings = embed_texts([c["text"] for c in chunks])
for c, emb in zip(chunks, embeddings):
c["embedding"] = emb
upsert_rows(chunks)
print(f" ✓ Stored {len(chunks)} appendix record chunks")
# ─────────────────────────────────────────────────────────────────────────────
# Unit name extraction from filename
# ─────────────────────────────────────────────────────────────────────────────
UNIT_PATTERNS = {
"calgary-highlanders": "Calgary Highlanders",
"calgary_highlanders": "Calgary Highlanders",
"blackwatch": "Black Watch (RHC)",
"black-watch": "Black Watch (RHC)",
"5cib": "5 Canadian Infantry Brigade",
}
def extract_unit(filename: str) -> str:
lower = filename.lower()
for key, name in UNIT_PATTERNS.items():
if key in lower:
return name
return "unknown"
# ─────────────────────────────────────────────────────────────────────────────
# Main
# ─────────────────────────────────────────────────────────────────────────────
def main():
parser = argparse.ArgumentParser(description="Embed war diary JSON outputs into pgvector")
parser.add_argument("--input_dir", required=True, help="Folder containing step3/step4 JSON files")
parser.add_argument("--nationality", default="canadian", help="Nationality tag (default: canadian)")
parser.add_argument("--dry_run", action="store_true", help="Print what would be done without writing to DB")
args = parser.parse_args()
input_dir = Path(args.input_dir)
if not input_dir.exists():
print(f"ERROR: input_dir not found: {input_dir}")
sys.exit(1)
if not args.dry_run:
init_db()
init_extended_schema()
# Find all JSON files that are narratives or appx_records (skip positions)
narrative_files = sorted(input_dir.glob("*_narratives.json"))
appx_record_files = sorted(input_dir.glob("*_appx_records.json"))
skipped = [f.name for f in input_dir.glob("*.json")
if f not in narrative_files + appx_record_files]
print(f"\n{'DRY RUN — ' if args.dry_run else ''}War Diary Embedding Ingest")
print(f"Input dir : {input_dir}")
print(f"Nationality: {args.nationality}")
print(f"Narratives : {len(narrative_files)} file(s)")
print(f"Appx records: {len(appx_record_files)} file(s)")
if skipped:
print(f"Skipping (positions/other): {', '.join(skipped)}\n")
for f in narrative_files:
unit = extract_unit(f.name)
print(f"\n[narrative] {f.name} -> unit: {unit}")
ingest_narratives(f, args.nationality, unit, args.dry_run)
for f in appx_record_files:
unit = extract_unit(f.name)
print(f"\n[appx_record] {f.name} -> unit: {unit}")
ingest_appx_records(f, args.nationality, unit, args.dry_run)
print("\nDone.")
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,285 @@
"""
step6_generate_report.py
─────────────────────────────────────────────────────────────────────────────
Generates a Tier-3 descendant report by combining:
1. Semantic search against the pgvector DB (war diary chunks)
2. The prompt template from prompts/v2-tier3-report.md
3. Claude (Anthropic) for the final report generation
Usage
─────
# Default — uses the Bloggins placeholder soldier
python scripts/step6_generate_report.py
# Real soldier
python scripts/step6_generate_report.py \
--name "Pte. John Smith" \
--unit "Calgary Highlanders" \
--joined "mid September 1944" \
--wounded "late October 1944" \
--notes "Family knows he was in D Company. Was evacuated to England."
Output
──────
reports/<soldier-slug>_<timestamp>_report.md
"""
import os
import sys
import argparse
import textwrap
from datetime import datetime
from pathlib import Path
from dotenv import load_dotenv
# ── Paths ─────────────────────────────────────────────────────────────────────
REPO_ROOT = Path(__file__).resolve().parent.parent
PROMPT_FILE = REPO_ROOT / "prompts" / "v2-tier3-report.md"
REPORTS_DIR = REPO_ROOT / "reports"
# ── Add GoogleDocumentOCR to path so we can import db + embed_and_store ───────
sys.path.insert(0, str(REPO_ROOT / "GoogleDocumentOCR"))
load_dotenv()
# ── Search queries to give broad coverage of the service window ────────────────
SEARCH_QUERIES = [
"Calgary Highlanders September 1944 operations movements France",
"Calgary Highlanders October 1944 Scheldt fighting Holland",
"Calgary Highlanders November 1944 casualties wounded Maas",
"battalion advance attack objective company platoon",
"reinforcements joined unit billets rest weather rations",
"parades training administrative daily routine",
"casualties killed wounded evacuated",
"officers commanding lieutenant colonel major company commander",
"civilian population towns villages liberated",
"enemy German positions shelling mortars small arms fire",
]
CHUNKS_PER_QUERY = 5 # results per query
MAX_CONTEXT_CHARS = 80_000 # ~20k tokens — safe Claude window
def retrieve_diary_chunks(nationality: str = "canadian",
corpus: str = "war_diary_narrative") -> list[dict]:
"""
Run multiple semantic searches and return a deduplicated, sorted list
of diary chunks.
"""
from embed_and_store import embed_texts
from db import semantic_search
seen_ids = set()
chunks = []
print(f" Running {len(SEARCH_QUERIES)} semantic searches...")
for query in SEARCH_QUERIES:
embedding = embed_texts([query])[0]
rows = semantic_search(
embedding,
top_k=CHUNKS_PER_QUERY,
nationality=nationality,
corpus=corpus,
)
for row in rows:
filename, page_num, chunk_index, text, nat, corp, similarity = row
key = (filename, page_num, chunk_index)
if key not in seen_ids:
seen_ids.add(key)
chunks.append({
"filename": filename,
"page_num": page_num,
"chunk_index": chunk_index,
"text": text,
"similarity": similarity,
})
# Sort by filename then page so the context reads chronologically
chunks.sort(key=lambda c: (c["filename"], c["page_num"], c["chunk_index"]))
print(f" Retrieved {len(chunks)} unique chunks after deduplication.")
return chunks
def build_context_block(chunks: list[dict], max_chars: int = MAX_CONTEXT_CHARS) -> str:
"""
Format chunks as a readable diary-extract block, trimmed to max_chars.
"""
lines = []
current_file = None
char_count = 0
for c in chunks:
if c["filename"] != current_file:
current_file = c["filename"]
header = f"\n--- Source: {current_file} ---\n"
lines.append(header)
char_count += len(header)
entry = f"[page {c['page_num']}] {c['text']}\n\n"
if char_count + len(entry) > max_chars:
lines.append("\n[Context truncated — additional sources available in DB]\n")
break
lines.append(entry)
char_count += len(entry)
return "".join(lines)
def build_customer_situation(name: str, unit: str, joined: str,
wounded: str, notes: str) -> str:
knows = textwrap.dedent(f"""\
The family knows:
- Their relative, {name}, served with the {unit}.
- They joined the battalion around {joined}.
- They were wounded in action around {wounded}.
- They were evacuated and did not return to the unit.
""")
doesnt_know = textwrap.dedent("""\
The family does NOT know:
- The exact date they joined or were wounded.
- Specific actions they were personally involved in.
- Their company, platoon, or section.
""")
if notes:
knows += f"\nAdditional context provided by the family:\n{notes}\n"
return f"## The customer's situation\n{knows}\n{doesnt_know}"
def load_prompt_template() -> str:
return PROMPT_FILE.read_text(encoding="utf-8")
def assemble_full_prompt(template: str, customer_block: str,
context_block: str) -> str:
"""
Replace the customer situation section in the template with the real
soldier info, then append the diary context before the output instructions.
"""
# Strip the hardcoded customer section and inject the real one
start_marker = "## The customer's situation"
end_marker = "## What to produce"
if start_marker in template and end_marker in template:
before = template[:template.index(start_marker)]
after = template[template.index(end_marker):]
template = before + customer_block + "\n\n" + after
diary_section = (
"## War diary source material\n"
"The following excerpts are drawn from the pgvector database of "
"OCR-processed war diaries. Use them as your primary source.\n\n"
+ context_block
+ "\n\n"
)
# Insert diary context just before "## What to produce"
if end_marker in template:
idx = template.index(end_marker)
template = template[:idx] + diary_section + template[idx:]
else:
template += "\n\n" + diary_section
return template
def generate_report_with_claude(prompt: str, soldier_name: str) -> str:
import anthropic
client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
print(" Sending to Claude... (this may take 3060 seconds)")
message = client.messages.create(
model="claude-sonnet-4-5",
max_tokens=4096,
messages=[
{
"role": "user",
"content": prompt,
}
],
)
return message.content[0].text
def save_report(text: str, soldier_name: str) -> Path:
REPORTS_DIR.mkdir(exist_ok=True)
slug = soldier_name.lower().replace(" ", "_").replace(".", "").replace(",", "")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
out_path = REPORTS_DIR / f"{slug}_{timestamp}_report.md"
out_path.write_text(text, encoding="utf-8")
return out_path
def main():
parser = argparse.ArgumentParser(
description="Generate a Tier-3 descendant report from war diary pgvector DB"
)
parser.add_argument("--name", default="Pte. Bill Bloggins",
help="Soldier's name (default: placeholder Bloggins)")
parser.add_argument("--unit", default="Calgary Highlanders",
help="Unit name")
parser.add_argument("--joined", default="mid September 1944",
help="Approximate join date (plain English)")
parser.add_argument("--wounded", default="late October 1944",
help="Approximate wound/end date (plain English)")
parser.add_argument("--notes", default="",
help="Any extra family context (optional)")
parser.add_argument("--nationality", default="canadian",
help="DB nationality filter (default: canadian)")
parser.add_argument("--corpus", default="war_diary_narrative",
help="DB corpus filter (default: war_diary_narrative)")
parser.add_argument("--top-k", type=int, default=5,
help="Results per search query (default: 5)")
args = parser.parse_args()
print(f"\n=== Generating report for: {args.name} ===")
print(f" Unit : {args.unit}")
print(f" Joined : {args.joined}")
print(f" Wounded : {args.wounded}")
if args.notes:
print(f" Notes : {args.notes}")
print()
# 1. Retrieve diary chunks from pgvector
print("[1/4] Retrieving diary chunks from pgvector DB...")
chunks = retrieve_diary_chunks(
nationality=args.nationality,
corpus=args.corpus,
)
if not chunks:
print("ERROR: No chunks found. Check DB connection and filters.")
sys.exit(1)
# 2. Build context block
print("[2/4] Assembling context...")
context_block = build_context_block(chunks)
print(f" Context: {len(context_block):,} characters from {len(chunks)} chunks")
# 3. Assemble final prompt
print("[3/4] Building prompt from v2-tier3-report.md...")
template = load_prompt_template()
customer_block = build_customer_situation(
name=args.name, unit=args.unit,
joined=args.joined, wounded=args.wounded,
notes=args.notes,
)
full_prompt = assemble_full_prompt(template, customer_block, context_block)
print(f" Total prompt: {len(full_prompt):,} characters")
# 4. Generate with Claude
print("[4/4] Generating report with Claude...")
report_text = generate_report_with_claude(full_prompt, args.name)
# 5. Save
out_path = save_report(report_text, args.name)
print(f"\n✅ Report saved to: {out_path}")
print("-" * 60)
print(report_text[:500] + "...")
if __name__ == "__main__":
main()