updated ocr viewer, built and tested ai workflows and OCR.
Built out use cases Built out googledocumentOCR and a semantic search webpage
This commit is contained in:
nathan
473
GoogleDocumentOCR/web_app.py
Normal file
473
GoogleDocumentOCR/web_app.py
Normal file
@@ -0,0 +1,473 @@
|
||||
"""
|
||||
web_app.py — Terminal-style search UI
|
||||
Run: python GoogleDocumentOCR/web_app.py
|
||||
Then open: http://localhost:5000
|
||||
"""
|
||||
|
||||
import os, sys
|
||||
from pathlib import Path
|
||||
from flask import Flask, request, jsonify, render_template_string
|
||||
from dotenv import load_dotenv
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
load_dotenv()
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# ── HTML template ──────────────────────────────────────────────────────────────
|
||||
HTML = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>WAR DIARY SEARCH // HISTORICAL RECORDS SYSTEM</title>
|
||||
<style>
|
||||
:root {
|
||||
--green: #00ff41;
|
||||
--dimgreen: #00a82b;
|
||||
--amber: #ffb000;
|
||||
--red: #ff4444;
|
||||
--bg: #0a0a0a;
|
||||
--bg2: #0f0f0f;
|
||||
--border: #1a3a1a;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
body {
|
||||
background: var(--bg);
|
||||
color: var(--green);
|
||||
font-family: 'Courier New', Courier, monospace;
|
||||
font-size: 14px;
|
||||
height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* ── Header ── */
|
||||
#header {
|
||||
border-bottom: 1px solid var(--border);
|
||||
padding: 10px 16px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
flex-shrink: 0;
|
||||
background: var(--bg2);
|
||||
}
|
||||
#header .title {
|
||||
color: var(--amber);
|
||||
font-size: 13px;
|
||||
letter-spacing: 2px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
#header .status {
|
||||
font-size: 11px;
|
||||
color: var(--dimgreen);
|
||||
}
|
||||
#header .status span { color: var(--green); }
|
||||
|
||||
/* ── Filters bar ── */
|
||||
#filters {
|
||||
padding: 6px 16px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
display: flex;
|
||||
gap: 20px;
|
||||
align-items: center;
|
||||
font-size: 12px;
|
||||
color: var(--dimgreen);
|
||||
flex-shrink: 0;
|
||||
background: var(--bg2);
|
||||
}
|
||||
#filters label { color: var(--dimgreen); }
|
||||
#filters select {
|
||||
background: #000;
|
||||
color: var(--green);
|
||||
border: 1px solid var(--border);
|
||||
font-family: inherit;
|
||||
font-size: 12px;
|
||||
padding: 2px 6px;
|
||||
outline: none;
|
||||
cursor: pointer;
|
||||
}
|
||||
#filters select:focus { border-color: var(--green); }
|
||||
|
||||
/* ── Chat window ── */
|
||||
#chat {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 18px;
|
||||
scroll-behavior: smooth;
|
||||
}
|
||||
#chat::-webkit-scrollbar { width: 6px; }
|
||||
#chat::-webkit-scrollbar-track { background: #000; }
|
||||
#chat::-webkit-scrollbar-thumb { background: var(--border); }
|
||||
|
||||
/* ── Boot message ── */
|
||||
.boot {
|
||||
color: var(--dimgreen);
|
||||
font-size: 12px;
|
||||
line-height: 1.8;
|
||||
border-bottom: 1px solid var(--border);
|
||||
padding-bottom: 14px;
|
||||
}
|
||||
.boot .hi { color: var(--amber); }
|
||||
|
||||
/* ── Exchange (Q+A pair) ── */
|
||||
.exchange {}
|
||||
|
||||
.query-line {
|
||||
color: var(--amber);
|
||||
margin-bottom: 8px;
|
||||
word-break: break-word;
|
||||
}
|
||||
.query-line::before { content: '> '; color: var(--dimgreen); }
|
||||
|
||||
.answer-block {
|
||||
color: var(--green);
|
||||
line-height: 1.7;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
padding-left: 14px;
|
||||
border-left: 2px solid var(--border);
|
||||
}
|
||||
|
||||
.sources-block {
|
||||
margin-top: 10px;
|
||||
padding-left: 14px;
|
||||
border-left: 2px solid var(--border);
|
||||
}
|
||||
.sources-label {
|
||||
color: var(--dimgreen);
|
||||
font-size: 11px;
|
||||
letter-spacing: 1px;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.source-row {
|
||||
font-size: 11px;
|
||||
color: #336633;
|
||||
line-height: 1.6;
|
||||
}
|
||||
.source-row .sim { color: var(--dimgreen); }
|
||||
|
||||
/* ── Thinking indicator ── */
|
||||
.thinking {
|
||||
color: var(--dimgreen);
|
||||
font-size: 12px;
|
||||
}
|
||||
.dot-anim::after {
|
||||
content: '';
|
||||
animation: dots 1.2s steps(4, end) infinite;
|
||||
}
|
||||
@keyframes dots {
|
||||
0% { content: ''; }
|
||||
25% { content: '.'; }
|
||||
50% { content: '..'; }
|
||||
75% { content: '...'; }
|
||||
100% { content: ''; }
|
||||
}
|
||||
|
||||
.error-line { color: var(--red); padding-left: 14px; }
|
||||
|
||||
/* ── Input bar ── */
|
||||
#inputbar {
|
||||
border-top: 1px solid var(--border);
|
||||
padding: 12px 16px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-shrink: 0;
|
||||
background: var(--bg2);
|
||||
}
|
||||
#prompt-symbol {
|
||||
color: var(--amber);
|
||||
font-size: 15px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
#question {
|
||||
flex: 1;
|
||||
background: transparent;
|
||||
border: none;
|
||||
outline: none;
|
||||
color: var(--green);
|
||||
font-family: inherit;
|
||||
font-size: 14px;
|
||||
caret-color: var(--green);
|
||||
}
|
||||
#question::placeholder { color: #1a3a1a; }
|
||||
|
||||
#send-btn {
|
||||
background: transparent;
|
||||
border: 1px solid var(--border);
|
||||
color: var(--dimgreen);
|
||||
font-family: inherit;
|
||||
font-size: 12px;
|
||||
padding: 4px 10px;
|
||||
cursor: pointer;
|
||||
letter-spacing: 1px;
|
||||
transition: color 0.15s, border-color 0.15s;
|
||||
}
|
||||
#send-btn:hover:not(:disabled) {
|
||||
color: var(--green);
|
||||
border-color: var(--green);
|
||||
}
|
||||
#send-btn:disabled { opacity: 0.3; cursor: not-allowed; }
|
||||
|
||||
/* ── Scanline overlay ── */
|
||||
body::after {
|
||||
content: '';
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: repeating-linear-gradient(
|
||||
0deg,
|
||||
transparent,
|
||||
transparent 2px,
|
||||
rgba(0,0,0,0.07) 2px,
|
||||
rgba(0,0,0,0.07) 4px
|
||||
);
|
||||
pointer-events: none;
|
||||
z-index: 999;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div id="header">
|
||||
<div class="title">■ WAR DIARY SEARCH // HISTORICAL RECORDS SYSTEM</div>
|
||||
<div class="status">DB STATUS: <span>ONLINE</span> | CORPUS: <span>CANADIAN WWII</span></div>
|
||||
</div>
|
||||
|
||||
<div id="filters">
|
||||
<span>FILTERS:</span>
|
||||
<label>NATIONALITY
|
||||
<select id="nationality">
|
||||
<option value="">ALL</option>
|
||||
<option value="canadian" selected>CANADIAN</option>
|
||||
<option value="german">GERMAN</option>
|
||||
<option value="unknown">UNKNOWN</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>CORPUS
|
||||
<select id="corpus">
|
||||
<option value="">ALL</option>
|
||||
<option value="war_diary_narrative" selected>WAR DIARY NARRATIVE</option>
|
||||
<option value="war_diary_appendix">WAR DIARY APPENDIX</option>
|
||||
<option value="german_records">GERMAN RECORDS</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>RESULTS
|
||||
<select id="top_k">
|
||||
<option value="3">3</option>
|
||||
<option value="5" selected>5</option>
|
||||
<option value="10">10</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div id="chat">
|
||||
<div class="boot">
|
||||
<div class="hi">HISTORICAL RECORDS RETRIEVAL SYSTEM v1.0</div>
|
||||
<div>Corpus: Calgary Highlanders · 5 CIB · RHC Black Watch · Sep–Oct 1944</div>
|
||||
<div>Embedding model: BAAI/bge-base-en-v1.5 | LLM: Meta-Llama-3.1-8B</div>
|
||||
<div> </div>
|
||||
<div>Type a question and press ENTER or [SEND].</div>
|
||||
<div>Example: <span style="color:var(--green)">What was the Calgary Highlanders doing on October 23, 1944?</span></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="inputbar">
|
||||
<span id="prompt-symbol">></span>
|
||||
<input id="question" type="text"
|
||||
placeholder="Ask about the war diaries..."
|
||||
autocomplete="off" spellcheck="false" autofocus />
|
||||
<button id="send-btn">SEND</button>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const chatEl = document.getElementById('chat');
|
||||
const inputEl = document.getElementById('question');
|
||||
const sendBtn = document.getElementById('send-btn');
|
||||
|
||||
function escHtml(s) {
|
||||
return s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>');
|
||||
}
|
||||
|
||||
function scrollBottom() {
|
||||
chatEl.scrollTop = chatEl.scrollHeight;
|
||||
}
|
||||
|
||||
function addExchange(question, answer, sources, isError) {
|
||||
const div = document.createElement('div');
|
||||
div.className = 'exchange';
|
||||
|
||||
const qLine = document.createElement('div');
|
||||
qLine.className = 'query-line';
|
||||
qLine.textContent = question;
|
||||
div.appendChild(qLine);
|
||||
|
||||
if (isError) {
|
||||
const err = document.createElement('div');
|
||||
err.className = 'error-line';
|
||||
err.textContent = 'ERROR: ' + answer;
|
||||
div.appendChild(err);
|
||||
} else {
|
||||
const ans = document.createElement('div');
|
||||
ans.className = 'answer-block';
|
||||
ans.textContent = answer;
|
||||
div.appendChild(ans);
|
||||
|
||||
if (sources && sources.length) {
|
||||
const sb = document.createElement('div');
|
||||
sb.className = 'sources-block';
|
||||
const lbl = document.createElement('div');
|
||||
lbl.className = 'sources-label';
|
||||
lbl.textContent = '── SOURCES (' + sources.length + ') ──────────────';
|
||||
sb.appendChild(lbl);
|
||||
sources.forEach((s, i) => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'source-row';
|
||||
const simPct = (s.similarity * 100).toFixed(1);
|
||||
row.innerHTML =
|
||||
'[' + (i+1) + '] ' + escHtml(s.filename) +
|
||||
' pg.' + s.page_num +
|
||||
' <span class="sim">sim:' + simPct + '%</span>' +
|
||||
(s.corpus ? ' [' + escHtml(s.corpus) + ']' : '');
|
||||
sb.appendChild(row);
|
||||
});
|
||||
div.appendChild(sb);
|
||||
}
|
||||
}
|
||||
|
||||
chatEl.appendChild(div);
|
||||
scrollBottom();
|
||||
}
|
||||
|
||||
async function ask() {
|
||||
const question = inputEl.value.trim();
|
||||
if (!question) return;
|
||||
|
||||
const nationality = document.getElementById('nationality').value;
|
||||
const corpus = document.getElementById('corpus').value;
|
||||
const top_k = parseInt(document.getElementById('top_k').value);
|
||||
|
||||
inputEl.value = '';
|
||||
sendBtn.disabled = true;
|
||||
inputEl.disabled = true;
|
||||
|
||||
// Thinking indicator
|
||||
const thinking = document.createElement('div');
|
||||
thinking.className = 'thinking dot-anim';
|
||||
thinking.textContent = 'SEARCHING';
|
||||
chatEl.appendChild(thinking);
|
||||
scrollBottom();
|
||||
|
||||
try {
|
||||
const res = await fetch('/ask', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ question, nationality, corpus, top_k }),
|
||||
});
|
||||
const data = await res.json();
|
||||
chatEl.removeChild(thinking);
|
||||
|
||||
if (data.error) {
|
||||
addExchange(question, data.error, null, true);
|
||||
} else {
|
||||
addExchange(question, data.answer, data.sources, false);
|
||||
}
|
||||
} catch (e) {
|
||||
chatEl.removeChild(thinking);
|
||||
addExchange(question, e.message, null, true);
|
||||
}
|
||||
|
||||
sendBtn.disabled = false;
|
||||
inputEl.disabled = false;
|
||||
inputEl.focus();
|
||||
}
|
||||
|
||||
sendBtn.addEventListener('click', ask);
|
||||
inputEl.addEventListener('keydown', e => { if (e.key === 'Enter') ask(); });
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# ── API endpoint ───────────────────────────────────────────────────────────────
|
||||
@app.route("/")
|
||||
def index():
|
||||
return render_template_string(HTML)
|
||||
|
||||
|
||||
@app.route("/ask", methods=["POST"])
|
||||
def ask():
|
||||
from embed_and_store import embed_texts
|
||||
from db import semantic_search
|
||||
from openai import OpenAI
|
||||
|
||||
data = request.get_json(force=True)
|
||||
question = (data.get("question") or "").strip()
|
||||
nationality = data.get("nationality") or None
|
||||
corpus = data.get("corpus") or None
|
||||
top_k = int(data.get("top_k", 5))
|
||||
|
||||
if not question:
|
||||
return jsonify({"error": "No question provided."}), 400
|
||||
|
||||
try:
|
||||
# 1. Embed + search
|
||||
embedding = embed_texts([question])[0]
|
||||
rows = semantic_search(embedding, top_k=top_k,
|
||||
nationality=nationality, corpus=corpus)
|
||||
|
||||
if not rows:
|
||||
return jsonify({
|
||||
"answer": "No relevant records found for that query with the current filters.",
|
||||
"sources": []
|
||||
})
|
||||
|
||||
# 2. Build context
|
||||
context = ""
|
||||
sources = []
|
||||
for filename, page_num, chunk_index, text, nat, corp, similarity in rows:
|
||||
context += f"[Source: {filename}, page {page_num}]\n{text}\n\n"
|
||||
sources.append({
|
||||
"filename": filename,
|
||||
"page_num": page_num,
|
||||
"similarity": round(similarity, 4),
|
||||
"corpus": corp,
|
||||
})
|
||||
|
||||
# 3. LLM answer
|
||||
client = OpenAI(
|
||||
api_key=os.getenv("DEEPINFRA_API_KEY"),
|
||||
base_url="https://api.deepinfra.com/v1/openai",
|
||||
)
|
||||
|
||||
prompt = (
|
||||
"You are a WWII military historian. Answer the question using only "
|
||||
"the provided source documents. Cite source file and page number "
|
||||
"for each claim.\n\n"
|
||||
f"QUESTION: {question}\n\n"
|
||||
f"SOURCE DOCUMENTS:\n{context}\n\nAnswer:"
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="meta-llama/Meta-Llama-3.1-8B-Instruct",
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
max_tokens=1200,
|
||||
)
|
||||
|
||||
answer = response.choices[0].message.content.strip()
|
||||
return jsonify({"answer": answer, "sources": sources})
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n WAR DIARY SEARCH — http://localhost:5000\n")
|
||||
app.run(debug=False, port=5000)
|
||||
|
||||
Reference in New Issue
Block a user