epub_corrector/epub-corrector-ui-app/app.py

#!/usr/bin/env python3
import json, subprocess, threading, uuid, shutil
from pathlib import Path
from flask import Flask, request, jsonify, send_from_directory

BASE=Path('/home/nvme/epub-gec-experiment')
INPUT=Path('/home/orangepi/epub-corrector/input')
RUNS=BASE/'runs'/'ui'
MEMORY_DIR=BASE/'review_memory'
MEMORY_FILE=MEMORY_DIR/'auto_memory.json'

RUNS.mkdir(parents=True, exist_ok=True)
MEMORY_DIR.mkdir(parents=True, exist_ok=True)


def empty_memory():
    return {
        'approved': [],
        'sentence_rewrites': [],
        'preserved': [],
        'rejected': [],
    }


def load_memory():
    if not MEMORY_FILE.exists():
        return empty_memory()

    data = json.loads(
        MEMORY_FILE.read_text(encoding='utf-8')
    )

    memory = empty_memory()

    for section in memory:
        memory[section] = data.get(section, [])

    return memory


def merge_memory(new_decisions):
    memory = load_memory()

    key_functions = {
        'approved': lambda item: (
            item.get('original', '').casefold(),
            item.get('replacement', '').casefold(),
        ),
        'sentence_rewrites': lambda item: (
            item.get('original_context', ''),
            item.get('rewritten_context', ''),
        ),
        'preserved': lambda item: (
            item.get('value', '').casefold()
        ),
        'rejected': lambda item: (
            item.get('original', '').casefold(),
            item.get('replacement', '').casefold(),
        ),
    }

    for section, key_function in key_functions.items():
        merged = {}

        for item in memory[section]:
            key = key_function(item)

            if key:
                merged[key] = item

        for item in new_decisions.get(section, []):
            key = key_function(item)

            if key:
                merged[key] = item

        memory[section] = list(merged.values())

    # Una successiva approvazione prevale
    # sull'eventuale rifiuto della stessa coppia.
    approved_pairs = {
        key_functions['approved'](item)
        for item in memory['approved']
    }

    memory['rejected'] = [
        item
        for item in memory['rejected']
        if key_functions['rejected'](item)
        not in approved_pairs
    ]

    temporary = MEMORY_FILE.with_suffix('.tmp')

    temporary.write_text(
        json.dumps(
            memory,
            ensure_ascii=False,
            indent=2,
        ),
        encoding='utf-8',
    )

    temporary.replace(MEMORY_FILE)

    return memory
app=Flask(__name__, static_folder='static')
jobs={}

def run_job(job_id, epub):
    job=jobs[job_id]
    report=RUNS/f'{job_id}.json'; log=RUNS/f'{job_id}.log'
    lt_url = "".join([
        "http:",
        "//",
        "127.0.0.1",
        ":8010",
        "/v2/check",
    ])

    cmd = [
        str(BASE / ".venv-rknn/bin/python"),
        str(BASE / "epub_npu_dry_run_grouped_v2.py"),
        str(epub),
        "--memory",
        str(MEMORY_FILE),
        "--threshold",
        "0.999",
        "--min-margin",
        "0.002",
        "--languagetool-url",
        lt_url,
        "--output",
        str(report),
    ]
    job.update(status='running',report=str(report),log=str(log),cmd=cmd)
    with log.open('w') as f:
        p=subprocess.Popen(cmd,stdout=f,stderr=subprocess.STDOUT,text=True)
        job['pid']=p.pid; rc=p.wait()
    job['status']='done' if rc==0 else 'failed'; job['returncode']=rc

@app.get('/')
def index(): return send_from_directory('static','index.html')

@app.get('/api/books')
def books(): return jsonify([{'name':p.name,'path':str(p)} for p in sorted(INPUT.glob('*.epub'))])

@app.post('/api/start')
def start():
    data=request.get_json(); epub=Path(data['path'])
    if epub.parent.resolve()!=INPUT.resolve() or not epub.exists(): return jsonify(error='EPUB non valido'),400
    jid=uuid.uuid4().hex[:10]; jobs[jid]={'id':jid,'book':str(epub),'status':'queued'}
    threading.Thread(target=run_job,args=(jid,epub),daemon=True).start(); return jsonify(jobs[jid])

@app.get('/api/job/<jid>')
def job(jid):
    j=jobs.get(jid)
    if not j:return jsonify(error='Job non trovato'),404
    if j.get('log') and Path(j['log']).exists(): j['tail']='\n'.join(Path(j['log']).read_text(errors='replace').splitlines()[-25:])
    return jsonify(j)

@app.get('/api/report/<jid>')
def report(jid):
    j=jobs.get(jid); p=Path(j.get('report','')) if j else None
    if not p or not p.exists(): return jsonify(error='Rapporto non pronto'),404
    data=json.loads(p.read_text()); items=[]
    for i,x in enumerate(data.get('groups',[])):
        if x.get('decision') in {'AUTO_APPLY','REVIEW_HIGH','REVIEW_AMBIGUOUS'} and x.get('best_proposal'):
            items.append({'id':i,'decision':x['decision'],'file':x['file'],'paragraph':x['paragraph'],'offset':x['offset'],'original':x['original'],'proposal':x['best_proposal'],'probability':x.get('best_probability'),'margin':x.get('margin'),'context':x['context']})
    return jsonify(items=items,summary=data.get('summary',{}))

@app.post('/api/apply/<jid>')
def apply(jid):
    j=jobs.get(jid)
    if not j:return jsonify(error='Job non trovato'),404
    decisions = request.get_json()

    manifest = RUNS / f'{jid}_approved.json'

    manifest.write_text(
        json.dumps(
            decisions,
            ensure_ascii=False,
            indent=2,
        ),
        encoding='utf-8',
    )

    memory = merge_memory(decisions)

    out = (
        BASE
        / 'output'
        / f"{Path(j['book']).stem}_ui_reviewed.epub"
    )

    rep = RUNS / f'{jid}_applied.json'
    cmd=[str(BASE/'.venv-rknn/bin/python'),str(BASE/'apply_approved_epub_changes.py'),j['book'],'--changes',str(manifest),'--output',str(out),'--report',str(rep)]
    r=subprocess.run(cmd,capture_output=True,text=True)
    if r.returncode:return jsonify(error=r.stdout+r.stderr),400
    return jsonify(
        output=str(out),
        report=str(rep),
        memory=str(MEMORY_FILE),
        memory_counts={
            key: len(value)
            for key, value in memory.items()
        },
        message=r.stdout,
    )

@app.post('/api/languagetool/<action>')
def lt(action):
    if action not in {'start','stop','restart'}:return jsonify(error='Azione non valida'),400
    r=subprocess.run(['sudo','docker',action,'languagetool-epub'],capture_output=True,text=True)
    return jsonify(ok=r.returncode==0,output=r.stdout+r.stderr)

app.run(host='0.0.0.0',port=8090,threaded=True)