feat: Wiederaufnahme aus working/, --check-config, feldtauglicher Updater (v0.6.0)

Datenverlust behoben:
- Nach hartem Stopp blieb das Original in working/ liegen und wurde nie
  wieder angefasst (_scan_existing sah nur incoming/). Es wird jetzt beim
  Start an Ort und Stelle wieder aufgegriffen, mit Kollisionsschutz gegen
  gleichnamige neue Scans; angefangene __ocr_-Fragmente werden geloescht.
- TimeoutStopSec 30 -> 300, damit laufendes OCR zu Ende laufen darf.

Config-Drift sichtbar gemacht:
- Neues --check-config (Exit 0 sauber / 1 Warnungen / 2 Fehler), das
  update.sh vor dem Neustart ueber alle Instanz-Configs laufen laesst.
- Warnungen fuer [ocr].timeout >= 900 (seit 0.4.0 pro SEITE) und gesetztes
  pdfa_level, beim Dienststart wie im Check.
- Unbekannte Config-Keys werden nicht mehr still verworfen, sondern genannt.

Updater feldtauglich:
- venv-Health-Check erkennt toten Symlink UND Versions-Drift gegen das
  System-Python; --rebuild-venv als ausdruecklicher Weg nach einem Debian-
  Major-Upgrade. Neubau ist ganz-oder-gar-nicht mit Rollback.
- apt-Pakete werden auch beim Update synchronisiert (Quelle: install.sh).
- Instanz-Erfassung inkl. activating/failed, Verifikation prueft is-failed
  und NRestarts statt sleep 1 + is-active.
- Backup enthaelt Configs, Unit, Drop-ins und pip-freeze.txt, liegt auf
  0600 und rotiert auf 5; schlaegt es fehl, bricht das Update vorher ab.
- ERR-Trap faehrt die vorher laufenden Instanzen wieder hoch.
- lxc-compat.conf wird beim Update nachgezogen.
- requirements.txt gepinnt (ocrmypdf 16.13.0, geprueft fuer Python 3.11+3.13).

Doku in Installation / Update / OS-Upgrade aufgeteilt (docs/).
135 Tests gruen.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-22 22:04:04 +02:00
parent 2062476252
commit 3e24aa2ecd
20 changed files with 3095 additions and 347 deletions
+165
View File
@@ -0,0 +1,165 @@
"""Tests für `--check-config`.
Die Exit-Codes werden vom Updater ausgewertet und müssen verlässlich sein:
0 = sauber, 1 = nur Warnungen, 2 = Fehler.
"""
from __future__ import annotations
import sys
from pathlib import Path
from unittest.mock import patch
import pytest
from pdf_ocr_hotfolder.__main__ import CHECK_ERROR, CHECK_OK, CHECK_WARN, main
def _cfg_file(tmp_path: Path, tmp_config, extra: str = "") -> Path:
cfg_file = tmp_path / "cfg.toml"
cfg_file.write_text(f"""
[paths]
incoming = "{tmp_config.paths.incoming}"
outgoing = "{tmp_config.paths.outgoing}"
working = "{tmp_config.paths.working}"
error = "{tmp_config.paths.error}"
""" + extra)
return cfg_file
def _check(monkeypatch, cfg_file: Path, binaries_present: bool = True) -> int:
monkeypatch.setattr(sys, "argv",
["pdf-ocr-hotfolder", "--config", str(cfg_file),
"--check-config"])
which = "/usr/bin/fake" if binaries_present else None
with patch("pdf_ocr_hotfolder.service.shutil.which", return_value=which):
return main()
# ---------------- Exit 0 ----------------
def test_clean_config_returns_0(tmp_path, tmp_config, monkeypatch, capsys) -> None:
rc = _check(monkeypatch, _cfg_file(tmp_path, tmp_config))
assert rc == CHECK_OK
out = capsys.readouterr().out
assert "Config sauber" in out
def test_check_does_not_process_files(tmp_path, tmp_config, monkeypatch) -> None:
"""Der Check darf nichts verarbeiten und nichts verschieben."""
pdf = tmp_config.paths.incoming / "scan.pdf"
pdf.write_bytes(b"%PDF-1.4\n")
assert _check(monkeypatch, _cfg_file(tmp_path, tmp_config)) == CHECK_OK
assert pdf.exists()
assert list(tmp_config.paths.outgoing.iterdir()) == []
# ---------------- Exit 1 (nur Warnungen) ----------------
def test_legacy_timeout_returns_1(tmp_path, tmp_config, monkeypatch, capsys) -> None:
rc = _check(monkeypatch, _cfg_file(tmp_path, tmp_config,
"\n[ocr]\ntimeout = 1800\n"))
assert rc == CHECK_WARN
out = capsys.readouterr().out
assert "WARNUNG" in out
assert "PRO SEITE" in out
def test_unknown_key_returns_1(tmp_path, tmp_config, monkeypatch, capsys) -> None:
rc = _check(monkeypatch, _cfg_file(tmp_path, tmp_config,
'\n[ocr]\nlangauges = "deu"\n'))
assert rc == CHECK_WARN
assert "[ocr].langauges" in capsys.readouterr().out
def test_pdfa_level_with_healthy_ghostscript_returns_1(
tmp_path, tmp_config, monkeypatch, capsys) -> None:
"""Gesundes Ghostscript: nur Hinweis (Exit 1), kein Abbruch."""
monkeypatch.setattr(sys, "argv",
["pdf-ocr-hotfolder", "--config",
str(_cfg_file(tmp_path, tmp_config,
'\n[ocr]\npdfa_level = "2"\n')),
"--check-config"])
with patch("pdf_ocr_hotfolder.service.shutil.which", return_value="/usr/bin/fake"), \
patch("pdf_ocr_hotfolder.service.detect_ghostscript_version",
return_value="10.02.1"):
rc = main()
assert rc == CHECK_WARN
assert "Ghostscript" in capsys.readouterr().out
# ---------------- Exit 2 (Fehler) ----------------
def test_missing_config_file_returns_2(tmp_path, monkeypatch, capsys) -> None:
monkeypatch.setattr(sys, "argv",
["pdf-ocr-hotfolder", "--config",
str(tmp_path / "gibtsnicht.toml"), "--check-config"])
assert main() == CHECK_ERROR
assert "nicht gefunden" in capsys.readouterr().err
def test_broken_paths_section_returns_2(tmp_path, monkeypatch, capsys) -> None:
cfg_file = tmp_path / "cfg.toml"
cfg_file.write_text('[ocr]\nlanguages = "deu"\n')
assert _check(monkeypatch, cfg_file) == CHECK_ERROR
assert "[paths]" in capsys.readouterr().err
def test_invalid_toml_returns_2(tmp_path, monkeypatch, capsys) -> None:
"""Kaputtes TOML: saubere Meldung statt Traceback."""
cfg_file = tmp_path / "cfg.toml"
cfg_file.write_text("[paths\nincoming = ")
assert _check(monkeypatch, cfg_file) == CHECK_ERROR
assert "TOML" in capsys.readouterr().err
def test_missing_binaries_return_2(tmp_path, tmp_config, monkeypatch, capsys) -> None:
rc = _check(monkeypatch, _cfg_file(tmp_path, tmp_config),
binaries_present=False)
assert rc == CHECK_ERROR
err = capsys.readouterr().err
assert "tesseract" in err
def test_invalid_name_mode_returns_2(tmp_path, tmp_config, monkeypatch, capsys) -> None:
rc = _check(monkeypatch, _cfg_file(tmp_path, tmp_config,
'\n[output]\nname_mode = "praefix"\n'))
assert rc == CHECK_ERROR
assert "name_mode" in capsys.readouterr().err
def test_archive_without_dir_returns_2(tmp_path, tmp_config, monkeypatch, capsys) -> None:
rc = _check(monkeypatch, _cfg_file(tmp_path, tmp_config,
'\n[output]\noriginal_on_success = "archive"\n'))
assert rc == CHECK_ERROR
assert "archive_dir" in capsys.readouterr().err
def test_error_beats_warning(tmp_path, tmp_config, monkeypatch) -> None:
"""Fehler + Warnung → Exit 2, nicht 1."""
rc = _check(monkeypatch,
_cfg_file(tmp_path, tmp_config,
'\n[ocr]\ntimeout = 1800\n\n[output]\nname_mode = "x"\n'))
assert rc == CHECK_ERROR
# ---------------- Zusammenspiel mit anderen Optionen ----------------
def test_check_config_wins_over_once(tmp_path, tmp_config, monkeypatch) -> None:
"""--check-config hat Vorrang: es wird nichts verarbeitet."""
pdf = tmp_config.paths.incoming / "scan.pdf"
pdf.write_bytes(b"%PDF-1.4\n")
monkeypatch.setattr(sys, "argv",
["pdf-ocr-hotfolder", "--config",
str(_cfg_file(tmp_path, tmp_config)),
"--once", "--check-config"])
with patch("pdf_ocr_hotfolder.service.shutil.which", return_value="/usr/bin/fake"):
assert main() == CHECK_OK
assert pdf.exists()
@pytest.mark.parametrize("code,expected", [(CHECK_OK, 0), (CHECK_WARN, 1),
(CHECK_ERROR, 2)])
def test_exit_code_constants(code: int, expected: int) -> None:
"""Die Konstanten sind Teil der Schnittstelle zum Updater."""
assert code == expected
+198
View File
@@ -0,0 +1,198 @@
"""Tests für Legacy-Warnungen und unbekannte Config-Keys.
Zwei stille Fallen:
- [ocr].timeout bedeutet seit 0.4.0 Sekunden pro SEITE (vorher Gesamtlauf)
- unbekannte Keys (Tippfehler!) wurden beim Laden stumm verworfen
"""
from __future__ import annotations
import logging
import sys
from pathlib import Path
from unittest.mock import patch
from pdf_ocr_hotfolder.config import (
config_warnings,
legacy_warnings,
load_config,
unknown_key_warnings,
)
_PATHS = """
[paths]
incoming = "/tmp/in"
outgoing = "/tmp/out"
working = "/tmp/work"
error = "/tmp/err"
"""
def _write(tmp_path: Path, extra: str = "") -> Path:
cfg = tmp_path / "config.toml"
cfg.write_text(_PATHS + extra)
return cfg
# ---------------- Legacy: [ocr].timeout ----------------
def test_legacy_timeout_warns(tmp_path: Path) -> None:
"""Ein Altwert (Gesamt-Timeout 1800) muss deutlich benannt werden."""
cfg = load_config(_write(tmp_path, "\n[ocr]\ntimeout = 1800\n"))
warnings = legacy_warnings(cfg)
assert len(warnings) == 1
assert "timeout" in warnings[0]
assert "PRO SEITE" in warnings[0]
assert "300" in warnings[0]
def test_timeout_at_threshold_warns(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path, "\n[ocr]\ntimeout = 900\n"))
assert legacy_warnings(cfg)
def test_sane_timeout_does_not_warn(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path, "\n[ocr]\ntimeout = 300\n"))
assert legacy_warnings(cfg) == []
def test_default_config_has_no_warnings(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path))
assert config_warnings(cfg) == []
# ---------------- Legacy: [ocr].pdfa_level ----------------
def test_pdfa_level_warns_about_ghostscript(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path, '\n[ocr]\npdfa_level = "2"\n'))
warnings = legacy_warnings(cfg)
assert len(warnings) == 1
assert "pdfa_level" in warnings[0]
assert "Ghostscript" in warnings[0]
assert "10.02.0" in warnings[0]
def test_empty_pdfa_level_does_not_warn(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path, '\n[ocr]\npdfa_level = ""\n'))
assert legacy_warnings(cfg) == []
def test_both_legacy_warnings_together(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path, '\n[ocr]\ntimeout = 1800\npdfa_level = "1"\n'))
assert len(legacy_warnings(cfg)) == 2
# ---------------- Unbekannte Keys ----------------
def test_typo_key_is_collected(tmp_path: Path) -> None:
"""`langauges` statt `languages` darf nicht mehr stumm verschwinden."""
cfg = load_config(_write(tmp_path, '\n[ocr]\nlangauges = "deu"\n'))
assert cfg.unknown_keys == ["[ocr].langauges"]
assert "[ocr].langauges" in unknown_key_warnings(cfg)[0]
# Der Rest wird weiterhin normal geladen
assert cfg.ocr.languages == "deu+eng"
def test_known_keys_are_not_reported(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path, '\n[ocr]\nlanguages = "deu"\njobs = 2\n'))
assert cfg.unknown_keys == []
assert cfg.ocr.languages == "deu"
def test_unknown_keys_in_all_sections(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path, """
[ocr]
foo = 1
[output]
bar = "x"
[verapdf]
baz = true
[upload.folder]
qux = ""
[upload.nextcloud]
quux = ""
[upload.sftp]
corge = 0
[notify.email]
grault = ""
[logging]
level = "INFO"
garply = 1
"""))
assert cfg.unknown_keys == [
"[ocr].foo", "[output].bar", "[verapdf].baz",
"[upload.folder].qux", "[upload.nextcloud].quux", "[upload.sftp].corge",
"[notify.email].grault", "[logging].garply",
]
def test_unknown_top_level_key_is_reported(tmp_path: Path) -> None:
"""Ein Key ausserhalb jeder Sektion (z.B. vergessene Sektionszeile)."""
cfg_file = tmp_path / "config.toml"
cfg_file.write_text('log_level = "DEBUG"\n' + _PATHS)
cfg = load_config(cfg_file)
assert cfg.unknown_keys == ["log_level"]
def test_unknown_section_is_reported(tmp_path: Path) -> None:
cfg = load_config(_write(tmp_path, '\n[ocrr]\nlanguages = "deu"\n\n[upload.ftp]\nhost = "x"\n'))
assert "[ocrr]" in cfg.unknown_keys
assert "[upload.ftp]" in cfg.unknown_keys
def test_unknown_key_inside_paths(tmp_path: Path) -> None:
"""Ein zusätzlicher Key direkt in [paths] wird ebenfalls gemeldet."""
cfg_file = tmp_path / "config.toml"
cfg_file.write_text(_PATHS + 'archive = "/tmp/a"\n')
cfg = load_config(cfg_file)
assert cfg.unknown_keys == ["[paths].archive"]
def test_load_config_works_without_logging(tmp_path: Path) -> None:
"""load_config() darf nichts loggen müssen — Tests rufen sie direkt auf."""
cfg_file = _write(tmp_path, '\n[ocr]\nlangauges = "deu"\n')
with patch("logging.Logger.warning") as warn:
cfg = load_config(cfg_file)
warn.assert_not_called()
assert cfg.unknown_keys
# ---------------- Warnungen beim Dienststart ----------------
def _argv(monkeypatch, cfg_file: Path, *extra: str) -> None:
monkeypatch.setattr(sys, "argv",
["pdf-ocr-hotfolder", "--config", str(cfg_file), *extra])
def test_warnings_are_logged_on_service_start(tmp_path: Path, tmp_config,
monkeypatch, caplog) -> None:
"""Beim normalen Start landen die Warnungen im Log (nicht nur im Check)."""
cfg_file = tmp_path / "cfg.toml"
cfg_file.write_text(f"""
[paths]
incoming = "{tmp_config.paths.incoming}"
outgoing = "{tmp_config.paths.outgoing}"
working = "{tmp_config.paths.working}"
error = "{tmp_config.paths.error}"
[ocr]
timeout = 1800
langauges = "deu"
""")
_argv(monkeypatch, cfg_file, "--once")
from pdf_ocr_hotfolder.__main__ import main
with caplog.at_level(logging.WARNING, logger="pdf_ocr_hotfolder.__main__"), \
patch("pdf_ocr_hotfolder.service.shutil.which", return_value="/usr/bin/fake"):
assert main() == 0
text = caplog.text
assert "PRO SEITE" in text
assert "[ocr].langauges" in text
+204
View File
@@ -0,0 +1,204 @@
"""Tests für die Wiederaufnahme abgebrochener Läufe aus working/.
Hintergrund: `process_pdf()` verschiebt das Original vor dem OCR nach
working/. Wird der Dienst dort hart gestoppt (SIGKILL nach TimeoutStopSec),
blieb die Datei bisher für immer liegen — weder outgoing/, noch error/, noch
eine Mail. ocrmypdf läuft in diesen Tests nie wirklich.
"""
from __future__ import annotations
import logging
from pathlib import Path
from unittest.mock import patch
from pdf_ocr_hotfolder.config import OcrConfig, OutputConfig, VeraPdfConfig
from pdf_ocr_hotfolder.processor import OCR_TEMP_PREFIX, ProcessResult, process_pdf
from pdf_ocr_hotfolder.service import HotfolderService
def _fake_success(src: Path, working_dir, outgoing_dir, error_dir, **kwargs):
"""Simuliert einen erfolgreichen Durchlauf inkl. Entsorgung des Originals."""
out = outgoing_dir / f"OCR_{src.name}"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_bytes(b"%PDF-1.4 ocr\n")
src.unlink(missing_ok=True)
return ProcessResult(src, out, True)
def _run_once(tmp_config, fake_process=_fake_success):
"""run_once() mit gemocktem Preflight und gemocktem process_pdf."""
seen: list[Path] = []
def spy(src, *args, **kwargs):
seen.append(src)
return fake_process(src, *args, **kwargs)
with patch("pdf_ocr_hotfolder.service.check_preflight", return_value=None), \
patch("pdf_ocr_hotfolder.service.process_pdf", side_effect=spy), \
patch("pdf_ocr_hotfolder.service._wait_until_stable", return_value=True):
service = HotfolderService(tmp_config)
try:
service.run_once()
finally:
service._executor.shutdown(wait=False)
return service, seen
# ---------------- Aufgreifen aus working/ ----------------
def test_leftover_in_working_is_picked_up(tmp_config) -> None:
"""Eine in working/ liegen gebliebene PDF wird wieder verarbeitet."""
leftover = tmp_config.paths.working / "abgebrochen.pdf"
leftover.write_bytes(b"%PDF-1.4\n")
service, seen = _run_once(tmp_config)
assert [p.name for p in seen] == ["abgebrochen.pdf"]
assert seen[0].parent == tmp_config.paths.working
assert service.success_count == 1
assert not leftover.exists()
assert (tmp_config.paths.outgoing / "OCR_abgebrochen.pdf").exists()
def test_resume_logs_warning(tmp_config, caplog) -> None:
"""Die Wiederaufnahme muss deutlich im Log stehen."""
(tmp_config.paths.working / "abgebrochen.pdf").write_bytes(b"%PDF-1.4\n")
with caplog.at_level(logging.WARNING, logger="pdf_ocr_hotfolder.service"):
_run_once(tmp_config)
text = caplog.text
assert "Abgebrochener Lauf wird fortgesetzt" in text
assert "abgebrochen.pdf" in text
def test_ocr_fragment_is_removed_and_not_processed(tmp_config, caplog) -> None:
"""__ocr_-Fragmente sind unbrauchbar: löschen, nicht als Eingabe nehmen."""
leftover = tmp_config.paths.working / "scan.pdf"
leftover.write_bytes(b"%PDF-1.4\n")
fragment = tmp_config.paths.working / f"{OCR_TEMP_PREFIX}OCR_scan.pdf"
fragment.write_bytes(b"%PDF-1.4 halbfertig\n")
with caplog.at_level(logging.WARNING, logger="pdf_ocr_hotfolder.service"):
service, seen = _run_once(tmp_config)
assert [p.name for p in seen] == ["scan.pdf"]
assert not fragment.exists()
assert service.error_count == 0
assert "Fragment" in caplog.text
def test_non_pdf_in_working_is_ignored(tmp_config) -> None:
"""Fremddateien in working/ werden nicht angefasst."""
junk = tmp_config.paths.working / "notizen.txt"
junk.write_text("kein PDF")
_service, seen = _run_once(tmp_config)
assert seen == []
assert junk.exists()
def test_incoming_and_working_both_scanned(tmp_config) -> None:
"""incoming/ wird weiterhin gescannt — zusätzlich zu working/."""
(tmp_config.paths.incoming / "neu.pdf").write_bytes(b"%PDF-1.4\n")
(tmp_config.paths.working / "alt.pdf").write_bytes(b"%PDF-1.4\n")
service, seen = _run_once(tmp_config)
assert sorted(p.name for p in seen) == ["alt.pdf", "neu.pdf"]
assert service.success_count == 2
def test_name_collision_between_working_and_incoming(tmp_config, caplog) -> None:
"""Gleicher Name in beiden Ordnern: die working-Datei wird umbenannt.
Sonst würden sich beide dieselbe working- und dieselbe outgoing-Datei
teilen und eine der beiden ginge verloren.
"""
(tmp_config.paths.incoming / "scan.pdf").write_bytes(b"%PDF-1.4 neu\n")
(tmp_config.paths.working / "scan.pdf").write_bytes(b"%PDF-1.4 alt\n")
with caplog.at_level(logging.WARNING, logger="pdf_ocr_hotfolder.service"):
service, seen = _run_once(tmp_config)
names = sorted(p.name for p in seen)
assert len(names) == 2
assert "scan.pdf" in names
# Die wiederaufgenommene Datei hat einen Zeitstempel bekommen
renamed = [n for n in names if n != "scan.pdf"][0]
assert renamed.startswith("scan_") and renamed.endswith(".pdf")
assert service.success_count == 2
assert "umbenannt" in caplog.text
# ---------------- process_pdf: kein zweiter Move ----------------
def _ocr_ok(src: Path, dst: Path, cfg) -> None:
dst.write_bytes(b"%PDF-1.4 ocr\n")
def test_process_pdf_resumes_without_second_move(tmp_config) -> None:
"""Eine Datei aus working/ darf nicht erneut nach working/ verschoben werden."""
src = tmp_config.paths.working / "scan.pdf"
src.write_bytes(b"%PDF-1.4\n")
with patch("pdf_ocr_hotfolder.processor.run_ocr", side_effect=_ocr_ok):
result = process_pdf(
src=src,
working_dir=tmp_config.paths.working,
outgoing_dir=tmp_config.paths.outgoing,
error_dir=tmp_config.paths.error,
ocr_cfg=OcrConfig(),
vera_cfg=VeraPdfConfig(enabled=False),
output_cfg=OutputConfig(),
)
assert result.success
assert (tmp_config.paths.outgoing / "OCR_scan.pdf").exists()
# Original entsorgt, keine Reste in working/
assert list(tmp_config.paths.working.iterdir()) == []
def test_process_pdf_resume_logs_warning(tmp_config, caplog) -> None:
src = tmp_config.paths.working / "scan.pdf"
src.write_bytes(b"%PDF-1.4\n")
with caplog.at_level(logging.WARNING, logger="pdf_ocr_hotfolder.processor"), \
patch("pdf_ocr_hotfolder.processor.run_ocr", side_effect=_ocr_ok):
process_pdf(
src=src,
working_dir=tmp_config.paths.working,
outgoing_dir=tmp_config.paths.outgoing,
error_dir=tmp_config.paths.error,
ocr_cfg=OcrConfig(),
vera_cfg=VeraPdfConfig(enabled=False),
output_cfg=OutputConfig(),
)
assert "Wiederaufnahme" in caplog.text
def test_process_pdf_refuses_to_overwrite_working_file(tmp_config) -> None:
"""Belegter Name in working/: lieber Fehler als stilles Überschreiben."""
busy = tmp_config.paths.working / "scan.pdf"
busy.write_bytes(b"%PDF-1.4 laeuft gerade\n")
src = tmp_config.paths.incoming / "scan.pdf"
src.write_bytes(b"%PDF-1.4 neu\n")
with patch("pdf_ocr_hotfolder.processor.run_ocr", side_effect=_ocr_ok):
result = process_pdf(
src=src,
working_dir=tmp_config.paths.working,
outgoing_dir=tmp_config.paths.outgoing,
error_dir=tmp_config.paths.error,
ocr_cfg=OcrConfig(),
vera_cfg=VeraPdfConfig(enabled=False),
output_cfg=OutputConfig(),
)
assert not result.success
assert "scan.pdf" in result.error
# Beide Dateien unangetastet
assert busy.read_bytes() == b"%PDF-1.4 laeuft gerade\n"
assert src.read_bytes() == b"%PDF-1.4 neu\n"