You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
208 lines
8.7 KiB
208 lines
8.7 KiB
"""Local, explicit review workflow. Financial details never go to stdout.""" |
|
import json |
|
import re |
|
from datetime import date |
|
from pathlib import Path |
|
from uuid import UUID |
|
|
|
from django.conf import settings |
|
from django.core.management.base import BaseCommand, CommandError |
|
|
|
from finance.services import Actor, DomainError |
|
from ingestion import services |
|
|
|
|
|
MAX_BYTES = 10 * 1024 * 1024 |
|
PROFILES = ("BANK_CSV_V1", "OUROCARD_FULL_V1", "OUROCARD_PARTIAL_V1") |
|
|
|
|
|
def local_path(value): |
|
root = Path(getattr(settings, "IMPORT_ROOT", "/app/imports")).resolve() |
|
requested = Path(value) |
|
if ".." in requested.parts: |
|
raise CommandError("INVALID_PATH") |
|
resolved = (root / requested).resolve() |
|
if not resolved.is_relative_to(root) or resolved == root: |
|
raise CommandError("INVALID_PATH") |
|
return resolved |
|
|
|
|
|
def read_bytes(value): |
|
path = local_path(value) |
|
if not path.is_file(): |
|
raise CommandError("INVALID_INPUT") |
|
with path.open("rb") as stream: |
|
data = stream.read(MAX_BYTES + 1) |
|
if len(data) > MAX_BYTES: |
|
raise CommandError("INPUT_TOO_LARGE") |
|
return data |
|
|
|
|
|
def unique_object(pairs): |
|
result = {} |
|
for key, value in pairs: |
|
if key in result: |
|
raise ValueError("duplicate key") |
|
result[key] = value |
|
return result |
|
|
|
|
|
def reject_constant(value): |
|
raise ValueError("nonfinite JSON number") |
|
|
|
|
|
def read_json(value): |
|
data = read_bytes(value) |
|
try: |
|
return json.loads(data.decode("utf-8-sig"), object_pairs_hook=unique_object, |
|
parse_constant=reject_constant) |
|
except (ValueError, RecursionError): |
|
raise CommandError("INVALID_JSON") from None |
|
|
|
|
|
def identifier(value): |
|
try: |
|
return str(UUID(str(value))) |
|
except (ValueError, AttributeError, TypeError): |
|
raise CommandError("INVALID_IDENTIFIER") from None |
|
|
|
|
|
def revision_number(value): |
|
if type(value) is not int or value < 1: |
|
raise CommandError("INVALID_REVISION") |
|
return value |
|
|
|
|
|
def checked_decisions(payload, batch_id, actor): |
|
minimum = {"batch_id", "revision", "decisions"} |
|
if not isinstance(payload, dict) or not minimum.issubset(payload): |
|
raise CommandError("INVALID_REVIEW") |
|
if identifier(payload["batch_id"]) != batch_id: |
|
raise CommandError("INVALID_REVIEW") |
|
revision_number(payload["revision"]) |
|
if not isinstance(payload["decisions"], list): |
|
raise CommandError("INVALID_REVIEW") |
|
locators = set() |
|
for decision in payload["decisions"]: |
|
if not isinstance(decision, dict) or not isinstance(decision.get("locator"), str): |
|
raise CommandError("INVALID_REVIEW") |
|
locator = decision["locator"] |
|
if locator in locators: |
|
raise CommandError("INVALID_REVIEW") |
|
locators.add(locator) |
|
if set(payload) != minimum: |
|
current = services.export_review(batch_id=batch_id) |
|
if set(payload) != set(current): |
|
raise CommandError("INVALID_REVIEW") |
|
# All exported fields except the dedicated decisions array are evidence. |
|
# JSON encoding also distinguishes true/1 and numeric/string edits. |
|
for key in current: |
|
if key != "decisions" and json.dumps(payload[key], sort_keys=True) != json.dumps(current[key], sort_keys=True): |
|
raise CommandError("READ_ONLY_REVIEW_CHANGED") |
|
return payload["revision"], payload["decisions"] |
|
|
|
|
|
class Command(BaseCommand): |
|
help = "Preview, export, apply, refresh, commit or cancel a local reviewed import." |
|
requires_system_checks = [] |
|
|
|
def add_arguments(self, parser): |
|
commands = parser.add_subparsers(dest="operation", required=True) |
|
for operation in ("preview", "export", "apply", "refresh", "commit", "cancel"): |
|
sub = commands.add_parser(operation) |
|
sub.add_argument("--actor-kind", required=True, choices=("HUMAN", "APPLICATION", "AGENT")) |
|
sub.add_argument("--actor-identifier", required=True) |
|
if operation == "preview": |
|
sub.add_argument("--input", required=True) |
|
sub.add_argument("--mapping", required=True) |
|
sub.add_argument("--profile", required=True, choices=PROFILES) |
|
sub.add_argument("--context-start") |
|
sub.add_argument("--context-end") |
|
else: |
|
sub.add_argument("--batch", required=True) |
|
if operation == "export": |
|
sub.add_argument("--output", required=True) |
|
if operation == "apply": |
|
sub.add_argument("--input", required=True) |
|
if operation in {"refresh", "commit", "cancel"}: |
|
sub.add_argument("--revision", required=True, type=int) |
|
|
|
def handle(self, *args, **options): |
|
try: |
|
self.perform(options) |
|
except CommandError: |
|
raise |
|
except DomainError as exc: |
|
code = exc.code |
|
if not isinstance(code, str) or re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,79}", code) is None: |
|
code = "IMPORT_FAILED" |
|
raise CommandError(code) from None |
|
except Exception: |
|
# Source paths, source text and database errors are local financial data. |
|
raise CommandError("IMPORT_FAILED") from None |
|
|
|
def perform(self, options): |
|
actor = Actor(options["actor_kind"], options["actor_identifier"]) |
|
if actor.kind not in {"HUMAN", "APPLICATION", "AGENT"} or not actor.identifier.strip() or len(actor.identifier) > 200: |
|
raise CommandError("INVALID_ACTOR") |
|
operation = options["operation"] |
|
if operation == "preview": |
|
mapping = read_json(options["mapping"]) |
|
if not isinstance(mapping, dict) or not mapping: |
|
raise CommandError("INVALID_MAPPING") |
|
for section, account in mapping.items(): |
|
if re.fullmatch(r"[1-9][0-9]*", section) is None: |
|
raise CommandError("INVALID_MAPPING") |
|
if account is not None: |
|
if not isinstance(account, str): |
|
raise CommandError("INVALID_MAPPING") |
|
identifier(account) |
|
start, end = options.get("context_start"), options.get("context_end") |
|
context = None |
|
if start or end: |
|
try: |
|
first, last = date.fromisoformat(start), date.fromisoformat(end) |
|
if not 0 <= (last - first).days <= 365: |
|
raise ValueError |
|
except (TypeError, ValueError): |
|
raise CommandError("INVALID_CONTEXT") from None |
|
context = {"start_date": first.isoformat(), "end_date": last.isoformat()} |
|
batch = services.create_preview( |
|
data=read_bytes(options["input"]), |
|
profile=options["profile"], |
|
section_mapping=mapping, |
|
context=context, |
|
actor=actor, |
|
) |
|
else: |
|
batch_id = identifier(options["batch"]) |
|
if operation == "export": |
|
path = local_path(options["output"]) |
|
review = services.export_review(batch_id=batch_id) |
|
encoded = json.dumps(review, ensure_ascii=False, indent=2, allow_nan=False) + "\n" |
|
# Exclusive creation protects source documents and previous reviews. |
|
with path.open("x", encoding="utf-8") as stream: |
|
stream.write(encoded) |
|
self.stdout.write(json.dumps({"batch_id": batch_id, "state": review["state"], |
|
"revision": review["revision"], "item_count": len(review["items"])})) |
|
return |
|
if operation == "apply": |
|
revision, decisions = checked_decisions(read_json(options["input"]), batch_id, actor) |
|
batch = services.apply_decisions( |
|
batch_id=batch_id, |
|
revision=revision, |
|
decisions=decisions, |
|
actor=actor, |
|
) |
|
else: |
|
revision = revision_number(options["revision"]) |
|
action = {"refresh": services.refresh_preview, "commit": services.commit_preview, |
|
"cancel": services.cancel_preview}[operation] |
|
batch = action(batch_id=batch_id, revision=revision, actor=actor) |
|
counts = getattr(batch, "result", {}) if batch.state == "COMMITTED" else getattr(batch, "summary", {}) |
|
safe_counts = { |
|
key: value for key, value in counts.items() |
|
if key in {"rows", "error_rows", "decided_rows", "created", "matched", "skipped"} |
|
} |
|
self.stdout.write(json.dumps({"batch_id": str(batch.pk), "state": batch.state, |
|
"revision": batch.revision, "counts": safe_counts}))
|
|
|