11 changed files with 622 additions and 2 deletions
@ -0,0 +1,48 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}Imports · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<h1>Imports</h1> |
||||
<p>Copy up to 10 bank CSV or Ourocard TXT files into the local <code>imports/</code> folder, then scan and map each section to an active BRL account.</p> |
||||
<p><a href="{% url 'account-create' %}">Create an account</a> before mapping if needed.</p> |
||||
{% if error %}<p class="error-summary" role="alert">{{ error }}</p>{% endif %} |
||||
<form method="post"> |
||||
{% csrf_token %} |
||||
<button name="action" value="scan">Scan files</button> |
||||
</form> |
||||
{% if files %} |
||||
<form method="post"> |
||||
{% csrf_token %} |
||||
{% if scan_token %}<input type="hidden" name="scan_token" value="{{ scan_token }}">{% endif %} |
||||
{% if prepared_token %}<input type="hidden" name="prepared_token" value="{{ prepared_token }}">{% endif %} |
||||
<div class="table-wrap"> |
||||
<table> |
||||
<caption>Local file status</caption> |
||||
<thead><tr><th scope="col">File</th><th scope="col">Profile</th><th scope="col">Sections and accounts</th><th scope="col">Status</th><th scope="col">Counts</th></tr></thead> |
||||
<tbody> |
||||
{% for file in files %} |
||||
<tr> |
||||
<th scope="row">{{ file.name }}</th> |
||||
<td>{{ file.profile|default:"Unknown" }}</td> |
||||
<td>{% if file.form %}{% for field in file.form %}<div class="field">{{ field.label_tag }}{{ field }}</div>{% endfor %}{% else %}{% for section in file.sections %}<p>Section {{ section }}</p>{% empty %}—{% endfor %}{% endif %}</td> |
||||
<td>{{ file.state }}{% if file.code %}<p class="error">{{ file.code }}</p>{% endif %}</td> |
||||
<td>{% if file.counts != None %}{% if file.state == "READY" or file.state == "ALREADY_IMPORTED" %}<p>Planned</p>{% endif %}<p>Created: {{ file.counts.created }}</p><p>Matched: {{ file.counts.matched }}</p><p>Skipped: {{ file.counts.skipped }}</p>{% else %}—{% endif %}</td> |
||||
</tr> |
||||
{% endfor %} |
||||
</tbody> |
||||
</table> |
||||
</div> |
||||
{% if scan_token %} |
||||
<p>Preparation checks each file and its mapping. Transactions are written only when you import ready files.</p> |
||||
<button name="action" value="prepare">Prepare imports</button> |
||||
{% endif %} |
||||
{% if prepared_token %} |
||||
<p>Import the ready files and finish exact reimport skips. Files needing attention remain separate.</p> |
||||
<button name="action" value="commit">Import ready files</button> |
||||
{% endif %} |
||||
</form> |
||||
<p>For NEEDS_ATTENTION, correct the source or account mapping and scan again. REVIEW_REQUIRED needs a local manual review before importing; unsupported files must be replaced with a supported CSV or TXT export.</p> |
||||
<p>Scan and prepared selections expire after 15 minutes. Scan again after changing a file.</p> |
||||
{% elif not error %} |
||||
<p>No files listed. Add supported files to <code>imports/</code> and choose Scan files.</p> |
||||
{% endif %} |
||||
{% endblock %} |
||||
@ -0,0 +1,298 @@
|
||||
"""The local owner imports synthetic files without sending source rows to the UI.""" |
||||
|
||||
from pathlib import Path |
||||
import random |
||||
import string |
||||
|
||||
import pytest |
||||
from django.core import signing |
||||
from django.test import Client |
||||
from django.urls import reverse |
||||
|
||||
from finance.models import AuditEvent, SourceRecord, Transaction |
||||
from finance.services import Actor, archive, create_account |
||||
from ingestion import services |
||||
from ingestion.models import ImportBatch |
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True) |
||||
ACTOR = Actor(kind="AGENT", identifier="synthetic-guided-web") |
||||
DATA = Path("tests/fixtures/synthetic/ingestion/bank.csv").read_bytes() |
||||
|
||||
|
||||
@pytest.fixture |
||||
def source(tmp_path, settings): |
||||
settings.IMPORT_ROOT = tmp_path |
||||
(tmp_path / "bank.csv").write_bytes(DATA) |
||||
return tmp_path |
||||
|
||||
|
||||
@pytest.fixture |
||||
def bank(): |
||||
return create_account(name="Synthetic web bank", kind="BANK", actor=ACTOR) |
||||
|
||||
|
||||
def scan(client): |
||||
response = client.post(reverse("imports-index"), {"action": "scan"}, follow=True) |
||||
assert response.status_code == 200 |
||||
assert not response.redirect_chain |
||||
return response |
||||
|
||||
|
||||
def prepare(client, scanned, bank, **overrides): |
||||
values = { |
||||
"action": "prepare", "scan_token": scanned.context["scan_token"], |
||||
"file-0-section_0": str(bank.pk), |
||||
} |
||||
values.update(overrides) |
||||
return client.post(reverse("imports-index"), values, follow=True) |
||||
|
||||
|
||||
def commit(client, prepared): |
||||
return client.post(reverse("imports-index"), { |
||||
"action": "commit", "prepared_token": prepared.context["prepared_token"], |
||||
}, follow=True) |
||||
|
||||
|
||||
def assert_private(response, source): |
||||
content = response.content.decode() |
||||
for forbidden in ("Synthetic purchase", "Synthetic income", "-12.34", "100.00", str(source)): |
||||
assert forbidden not in content |
||||
|
||||
|
||||
def test_navigation_opens_imports_without_automatically_scanning(client, source): |
||||
home = client.get(reverse("home")) |
||||
assert 'href="/imports/"' in home.content.decode() |
||||
response = client.get(reverse("imports-index")) |
||||
assert response.status_code == 200 |
||||
assert "Scan files" in response.content.decode() |
||||
assert "bank.csv" not in response.content.decode() |
||||
assert not ImportBatch.objects.exists() |
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["scan", "prepare", "commit"]) |
||||
def test_import_actions_require_csrf(source, action): |
||||
browser = Client(enforce_csrf_checks=True) |
||||
response = browser.post(reverse("imports-index"), {"action": action}) |
||||
assert response.status_code == 403 |
||||
assert not ImportBatch.objects.exists() |
||||
|
||||
|
||||
def test_scan_shows_safe_metadata_and_only_active_brl_account_options(client, source, bank): |
||||
retired = create_account(name="Synthetic archived", kind="BANK", actor=ACTOR) |
||||
archive(retired, actor=ACTOR) |
||||
create_account(name="Synthetic foreign", kind="BANK", currency="USD", actor=ACTOR) |
||||
(source / "unsupported.ofx").write_text("Synthetic private source text") |
||||
response = scan(client) |
||||
content = response.content.decode() |
||||
assert "bank.csv" in content and "BANK_CSV_V1" in content and "Section 1" in content |
||||
assert "unsupported.ofx" in content and "UNSUPPORTED_FILE" in content |
||||
assert bank.name in content |
||||
assert "Synthetic archived" not in content and "Synthetic foreign" not in content |
||||
assert "Synthetic private source text" not in content |
||||
assert_private(response, source) |
||||
assert not ImportBatch.objects.exists() |
||||
payload = signing.loads(response.context["scan_token"], salt="guided-imports-scan") |
||||
assert len(payload) == 1 |
||||
assert set(payload[0]) == {"name", "document_hash", "profile", "sections"} |
||||
|
||||
|
||||
def test_mapping_prepares_then_explicit_commit_creates_with_local_human_actor(client, source, bank): |
||||
prepared = prepare(client, scan(client), bank) |
||||
assert prepared.status_code == 200 |
||||
assert not prepared.redirect_chain |
||||
assert prepared.context["files"][0]["state"] == "READY" |
||||
assert prepared.context["files"][0]["counts"] == {"created": 2, "matched": 0, "skipped": 0} |
||||
assert not Transaction.objects.exists() |
||||
assert_private(prepared, source) |
||||
payload = signing.loads(prepared.context["prepared_token"], salt="guided-imports-prepared") |
||||
assert set(payload[0]) == {"name", "document_hash", "profile", "sections", "batch_id", "revision"} |
||||
response = commit(client, prepared) |
||||
assert response.redirect_chain == [(reverse("imports-index"), 302)] |
||||
assert response.context["files"][0]["state"] == "COMMITTED" |
||||
assert response.context["files"][0]["counts"]["created"] == 2 |
||||
assert Transaction.objects.count() == SourceRecord.objects.count() == 2 |
||||
assert set(Transaction.objects.values_list("account_id", flat=True)) == {bank.pk} |
||||
assert set(AuditEvent.objects.filter(operation__startswith="import_").values_list( |
||||
"actor_kind", "actor_identifier")) == {("HUMAN", "local-owner")} |
||||
assert_private(response, source) |
||||
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
||||
|
||||
|
||||
def test_scan_and_prepared_form_metadata_do_not_depend_on_session_payload(client, source, bank): |
||||
scanned = scan(client) |
||||
scan_token = scanned.context["scan_token"] |
||||
assert f'name="scan_token" value="{scan_token}"' in scanned.content.decode() |
||||
assert "scan_token" not in str(dict(client.session)) |
||||
client.cookies.pop("sessionid", None) |
||||
prepared = prepare(client, scanned, bank) |
||||
prepared_token = prepared.context["prepared_token"] |
||||
assert f'name="prepared_token" value="{prepared_token}"' in prepared.content.decode() |
||||
assert "prepared_token" not in str(dict(client.session)) |
||||
client.cookies.pop("sessionid", None) |
||||
response = commit(client, prepared) |
||||
assert response.context["files"][0]["counts"]["created"] == 2 |
||||
assert "document_hash" not in str(dict(client.session)) |
||||
assert "batch_id" not in str(dict(client.session)) |
||||
|
||||
|
||||
def test_oversized_scan_has_safe_result_and_leaves_every_file_untouched(client, source, bank): |
||||
for index in range(20): |
||||
(source / f"synthetic-{index}.csv").write_bytes(DATA) |
||||
response = scan(client) |
||||
assert "SCAN_TOO_LARGE" in response.content.decode() |
||||
assert not response.context.get("scan_token") |
||||
assert not ImportBatch.objects.exists() |
||||
assert len(list(source.iterdir())) == 21 |
||||
if "sessionid" in client.cookies: |
||||
assert len(client.cookies["sessionid"].value) < 4096 |
||||
|
||||
|
||||
def test_long_filename_summaries_fit_browser_cookie_and_do_not_accumulate_across_tabs(client, source, bank): |
||||
(source / "bank.csv").unlink() |
||||
names = random.Random(41) |
||||
filenames = ["".join(names.choices(string.ascii_letters + string.digits, k=250)) + ".csv" |
||||
for _ in range(10)] |
||||
for filename in filenames: |
||||
(source / filename).write_bytes(DATA) |
||||
scanned = scan(client) |
||||
fields = {f"file-{index}-section_0": str(bank.pk) for index in range(10)} |
||||
prepared = prepare(client, scanned, bank, **fields) |
||||
# A second tab scans other files before the first tab finishes its old form. |
||||
for filename in filenames: |
||||
(source / filename).unlink() |
||||
(source / filename.replace(".csv", ".ofx")).write_text("Synthetic unsupported") |
||||
scan(client) |
||||
response = commit(client, prepared) |
||||
assert len(response.context["files"]) == 10 |
||||
assert all(row["code"] == "UNREADABLE_FILE" for row in response.context["files"]) |
||||
assert len(client.cookies["sessionid"].OutputString().encode()) < 4096 |
||||
assert not Transaction.objects.exists() |
||||
|
||||
|
||||
def test_exact_reimport_reports_skips_without_new_transactions(client, source, bank): |
||||
commit(client, prepare(client, scan(client), bank)) |
||||
prepared = prepare(client, scan(client), bank) |
||||
assert prepared.context["files"][0]["state"] == "ALREADY_IMPORTED" |
||||
result = commit(client, prepared) |
||||
assert result.context["files"][0]["counts"] == {"created": 0, "matched": 0, "skipped": 2} |
||||
assert Transaction.objects.count() == SourceRecord.objects.count() == 2 |
||||
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
||||
|
||||
|
||||
def test_multi_section_card_has_one_typed_mapping_per_section(client, source): |
||||
(source / "bank.csv").unlink() |
||||
(source / "cards.txt").write_text( |
||||
"1 - SYNTHETIC HOLDER\n" + f"15.01.2026{'Synthetic first':37}BR 12,50 0,00\n" |
||||
+ "2 - SYNTHETIC OTHER\n" + f"16.01.2026{'Synthetic second':37}BR 7,25 0,00\n" |
||||
) |
||||
first = create_account(name="Synthetic first card", kind="CREDIT_CARD", actor=ACTOR) |
||||
second = create_account(name="Synthetic second card", kind="CREDIT_CARD", actor=ACTOR) |
||||
scanned = scan(client) |
||||
assert "Section 1" in scanned.content.decode() and "Section 2" in scanned.content.decode() |
||||
assert "SYNTHETIC HOLDER" not in scanned.content.decode() |
||||
response = commit(client, prepare(client, scanned, first, **{"file-0-section_1": str(second.pk)})) |
||||
assert response.context["files"][0]["counts"]["created"] == 2 |
||||
assert set(Transaction.objects.values_list("account_id", flat=True)) == {first.pk, second.pk} |
||||
|
||||
|
||||
@pytest.mark.parametrize("mapping", ["", "bad-id", "archived", "foreign"]) |
||||
def test_invalid_mapping_is_safe_and_rejected_before_preview(client, source, bank, mapping): |
||||
scanned = scan(client) |
||||
if mapping == "archived": |
||||
archive(bank, actor=ACTOR) |
||||
mapping = str(bank.pk) |
||||
elif mapping == "foreign": |
||||
mapping = str(create_account(name="Synthetic USD", kind="BANK", currency="USD", actor=ACTOR).pk) |
||||
result = prepare(client, scanned, bank, **{"file-0-section_0": mapping}) |
||||
assert result.context["files"][0]["code"] == "INVALID_SECTION_MAPPING" |
||||
assert not ImportBatch.objects.exists() |
||||
assert_private(result, source) |
||||
|
||||
|
||||
def test_review_exception_does_not_prevent_another_file_commit(client, source, bank): |
||||
(source / "ambiguous.csv").write_bytes(DATA + DATA.splitlines(keepends=True)[1]) |
||||
prepared = prepare(client, scan(client), bank, **{"file-1-section_0": str(bank.pk)}) |
||||
assert [row["state"] for row in prepared.context["files"]] == ["NEEDS_ATTENTION", "READY"] |
||||
result = commit(client, prepared) |
||||
assert [row["state"] for row in result.context["files"]] == ["NEEDS_ATTENTION", "COMMITTED"] |
||||
assert result.context["files"][0]["code"] == "REVIEW_REQUIRED" |
||||
assert Transaction.objects.count() == 2 |
||||
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["prepare", "commit"]) |
||||
def test_unexpected_per_file_error_never_exposes_source_or_blocks_next_file(client, source, bank, monkeypatch, stage): |
||||
from core import views |
||||
|
||||
(source / "first.csv").write_bytes(DATA.replace(b"12.34", b"22.34")) |
||||
scanned = scan(client) |
||||
if stage == "commit": |
||||
prepared = prepare(client, scanned, bank, **{"file-1-section_0": str(bank.pk)}) |
||||
service_name = "prepare_import" if stage == "prepare" else "commit_prepared_import" |
||||
original = getattr(views.guided_imports, service_name) |
||||
|
||||
def fail_one_file(**kwargs): |
||||
row = kwargs["discovered" if stage == "prepare" else "prepared"] |
||||
if row.name == "bank.csv": |
||||
raise RuntimeError(f"{source}: Synthetic private source text 1234.56") |
||||
return original(**kwargs) |
||||
|
||||
monkeypatch.setattr(views.guided_imports, service_name, fail_one_file) |
||||
if stage == "prepare": |
||||
prepared = prepare(client, scanned, bank, **{"file-1-section_0": str(bank.pk)}) |
||||
result = commit(client, prepared) |
||||
assert [row["state"] for row in result.context["files"]] == ["NEEDS_ATTENTION", "COMMITTED"] |
||||
assert result.context["files"][0]["code"] == "IMPORT_FAILED" |
||||
assert "Synthetic private source text" not in result.content.decode() |
||||
assert "1234.56" not in result.content.decode() |
||||
assert Transaction.objects.count() == 2 |
||||
assert_private(result, source) |
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["prepare", "commit"]) |
||||
@pytest.mark.parametrize("alteration", ["tamper", "expired", "wrong-stage"]) |
||||
def test_signed_workflow_rejects_altered_expired_and_wrong_stage_tokens( |
||||
client, source, bank, monkeypatch, stage, alteration |
||||
): |
||||
scanned = scan(client) |
||||
prepared = prepare(client, scanned, bank) if stage == "commit" else None |
||||
token_name = "scan_token" if stage == "prepare" else "prepared_token" |
||||
token = (prepared if prepared else scanned).context[token_name] |
||||
if alteration == "tamper": |
||||
token += "altered" |
||||
elif alteration == "expired": |
||||
import time |
||||
original_time = time.time() |
||||
monkeypatch.setattr(signing.time, "time", lambda: original_time + 901) |
||||
else: |
||||
token = scanned.context["scan_token"] if stage == "commit" else signing.dumps( |
||||
[], salt="guided-imports-prepared") |
||||
response = client.post(reverse("imports-index"), { |
||||
"action": stage, token_name: token, "file-0-section_0": str(bank.pk), |
||||
}) |
||||
assert response.status_code == 400 |
||||
assert "Scan again" in response.content.decode() |
||||
assert not Transaction.objects.exists() |
||||
assert_private(response, source) |
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["prepare", "commit"]) |
||||
def test_changed_source_rejects_previously_signed_metadata(client, source, bank, stage): |
||||
scanned = scan(client) |
||||
prepared = prepare(client, scanned, bank) if stage == "commit" else None |
||||
(source / "bank.csv").write_bytes(DATA.replace(b"12.34", b"22.34")) |
||||
response = commit(client, prepared) if prepared else prepare(client, scanned, bank) |
||||
assert response.context["files"][0]["code"] == "FILE_CHANGED" |
||||
assert not Transaction.objects.exists() |
||||
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
||||
|
||||
|
||||
def test_stale_prepared_revision_requires_attention_without_writing(client, source, bank): |
||||
prepared = prepare(client, scan(client), bank) |
||||
batch = ImportBatch.objects.get() |
||||
services.refresh_preview(batch_id=batch.pk, revision=batch.revision, actor=ACTOR) |
||||
response = commit(client, prepared) |
||||
assert response.context["files"][0]["code"] == "STALE_REVISION" |
||||
assert not Transaction.objects.exists() |
||||
assert not ImportBatch.objects.filter(state="PREVIEW").exists() |
||||
Loading…
Reference in new issue