"""The local owner imports synthetic files without sending source rows to the UI.""" from pathlib import Path import random import string import pytest from django.core import signing from django.test import Client from django.urls import reverse from finance.models import AuditEvent, SourceRecord, Transaction from finance.services import Actor, archive, create_account from ingestion import services from ingestion.models import ImportBatch pytestmark = pytest.mark.django_db(transaction=True) ACTOR = Actor(kind="AGENT", identifier="synthetic-guided-web") DATA = Path("tests/fixtures/synthetic/ingestion/bank.csv").read_bytes() @pytest.fixture def source(tmp_path, settings): settings.IMPORT_ROOT = tmp_path (tmp_path / "bank.csv").write_bytes(DATA) return tmp_path @pytest.fixture def bank(): return create_account(name="Synthetic web bank", kind="BANK", actor=ACTOR) def scan(client): response = client.post(reverse("imports-index"), {"action": "scan"}, follow=True) assert response.status_code == 200 assert not response.redirect_chain return response def prepare(client, scanned, bank, **overrides): values = { "action": "prepare", "scan_token": scanned.context["scan_token"], "file-0-section_0": str(bank.pk), } values.update(overrides) return client.post(reverse("imports-index"), values, follow=True) def commit(client, prepared): return client.post(reverse("imports-index"), { "action": "commit", "prepared_token": prepared.context["prepared_token"], }, follow=True) def assert_private(response, source): content = response.content.decode() for forbidden in ("Synthetic purchase", "Synthetic income", "-12.34", "100.00", str(source)): assert forbidden not in content def test_navigation_opens_imports_without_automatically_scanning(client, source): home = client.get(reverse("home")) assert 'href="/imports/"' in home.content.decode() response = client.get(reverse("imports-index")) assert response.status_code == 200 assert "Scan files" in response.content.decode() assert "bank.csv" not in response.content.decode() assert not ImportBatch.objects.exists() @pytest.mark.parametrize("action", ["scan", "prepare", "commit"]) def test_import_actions_require_csrf(source, action): browser = Client(enforce_csrf_checks=True) response = browser.post(reverse("imports-index"), {"action": action}) assert response.status_code == 403 assert not ImportBatch.objects.exists() def test_scan_shows_safe_metadata_and_only_active_brl_account_options(client, source, bank): retired = create_account(name="Synthetic archived", kind="BANK", actor=ACTOR) archive(retired, actor=ACTOR) create_account(name="Synthetic foreign", kind="BANK", currency="USD", actor=ACTOR) (source / "unsupported.ofx").write_text("Synthetic private source text") response = scan(client) content = response.content.decode() assert "bank.csv" in content and "BANK_CSV_V1" in content and "Section 1" in content assert "unsupported.ofx" in content and "UNSUPPORTED_FILE" in content assert bank.name in content assert "Synthetic archived" not in content and "Synthetic foreign" not in content assert "Synthetic private source text" not in content assert_private(response, source) assert not ImportBatch.objects.exists() payload = signing.loads(response.context["scan_token"], salt="guided-imports-scan") assert len(payload) == 1 assert set(payload[0]) == {"name", "document_hash", "profile", "sections"} def test_mapping_prepares_then_explicit_commit_creates_with_local_human_actor(client, source, bank): prepared = prepare(client, scan(client), bank) assert prepared.status_code == 200 assert not prepared.redirect_chain assert prepared.context["files"][0]["state"] == "READY" assert prepared.context["files"][0]["counts"] == {"created": 2, "matched": 0, "skipped": 0} assert not Transaction.objects.exists() assert_private(prepared, source) payload = signing.loads(prepared.context["prepared_token"], salt="guided-imports-prepared") assert set(payload[0]) == {"name", "document_hash", "profile", "sections", "batch_id", "revision"} response = commit(client, prepared) assert response.redirect_chain == [(reverse("imports-index"), 302)] assert response.context["files"][0]["state"] == "COMMITTED" assert response.context["files"][0]["counts"]["created"] == 2 assert Transaction.objects.count() == SourceRecord.objects.count() == 2 assert set(Transaction.objects.values_list("account_id", flat=True)) == {bank.pk} assert set(AuditEvent.objects.filter(operation__startswith="import_").values_list( "actor_kind", "actor_identifier")) == {("HUMAN", "local-owner")} assert_private(response, source) assert not ImportBatch.objects.filter(state="PREVIEW").exists() def test_scan_and_prepared_form_metadata_do_not_depend_on_session_payload(client, source, bank): scanned = scan(client) scan_token = scanned.context["scan_token"] assert f'name="scan_token" value="{scan_token}"' in scanned.content.decode() assert "scan_token" not in str(dict(client.session)) client.cookies.pop("sessionid", None) prepared = prepare(client, scanned, bank) prepared_token = prepared.context["prepared_token"] assert f'name="prepared_token" value="{prepared_token}"' in prepared.content.decode() assert "prepared_token" not in str(dict(client.session)) client.cookies.pop("sessionid", None) response = commit(client, prepared) assert response.context["files"][0]["counts"]["created"] == 2 assert "document_hash" not in str(dict(client.session)) assert "batch_id" not in str(dict(client.session)) def test_oversized_scan_has_safe_result_and_leaves_every_file_untouched(client, source, bank): for index in range(20): (source / f"synthetic-{index}.csv").write_bytes(DATA) response = scan(client) assert "SCAN_TOO_LARGE" in response.content.decode() assert not response.context.get("scan_token") assert not ImportBatch.objects.exists() assert len(list(source.iterdir())) == 21 if "sessionid" in client.cookies: assert len(client.cookies["sessionid"].value) < 4096 def test_long_filename_summaries_fit_browser_cookie_and_do_not_accumulate_across_tabs(client, source, bank): (source / "bank.csv").unlink() names = random.Random(41) filenames = ["".join(names.choices(string.ascii_letters + string.digits, k=250)) + ".csv" for _ in range(10)] for filename in filenames: (source / filename).write_bytes(DATA) scanned = scan(client) fields = {f"file-{index}-section_0": str(bank.pk) for index in range(10)} prepared = prepare(client, scanned, bank, **fields) # A second tab scans other files before the first tab finishes its old form. for filename in filenames: (source / filename).unlink() (source / filename.replace(".csv", ".ofx")).write_text("Synthetic unsupported") scan(client) response = commit(client, prepared) assert len(response.context["files"]) == 10 assert all(row["code"] == "UNREADABLE_FILE" for row in response.context["files"]) assert len(client.cookies["sessionid"].OutputString().encode()) < 4096 assert not Transaction.objects.exists() def test_exact_reimport_reports_skips_without_new_transactions(client, source, bank): commit(client, prepare(client, scan(client), bank)) prepared = prepare(client, scan(client), bank) assert prepared.context["files"][0]["state"] == "ALREADY_IMPORTED" result = commit(client, prepared) assert result.context["files"][0]["counts"] == {"created": 0, "matched": 0, "skipped": 2} assert Transaction.objects.count() == SourceRecord.objects.count() == 2 assert not ImportBatch.objects.filter(state="PREVIEW").exists() def test_multi_section_card_has_one_typed_mapping_per_section(client, source): (source / "bank.csv").unlink() (source / "cards.txt").write_text( "1 - SYNTHETIC HOLDER\n" + f"15.01.2026{'Synthetic first':37}BR 12,50 0,00\n" + "2 - SYNTHETIC OTHER\n" + f"16.01.2026{'Synthetic second':37}BR 7,25 0,00\n" ) first = create_account(name="Synthetic first card", kind="CREDIT_CARD", actor=ACTOR) second = create_account(name="Synthetic second card", kind="CREDIT_CARD", actor=ACTOR) scanned = scan(client) assert "Section 1" in scanned.content.decode() and "Section 2" in scanned.content.decode() assert "SYNTHETIC HOLDER" not in scanned.content.decode() response = commit(client, prepare(client, scanned, first, **{"file-0-section_1": str(second.pk)})) assert response.context["files"][0]["counts"]["created"] == 2 assert set(Transaction.objects.values_list("account_id", flat=True)) == {first.pk, second.pk} @pytest.mark.parametrize("mapping", ["", "bad-id", "archived", "foreign"]) def test_invalid_mapping_is_safe_and_rejected_before_preview(client, source, bank, mapping): scanned = scan(client) if mapping == "archived": archive(bank, actor=ACTOR) mapping = str(bank.pk) elif mapping == "foreign": mapping = str(create_account(name="Synthetic USD", kind="BANK", currency="USD", actor=ACTOR).pk) result = prepare(client, scanned, bank, **{"file-0-section_0": mapping}) assert result.context["files"][0]["code"] == "INVALID_SECTION_MAPPING" assert not ImportBatch.objects.exists() assert_private(result, source) def test_review_exception_does_not_prevent_another_file_commit(client, source, bank): (source / "ambiguous.csv").write_bytes(DATA + DATA.splitlines(keepends=True)[1]) prepared = prepare(client, scan(client), bank, **{"file-1-section_0": str(bank.pk)}) assert [row["state"] for row in prepared.context["files"]] == ["NEEDS_ATTENTION", "READY"] result = commit(client, prepared) assert [row["state"] for row in result.context["files"]] == ["NEEDS_ATTENTION", "COMMITTED"] assert result.context["files"][0]["code"] == "REVIEW_REQUIRED" assert Transaction.objects.count() == 2 assert not ImportBatch.objects.filter(state="PREVIEW").exists() @pytest.mark.parametrize("stage", ["prepare", "commit"]) def test_unexpected_per_file_error_never_exposes_source_or_blocks_next_file(client, source, bank, monkeypatch, stage): from core import views (source / "first.csv").write_bytes(DATA.replace(b"12.34", b"22.34")) scanned = scan(client) if stage == "commit": prepared = prepare(client, scanned, bank, **{"file-1-section_0": str(bank.pk)}) service_name = "prepare_import" if stage == "prepare" else "commit_prepared_import" original = getattr(views.guided_imports, service_name) def fail_one_file(**kwargs): row = kwargs["discovered" if stage == "prepare" else "prepared"] if row.name == "bank.csv": raise RuntimeError(f"{source}: Synthetic private source text 1234.56") return original(**kwargs) monkeypatch.setattr(views.guided_imports, service_name, fail_one_file) if stage == "prepare": prepared = prepare(client, scanned, bank, **{"file-1-section_0": str(bank.pk)}) result = commit(client, prepared) assert [row["state"] for row in result.context["files"]] == ["NEEDS_ATTENTION", "COMMITTED"] assert result.context["files"][0]["code"] == "IMPORT_FAILED" assert "Synthetic private source text" not in result.content.decode() assert "1234.56" not in result.content.decode() assert Transaction.objects.count() == 2 assert_private(result, source) @pytest.mark.parametrize("stage", ["prepare", "commit"]) @pytest.mark.parametrize("alteration", ["tamper", "expired", "wrong-stage"]) def test_signed_workflow_rejects_altered_expired_and_wrong_stage_tokens( client, source, bank, monkeypatch, stage, alteration ): scanned = scan(client) prepared = prepare(client, scanned, bank) if stage == "commit" else None token_name = "scan_token" if stage == "prepare" else "prepared_token" token = (prepared if prepared else scanned).context[token_name] if alteration == "tamper": token += "altered" elif alteration == "expired": import time original_time = time.time() monkeypatch.setattr(signing.time, "time", lambda: original_time + 901) else: token = scanned.context["scan_token"] if stage == "commit" else signing.dumps( [], salt="guided-imports-prepared") response = client.post(reverse("imports-index"), { "action": stage, token_name: token, "file-0-section_0": str(bank.pk), }) assert response.status_code == 400 assert "Scan again" in response.content.decode() assert not Transaction.objects.exists() assert_private(response, source) @pytest.mark.parametrize("stage", ["prepare", "commit"]) def test_changed_source_rejects_previously_signed_metadata(client, source, bank, stage): scanned = scan(client) prepared = prepare(client, scanned, bank) if stage == "commit" else None (source / "bank.csv").write_bytes(DATA.replace(b"12.34", b"22.34")) response = commit(client, prepared) if prepared else prepare(client, scanned, bank) assert response.context["files"][0]["code"] == "FILE_CHANGED" assert not Transaction.objects.exists() assert not ImportBatch.objects.filter(state="PREVIEW").exists() def test_stale_prepared_revision_requires_attention_without_writing(client, source, bank): prepared = prepare(client, scan(client), bank) batch = ImportBatch.objects.get() services.refresh_preview(batch_id=batch.pk, revision=batch.revision, actor=ACTOR) response = commit(client, prepared) assert response.context["files"][0]["code"] == "STALE_REVISION" assert not Transaction.objects.exists() assert not ImportBatch.objects.filter(state="PREVIEW").exists()