8 changed files with 714 additions and 7 deletions
@ -0,0 +1,97 @@
|
||||
# Python bytecode and caches |
||||
__pycache__/ |
||||
*.py[cod] |
||||
*$py.class |
||||
.pytest_cache/ |
||||
.mypy_cache/ |
||||
.ruff_cache/ |
||||
.coverage |
||||
coverage.xml |
||||
htmlcov/ |
||||
|
||||
# Virtual environments |
||||
.venv/ |
||||
venv/ |
||||
env/ |
||||
ENV/ |
||||
|
||||
# Packaging and build output |
||||
build/ |
||||
dist/ |
||||
*.egg-info/ |
||||
|
||||
# Environment files and secrets |
||||
.env |
||||
.env.* |
||||
!.env.example |
||||
*.pem |
||||
*.key |
||||
|
||||
# Local databases and runtime data |
||||
*.sqlite3 |
||||
*.sqlite3-journal |
||||
*.db |
||||
data/ |
||||
uploads/ |
||||
media/ |
||||
|
||||
# Financial source files and database exports |
||||
documents/ |
||||
imports/ |
||||
exports/ |
||||
backups/ |
||||
*.ofx |
||||
*.qfx |
||||
*.sql.gz |
||||
dump.sql |
||||
dump-*.sql |
||||
**/dump.sql |
||||
**/dump-*.sql |
||||
|
||||
# Logs and temporary files |
||||
*.log |
||||
*.tmp |
||||
*.temp |
||||
*.swp |
||||
*~ |
||||
|
||||
# Jupyter |
||||
.ipynb_checkpoints/ |
||||
|
||||
# Editors and operating systems |
||||
.idea/ |
||||
.vscode/ |
||||
*.code-workspace |
||||
.DS_Store |
||||
Thumbs.db |
||||
Desktop.ini |
||||
|
||||
# Frontend dependencies and build artifacts |
||||
node_modules/ |
||||
.next/ |
||||
coverage/ |
||||
|
||||
# Docker overrides |
||||
compose.override.yaml |
||||
docker-compose.override.yml |
||||
|
||||
# Financial documents and database dumps |
||||
*.csv |
||||
*.txt |
||||
*.pdf |
||||
*.sql |
||||
*.dump |
||||
*.backup |
||||
*.sql.bz2 |
||||
*.sql.xz |
||||
*.sql.zip |
||||
*.zip |
||||
|
||||
# Explicitly synthetic fixtures and dependency manifests |
||||
!tests/fixtures/synthetic/**/*.csv |
||||
!tests/fixtures/synthetic/**/*.txt |
||||
!tests/fixtures/synthetic/**/*.pdf |
||||
!requirements*.txt |
||||
|
||||
# Isolated worktrees |
||||
.worktrees/ |
||||
@ -0,0 +1,102 @@
|
||||
# Gizmoduck |
||||
|
||||
Gizmoduck is a privacy-conscious personal finance application for importing, |
||||
organizing, and understanding financial activity. It is a clean successor to |
||||
the Robopato prototype, carrying forward what was learned from parsing Banco do |
||||
Brasil account statements and Ourocard invoices without inheriting the |
||||
prototype's notebook-driven architecture or historical data. |
||||
|
||||
## Goals |
||||
|
||||
The initial product should make it possible to: |
||||
|
||||
- manage bank accounts and credit cards; |
||||
- import supported statement and invoice formats; |
||||
- preview and validate an import before committing it; |
||||
- detect duplicates and preserve import provenance; |
||||
- organize transactions by payee and category; |
||||
- represent transfers, refunds, and installments correctly; |
||||
- inspect monthly cash flow and spending summaries; and |
||||
- export and back up user-owned data. |
||||
|
||||
The first release is intended to be a reliable local application. Bank APIs, |
||||
investment tracking, mobile clients, receipt recognition, forecasting, and AI |
||||
categorization are possible later additions, not MVP requirements. |
||||
|
||||
## Project documentation |
||||
|
||||
Start with [contributor instructions](AGENTS.md), then read the |
||||
[product scope](docs/product.md), [architecture](docs/architecture.md), |
||||
[data policy](docs/data-policy.md), and [delivery roadmap](docs/roadmap.md). |
||||
Owners can follow the copy-paste [local workflow guide](docs/local-workflows.md) |
||||
from reviewed import through reports, export, and verified backup restoration. |
||||
|
||||
## Architecture |
||||
|
||||
The approved technical choices are: |
||||
|
||||
- Python and Django for the application; |
||||
- PostgreSQL for persistent storage; |
||||
- schema migrations from the first release; |
||||
- Docker Compose for local infrastructure; |
||||
- pytest for automated tests; and |
||||
- pure, source-specific parsers feeding a normalized import pipeline. |
||||
|
||||
Changes require explicit owner approval and a documented architectural decision. |
||||
Bank-account imports use CSV; Banco do Brasil/Ourocard credit-card imports use |
||||
TXT. OFX is not supported. |
||||
|
||||
The import flow should remain separate from database persistence: |
||||
|
||||
```text |
||||
source document |
||||
-> source-specific parser |
||||
-> normalized candidate transactions |
||||
-> validation and duplicate detection |
||||
-> import preview |
||||
-> committed transactions |
||||
-> categorization and reporting |
||||
``` |
||||
|
||||
## Privacy and security |
||||
|
||||
Financial documents, database exports, credentials, and local application data |
||||
must never be committed to Git. The repository `.gitignore` excludes common |
||||
financial source formats and local data directories, but contributors must |
||||
still inspect staged changes before every commit. |
||||
|
||||
Development and application processes should use separate, least-privilege |
||||
database accounts. Imports and material data changes should record their actor, |
||||
source, timestamp, and outcome so human, application, and automated activity can |
||||
be distinguished. |
||||
|
||||
Use only synthetic or thoroughly sanitized financial fixtures in automated |
||||
tests and documentation. |
||||
|
||||
## Project status |
||||
|
||||
The local MVP is implemented: contributor foundation, application skeleton, |
||||
financial domain, reviewed imports, owner screens, monthly reports, portable |
||||
exports, and verified PostgreSQL backups. The local Django application uses |
||||
PostgreSQL, explicit migrations, and isolated tests. See the |
||||
[financial domain guide](docs/financial-domain.md) for supported writes and |
||||
accounting conventions, and [reviewed imports](docs/imports.md) for the local |
||||
CSV/Ourocard preview and commit workflow. |
||||
|
||||
Implemented milestones: |
||||
|
||||
1. Define the MVP and privacy model. |
||||
2. Scaffold the application, database, migrations, and test suite. |
||||
3. Implement and test the core financial domain model. |
||||
4. Build the import-preview pipeline using sanitized fixtures. |
||||
5. Add transaction management, categorization, and basic reports. |
||||
|
||||
## Development |
||||
|
||||
Follow the [local development guide](docs/development.md) for configuration, |
||||
Docker Compose startup, migrations, tests, and database-role details, then use |
||||
the [local owner workflows](docs/local-workflows.md) for daily operation and |
||||
recovery. |
||||
|
||||
The readiness endpoint is available at `http://127.0.0.1:8000/health/` after |
||||
startup. Do not place real financial files inside tracked directories. |
||||
@ -0,0 +1,86 @@
|
||||
# Local development |
||||
|
||||
Requires Docker Desktop running Linux containers with Docker Compose. Host |
||||
Python and uv are optional; all application commands run in containers. |
||||
|
||||
## First start (PowerShell) |
||||
|
||||
Copy `.env.example` to `.env` only if `.env` does not already exist. Replace |
||||
all five values with independent random secrets. To generate one in PowerShell: |
||||
|
||||
```powershell |
||||
[Convert]::ToHexString([System.Security.Cryptography.RandomNumberGenerator]::GetBytes(32)) |
||||
``` |
||||
|
||||
Never share `.env` or output from `docker compose config` without `--quiet`. |
||||
|
||||
```powershell |
||||
docker compose config --quiet |
||||
docker compose build |
||||
docker compose up -d db |
||||
docker compose run --rm manage migrate |
||||
docker compose up -d web |
||||
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||
``` |
||||
|
||||
The health endpoint returns `{"status":"ok"}` when PostgreSQL is reachable, |
||||
and HTTP 503 with `{"status":"unavailable"}` when database access fails. |
||||
It does not verify schema currency. The dashboard is available at `/`. |
||||
The server binds only to loopback on the host. Ports 8000 and 5432 must be |
||||
available. PostgreSQL is exposed only at `127.0.0.1:5432` for local tools; use |
||||
database `gizmoduck`, runtime role `gizmoduck_app`, and the ignored `.env` value |
||||
of `RUNTIME_PASSWORD` for routine inspection. This is a local development server. |
||||
|
||||
## Verification |
||||
|
||||
```powershell |
||||
docker compose run --rm manage check |
||||
docker compose run --rm manage makemigrations --check --dry-run |
||||
docker compose run --rm manage migrate --check |
||||
docker compose run --rm test |
||||
docker compose run --rm permissions |
||||
``` |
||||
|
||||
The test suite covers the complete synthetic local MVP workflow. The |
||||
runtime-permission check exercises DML grants on both `finance_*` and |
||||
`ingestion_*` tables. See [reviewed imports](imports.md) for the explicit local |
||||
preview, decision, and commit workflow, and [local owner workflows](local-workflows.md) |
||||
for categorization, reports, exports, backups, and recovery. |
||||
|
||||
The test service starts its own PostgreSQL container. Its dedicated non-superuser |
||||
role can create the temporary pytest database. The test database uses tmpfs; |
||||
stopping the test database discards only its synthetic test data. The permissions |
||||
command checks real runtime grants on the migrated development schema, using |
||||
synthetic group rows in rolled-back transactions (sequence gaps are normal). |
||||
It verifies DML, sequence access, denied schema/table creation and role escalation, |
||||
and absence of database creation and administrator privileges. |
||||
|
||||
Runtime `gizmoduck_app` has only DML and sequence access. Management commands use |
||||
`gizmoduck_migrate`, which owns the schema. Bootstrap administrator credentials |
||||
are passed only to the database service. Web startup does not run migrations. |
||||
|
||||
## Changes and lifecycle |
||||
|
||||
Rebuild with `docker compose build` after editing application or test code; |
||||
then use `docker compose up -d web` to replace the web container. Dependencies |
||||
are installed from `uv.lock` with `uv sync --frozen`. To intentionally update |
||||
that lockfile, use uv with Python 3.13 and run `uv lock`, then rebuild and verify. |
||||
|
||||
```powershell |
||||
docker compose stop web db |
||||
docker compose --profile test stop test-db |
||||
docker compose --profile backup stop restore-db |
||||
``` |
||||
|
||||
Development data persists in the project-scoped `postgres_data` Docker volume. |
||||
Do not delete it to fix setup problems. Database initialization scripts run only |
||||
when the volume is empty. Editing `.env` does not rotate credentials stored in |
||||
an existing database: coordinate an explicit database password change and update |
||||
the corresponding local environment value together. Schema privilege changes |
||||
also require an explicit administrator/migration action on existing volumes. |
||||
Preserve or back up valuable data before any maintenance. |
||||
|
||||
If a startup fails, first confirm Docker is running, configuration is complete, |
||||
and ports 8000 and 5432 are free. Check service status with |
||||
`docker compose ps`; database or application logs may contain sensitive details |
||||
and must remain local. |
||||
@ -0,0 +1,215 @@
|
||||
# Local owner workflows |
||||
|
||||
This guide covers the local MVP from setup through import, categorization, |
||||
reporting, export, and verified backup. Commands assume PowerShell in the |
||||
repository root. Financial source files, review JSON, exports, backups, and |
||||
command output that contains financial details must remain local. |
||||
|
||||
## Start the application |
||||
|
||||
Follow the secret setup in [local development](development.md), then run: |
||||
|
||||
```powershell |
||||
docker compose config --quiet |
||||
docker compose build |
||||
docker compose up -d db |
||||
docker compose run --rm manage migrate |
||||
docker compose up -d web |
||||
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||
``` |
||||
|
||||
Open `http://127.0.0.1:8000/`. The application and PostgreSQL listen only on |
||||
loopback. The web UI uses port `8000`; PostgreSQL uses `127.0.0.1:5432` for |
||||
local database tools such as DBeaver. The database name is `gizmoduck`. Use |
||||
`gizmoduck_app` with the `RUNTIME_PASSWORD` value from the ignored `.env` for |
||||
routine inspection. The migration and bootstrap roles have broader privileges |
||||
and are for maintenance only. |
||||
|
||||
The navigation provides Accounts, Transactions, Categories, Payees, Reports, |
||||
and Exports. Create accounts, categories, and payees before assigning them. |
||||
References may be archived, but historical transactions continue to show them. |
||||
Transaction editing is limited to description, payee, and category. Amount, |
||||
dates, account, kind, import provenance, transfers, refunds, statements, and |
||||
installment identity are immutable. Corrections use a new transaction or the |
||||
domain workflows described in [the financial domain guide](financial-domain.md). |
||||
|
||||
## Import reviewed transactions |
||||
|
||||
Create the destination account at `http://127.0.0.1:8000/accounts/new/`. In |
||||
DBeaver, retrieve its UUID without exposing transaction data: |
||||
|
||||
```sql |
||||
SELECT id, name, kind, currency, archived |
||||
FROM finance_account |
||||
ORDER BY name, id; |
||||
``` |
||||
|
||||
Copy the source document into the ignored `imports/` directory. These commands |
||||
use the tracked synthetic CSV; replace `source.csv` and the UUID for a real |
||||
local import. Never copy a real document into `tests/fixtures/`. |
||||
|
||||
```powershell |
||||
New-Item -ItemType Directory -Force imports | Out-Null |
||||
Copy-Item tests/fixtures/synthetic/ingestion/bank.csv imports/source.csv |
||||
git check-ignore imports/source.csv imports/mapping.json imports/review-1.json |
||||
$account = '11111111-1111-4111-8111-111111111111' # Replace with the account UUID. |
||||
@{"1"=$account} | ConvertTo-Json | Set-Content -Encoding utf8 imports/mapping.json |
||||
$actor = @('--actor-kind', 'HUMAN', '--actor-identifier', 'local-owner') |
||||
docker compose run --rm manage import_review preview ` |
||||
--input source.csv --profile BANK_CSV_V1 --mapping mapping.json @actor |
||||
``` |
||||
|
||||
Copy the printed batch UUID, export the review, and inspect every candidate, |
||||
issue, and duplicate suggestion: |
||||
|
||||
```powershell |
||||
$batch = '22222222-2222-4222-8222-222222222222' # Replace with the printed batch UUID. |
||||
docker compose run --rm manage import_review export ` |
||||
--batch $batch --output review-1.json @actor |
||||
notepad imports/review-1.json |
||||
``` |
||||
|
||||
Edit the top-level `decisions` array. Every row needs `CREATE`, `MATCH`, or |
||||
`SKIP`; use the actual locators and current revision from the review. Apply the |
||||
file, export a fresh review under a new name, and commit only when all issues are |
||||
resolved: |
||||
|
||||
```powershell |
||||
docker compose run --rm manage import_review apply ` |
||||
--batch $batch --input review-1.json @actor |
||||
docker compose run --rm manage import_review export ` |
||||
--batch $batch --output review-2.json @actor |
||||
notepad imports/review-2.json |
||||
$revision = 2 # Replace with the revision printed by apply/export. |
||||
docker compose run --rm manage import_review commit ` |
||||
--batch $batch --revision $revision @actor |
||||
``` |
||||
|
||||
Only `commit` writes transactions and provenance. Preview, export, and apply are |
||||
inert. See [reviewed imports](imports.md) for all profiles, decision fields, |
||||
duplicate handling, refresh, cancellation, and Ourocard date contexts. |
||||
|
||||
## Categorize and report |
||||
|
||||
Open `http://127.0.0.1:8000/transactions/`, select an imported transaction, and |
||||
choose **Edit annotations**. Assign an active category or payee and save. These |
||||
POST forms use CSRF protection and record the fixed local human actor |
||||
`HUMAN/local-owner` in the audit trail. |
||||
|
||||
Open `http://127.0.0.1:8000/reports/monthly/`, select one calendar month and one |
||||
currency, then choose **Show report**. Reports use posting dates. Income is |
||||
inflow; expenses increase spending; refunds reduce spending in their posting |
||||
month. Transfers, including card-payment transfers, and opening balances are |
||||
excluded from cash flow. A card purchase is counted once as an expense. Spending |
||||
is grouped by expense category with a visible Uncategorized row, and account |
||||
balances remain separate by account and currency. |
||||
|
||||
## Export portable data |
||||
|
||||
Use `http://127.0.0.1:8000/exports/` to download a ZIP, or create one under the |
||||
ignored `exports/` directory: |
||||
|
||||
```powershell |
||||
New-Item -ItemType Directory -Force exports | Out-Null |
||||
git check-ignore exports/local-export.zip |
||||
docker compose run --rm manage export_finances --output local-export.zip |
||||
``` |
||||
|
||||
Choose a new plain `.zip` name for each run; exports never overwrite an existing |
||||
file. The ZIP contains UTF-8 CSV files for accounts, payees, categories, card |
||||
statements, installment plans, transactions, transfers, and source records. Its |
||||
`manifest.json` records schema version, creation time, row counts, and SHA-256 |
||||
checksums. Audit events and unfinished import reviews are excluded. Command |
||||
output contains only the package name, format version, and row counts. |
||||
|
||||
## Back up and verify restoration |
||||
|
||||
Create a PostgreSQL 17 custom-format dump and checksum manifest under the |
||||
ignored `backups/` directory: |
||||
|
||||
```powershell |
||||
New-Item -ItemType Directory -Force backups | Out-Null |
||||
git check-ignore backups/local-backup.dump backups/local-backup.dump.json |
||||
docker compose run --rm manage backup_database --output local-backup.dump |
||||
``` |
||||
|
||||
Choose a new plain `.dump` name for each backup. The command never overwrites an |
||||
existing dump or manifest. Its output contains only format and table counts. |
||||
The manifest records the source database, PostgreSQL client version, migrations, |
||||
row counts, byte size, and SHA-256 digest. |
||||
|
||||
The supported verification command creates synthetic data in disposable |
||||
`test-db`, backs it up, restores it into disposable `restore-db`, checks |
||||
migrations and table counts, then clears both databases. It deliberately runs |
||||
two complete cycles to prove cleanup and repeatability: |
||||
|
||||
```powershell |
||||
docker compose --profile backup run --rm verify-backup python manage.py test_backup_restore |
||||
``` |
||||
|
||||
This acceptance check never restores the live `db/gizmoduck` database. Verify a |
||||
named backup against the empty disposable restore database with: |
||||
|
||||
```powershell |
||||
$backup = 'local-backup.dump' # Replace with the exact backup name. |
||||
docker compose --profile backup run --rm verify-backup python manage.py verify_backup ` |
||||
--input $backup --target-name gizmoduck_restore ` |
||||
--target-host restore-db --target-user gizmoduck_restore |
||||
docker compose --profile backup rm -sf restore-db |
||||
``` |
||||
|
||||
The named verification checks its manifest and digest, restores it, and compares |
||||
migrations and table counts. Removing `restore-db` afterward clears only the |
||||
disposable tmpfs target so the next verification starts empty. The command |
||||
rejects the configured live host and database pair. |
||||
|
||||
## Deliberate live recovery |
||||
|
||||
Live recovery replaces financial state. Perform it only after identifying the |
||||
exact dump and its matching `.json` manifest. Keep the web service stopped until |
||||
all restore checks pass. |
||||
|
||||
First stop writes and take a separate safety backup of the current live database: |
||||
|
||||
```powershell |
||||
docker compose stop web |
||||
$safety = 'pre-restore-safety-20260908T120000Z.dump' # Use a new timestamped name. |
||||
docker compose run --rm manage backup_database --output $safety |
||||
``` |
||||
|
||||
Verify the intended named backup with the disposable procedure above before |
||||
touching the live database. Then name both the backup and live target |
||||
explicitly. The following PostgreSQL 17 command restores into `db/gizmoduck`; |
||||
`--clean` replaces objects represented by the backup, and the single transaction |
||||
rolls back on an error: |
||||
|
||||
```powershell |
||||
$backup = 'local-backup.dump' # Confirm this exact file and matching manifest first. |
||||
$targetHost = 'db' |
||||
$targetDatabase = 'gizmoduck' |
||||
docker compose run --rm -e BACKUP_FILE=$backup -e TARGET_HOST=$targetHost ` |
||||
-e TARGET_DATABASE=$targetDatabase --entrypoint sh manage -c ` |
||||
'PGPASSWORD="$DB_PASSWORD" pg_restore --exit-on-error --single-transaction --clean --if-exists --no-owner --no-acl --host "$TARGET_HOST" --port 5432 --username gizmoduck_migrate --dbname "$TARGET_DATABASE" "/app/backups/$BACKUP_FILE"' |
||||
``` |
||||
|
||||
Do not restart the web process yet. Bring schema state forward if necessary, |
||||
then validate Django and the live database connection: |
||||
|
||||
```powershell |
||||
docker compose run --rm manage migrate |
||||
docker compose run --rm manage check |
||||
docker compose run --rm manage migrate --check |
||||
docker compose run --rm manage shell -c "from django.db import connection; c=connection.cursor(); c.execute('SELECT 1'); print(c.fetchone()[0])" |
||||
``` |
||||
|
||||
Only after those checks succeed, restart the local application and verify its |
||||
readiness endpoint: |
||||
|
||||
```powershell |
||||
docker compose up -d web |
||||
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||
``` |
||||
|
||||
Keep the safety backup until the recovered data has been reviewed. If validation |
||||
fails, leave `web` stopped and investigate locally; logs and database output may |
||||
contain financial data. |
||||
@ -0,0 +1,45 @@
|
||||
# Delivery roadmap |
||||
|
||||
Each substantial subsystem receives its own design and implementation plan. |
||||
This document orders milestones; it is not a task tracker. |
||||
|
||||
## 1. Contributor foundation |
||||
|
||||
Establish agent instructions, MVP boundaries, binding architecture, and financial |
||||
data policy. Complete when the project documents are linked, consistent, and |
||||
Git exclusions protect local data while synthetic fixtures remain trackable. |
||||
|
||||
## 2. Runnable application skeleton |
||||
|
||||
Design and scaffold Django, PostgreSQL, Docker Compose, dependency tooling, |
||||
environment configuration, database roles, migrations, and pytest. Complete |
||||
when a clean checkout can follow documented setup, start the application, |
||||
apply migrations, and pass a database-backed smoke test. Use no real data. |
||||
|
||||
## 3. Financial domain |
||||
|
||||
Design accounts, credit cards, transactions, payees, categories, transfers, |
||||
refunds, installments, and provenance. Complete when migrations and tests |
||||
demonstrate the agreed financial invariants and relationships. |
||||
|
||||
## 4. Reviewed imports |
||||
|
||||
Design the normalized candidate contract and separate parser, normalization, |
||||
validation, duplicate detection, preview, and persistence stages. Implement |
||||
bank-account CSV and Banco do Brasil/Ourocard TXT using synthetic or |
||||
irreversibly sanitized fixtures. Complete when preview issues are understandable, |
||||
reimport behavior is tested, and committing an import preserves provenance |
||||
without unintended duplicates. OFX remains unsupported. |
||||
|
||||
## 5. Local MVP workflows |
||||
|
||||
Add transaction management, categorization, monthly cash-flow and spending |
||||
summaries, exports, and backups. Complete when an owner can follow the full |
||||
import-to-report workflow and verify backup restoration with synthetic data. |
||||
|
||||
**Status: complete.** The local owner workflow, portable export, PostgreSQL 17 |
||||
backup, and repeatable disposable restore path are documented and covered by the |
||||
complete synthetic acceptance suite. |
||||
|
||||
Bank APIs, investments, mobile clients, receipt recognition, forecasting, and |
||||
AI categorization require future scope decisions. See [product scope](product.md). |
||||
@ -0,0 +1,161 @@
|
||||
"""Synthetic acceptance workflow for the complete local MVP.""" |
||||
|
||||
import json |
||||
from contextlib import contextmanager |
||||
from decimal import Decimal |
||||
from pathlib import Path |
||||
from types import SimpleNamespace |
||||
from zipfile import ZipFile |
||||
|
||||
import pytest |
||||
from django.test import Client |
||||
from django.urls import reverse |
||||
|
||||
from core.backup import ( |
||||
DatabaseSnapshot, |
||||
DatabaseTarget, |
||||
create_backup, |
||||
verify_backup, |
||||
) |
||||
from finance.exports import build_export |
||||
from finance.models import SourceRecord, Transaction |
||||
from finance.services import Actor, create_account, create_category |
||||
from ingestion import services as import_services |
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True) |
||||
ACTOR = Actor(kind="AGENT", identifier="synthetic-mvp-acceptance") |
||||
FIXTURE = Path("tests/fixtures/synthetic/ingestion/bank.csv") |
||||
|
||||
|
||||
def test_synthetic_import_to_report_export_and_backup_workflow( |
||||
tmp_path, settings, monkeypatch |
||||
): |
||||
account = create_account( |
||||
name="Synthetic acceptance bank", |
||||
kind="BANK", |
||||
currency="BRL", |
||||
actor=ACTOR, |
||||
) |
||||
category = create_category( |
||||
name="Synthetic acceptance spending", |
||||
kind="EXPENSE", |
||||
actor=ACTOR, |
||||
) |
||||
batch = import_services.create_preview( |
||||
data=FIXTURE.read_bytes(), |
||||
profile="BANK_CSV_V1", |
||||
section_mapping={"1": str(account.pk)}, |
||||
actor=ACTOR, |
||||
) |
||||
assert Transaction.objects.count() == 0 |
||||
|
||||
decisions = [ |
||||
{"locator": item.locator, "action": "CREATE"} |
||||
for item in batch.items.order_by("locator") |
||||
] |
||||
batch = import_services.apply_decisions( |
||||
batch_id=batch.pk, |
||||
revision=batch.revision, |
||||
decisions=decisions, |
||||
actor=ACTOR, |
||||
) |
||||
batch = import_services.commit_preview( |
||||
batch_id=batch.pk, |
||||
revision=batch.revision, |
||||
actor=ACTOR, |
||||
) |
||||
assert batch.result["created"] == 2 |
||||
assert Transaction.objects.count() == SourceRecord.objects.count() == 2 |
||||
|
||||
expense = Transaction.objects.get(kind=Transaction.Kind.EXPENSE) |
||||
browser = Client(enforce_csrf_checks=True) |
||||
form = browser.get(reverse("transaction-annotate", args=[expense.pk])) |
||||
csrf_token = form.cookies["csrftoken"].value |
||||
response = browser.post( |
||||
reverse("transaction-annotate", args=[expense.pk]), |
||||
{ |
||||
"csrfmiddlewaretoken": csrf_token, |
||||
"description": expense.description, |
||||
"payee": "", |
||||
"category": str(category.pk), |
||||
}, |
||||
) |
||||
assert response.status_code == 302 |
||||
expense.refresh_from_db() |
||||
assert expense.category_id == category.pk |
||||
|
||||
report = browser.get( |
||||
reverse("monthly-report"), |
||||
{"month": "2026-02", "currency": "BRL"}, |
||||
) |
||||
assert report.status_code == 200 |
||||
assert report.context["summary"].inflow == Decimal("100.00") |
||||
assert report.context["summary"].expense == Decimal("12.34") |
||||
assert report.context["summary"].net_activity == Decimal("87.66") |
||||
assert report.context["spending_rows"][0].category_name == category.name |
||||
|
||||
export = build_export() |
||||
export_path = tmp_path / export.filename |
||||
export_path.write_bytes(export.content) |
||||
with ZipFile(export_path) as archive: |
||||
manifest = json.loads(archive.read("manifest.json")) |
||||
assert manifest["format"] == "gizmoduck-export" |
||||
assert manifest["files"]["transactions.csv"]["rows"] == 2 |
||||
assert manifest["files"]["source_records.csv"]["rows"] == 2 |
||||
assert category.name.encode() in archive.read("categories.csv") |
||||
|
||||
settings.BACKUP_ROOT = tmp_path |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
snapshot = DatabaseSnapshot( |
||||
migrations=(("finance", "0001_initial"), ("ingestion", "0001_initial")), |
||||
row_counts={"finance_transaction": 2, "finance_sourcerecord": 2}, |
||||
) |
||||
|
||||
@contextmanager |
||||
def exported_snapshot(_target): |
||||
yield "synthetic-mvp-snapshot", snapshot |
||||
|
||||
class Runner: |
||||
def __init__(self): |
||||
self.calls = [] |
||||
|
||||
def __call__(self, argv, **kwargs): |
||||
self.calls.append(list(argv)) |
||||
if argv[0] == "pg_dump" and argv != ["pg_dump", "--version"]: |
||||
dump_name = next( |
||||
value.removeprefix("--file=") |
||||
for value in argv |
||||
if value.startswith("--file=") |
||||
) |
||||
Path(dump_name).write_bytes(b"PGDMP synthetic acceptance") |
||||
return SimpleNamespace(stdout=b"pg_dump (PostgreSQL) 17.6\n") |
||||
|
||||
monkeypatch.setattr("core.backup._exported_snapshot", exported_snapshot) |
||||
backup_runner = Runner() |
||||
backup = create_backup("synthetic-mvp.dump", runner=backup_runner) |
||||
assert any(call[0] == "pg_dump" for call in backup_runner.calls) |
||||
assert json.loads(backup.manifest_path.read_text(encoding="utf-8"))[ |
||||
"row_counts" |
||||
] == snapshot.row_counts |
||||
|
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="gizmoduck_restore", |
||||
password="synthetic-restore-password", |
||||
) |
||||
observed = iter( |
||||
[ |
||||
DatabaseSnapshot(migrations=(), row_counts={}), |
||||
snapshot, |
||||
] |
||||
) |
||||
monkeypatch.setattr("core.backup._read_snapshot", lambda _target: next(observed)) |
||||
restore_runner = Runner() |
||||
result = verify_backup(backup.name, target, runner=restore_runner) |
||||
|
||||
assert result.row_counts == snapshot.row_counts |
||||
assert restore_runner.calls[0][0] == "pg_restore" |
||||
assert "--single-transaction" in restore_runner.calls[0] |
||||
Loading…
Reference in new issue