8 changed files with 714 additions and 7 deletions
@ -0,0 +1,97 @@ |
|||||||
|
# Python bytecode and caches |
||||||
|
__pycache__/ |
||||||
|
*.py[cod] |
||||||
|
*$py.class |
||||||
|
.pytest_cache/ |
||||||
|
.mypy_cache/ |
||||||
|
.ruff_cache/ |
||||||
|
.coverage |
||||||
|
coverage.xml |
||||||
|
htmlcov/ |
||||||
|
|
||||||
|
# Virtual environments |
||||||
|
.venv/ |
||||||
|
venv/ |
||||||
|
env/ |
||||||
|
ENV/ |
||||||
|
|
||||||
|
# Packaging and build output |
||||||
|
build/ |
||||||
|
dist/ |
||||||
|
*.egg-info/ |
||||||
|
|
||||||
|
# Environment files and secrets |
||||||
|
.env |
||||||
|
.env.* |
||||||
|
!.env.example |
||||||
|
*.pem |
||||||
|
*.key |
||||||
|
|
||||||
|
# Local databases and runtime data |
||||||
|
*.sqlite3 |
||||||
|
*.sqlite3-journal |
||||||
|
*.db |
||||||
|
data/ |
||||||
|
uploads/ |
||||||
|
media/ |
||||||
|
|
||||||
|
# Financial source files and database exports |
||||||
|
documents/ |
||||||
|
imports/ |
||||||
|
exports/ |
||||||
|
backups/ |
||||||
|
*.ofx |
||||||
|
*.qfx |
||||||
|
*.sql.gz |
||||||
|
dump.sql |
||||||
|
dump-*.sql |
||||||
|
**/dump.sql |
||||||
|
**/dump-*.sql |
||||||
|
|
||||||
|
# Logs and temporary files |
||||||
|
*.log |
||||||
|
*.tmp |
||||||
|
*.temp |
||||||
|
*.swp |
||||||
|
*~ |
||||||
|
|
||||||
|
# Jupyter |
||||||
|
.ipynb_checkpoints/ |
||||||
|
|
||||||
|
# Editors and operating systems |
||||||
|
.idea/ |
||||||
|
.vscode/ |
||||||
|
*.code-workspace |
||||||
|
.DS_Store |
||||||
|
Thumbs.db |
||||||
|
Desktop.ini |
||||||
|
|
||||||
|
# Frontend dependencies and build artifacts |
||||||
|
node_modules/ |
||||||
|
.next/ |
||||||
|
coverage/ |
||||||
|
|
||||||
|
# Docker overrides |
||||||
|
compose.override.yaml |
||||||
|
docker-compose.override.yml |
||||||
|
|
||||||
|
# Financial documents and database dumps |
||||||
|
*.csv |
||||||
|
*.txt |
||||||
|
*.pdf |
||||||
|
*.sql |
||||||
|
*.dump |
||||||
|
*.backup |
||||||
|
*.sql.bz2 |
||||||
|
*.sql.xz |
||||||
|
*.sql.zip |
||||||
|
*.zip |
||||||
|
|
||||||
|
# Explicitly synthetic fixtures and dependency manifests |
||||||
|
!tests/fixtures/synthetic/**/*.csv |
||||||
|
!tests/fixtures/synthetic/**/*.txt |
||||||
|
!tests/fixtures/synthetic/**/*.pdf |
||||||
|
!requirements*.txt |
||||||
|
|
||||||
|
# Isolated worktrees |
||||||
|
.worktrees/ |
||||||
@ -0,0 +1,102 @@ |
|||||||
|
# Gizmoduck |
||||||
|
|
||||||
|
Gizmoduck is a privacy-conscious personal finance application for importing, |
||||||
|
organizing, and understanding financial activity. It is a clean successor to |
||||||
|
the Robopato prototype, carrying forward what was learned from parsing Banco do |
||||||
|
Brasil account statements and Ourocard invoices without inheriting the |
||||||
|
prototype's notebook-driven architecture or historical data. |
||||||
|
|
||||||
|
## Goals |
||||||
|
|
||||||
|
The initial product should make it possible to: |
||||||
|
|
||||||
|
- manage bank accounts and credit cards; |
||||||
|
- import supported statement and invoice formats; |
||||||
|
- preview and validate an import before committing it; |
||||||
|
- detect duplicates and preserve import provenance; |
||||||
|
- organize transactions by payee and category; |
||||||
|
- represent transfers, refunds, and installments correctly; |
||||||
|
- inspect monthly cash flow and spending summaries; and |
||||||
|
- export and back up user-owned data. |
||||||
|
|
||||||
|
The first release is intended to be a reliable local application. Bank APIs, |
||||||
|
investment tracking, mobile clients, receipt recognition, forecasting, and AI |
||||||
|
categorization are possible later additions, not MVP requirements. |
||||||
|
|
||||||
|
## Project documentation |
||||||
|
|
||||||
|
Start with [contributor instructions](AGENTS.md), then read the |
||||||
|
[product scope](docs/product.md), [architecture](docs/architecture.md), |
||||||
|
[data policy](docs/data-policy.md), and [delivery roadmap](docs/roadmap.md). |
||||||
|
Owners can follow the copy-paste [local workflow guide](docs/local-workflows.md) |
||||||
|
from reviewed import through reports, export, and verified backup restoration. |
||||||
|
|
||||||
|
## Architecture |
||||||
|
|
||||||
|
The approved technical choices are: |
||||||
|
|
||||||
|
- Python and Django for the application; |
||||||
|
- PostgreSQL for persistent storage; |
||||||
|
- schema migrations from the first release; |
||||||
|
- Docker Compose for local infrastructure; |
||||||
|
- pytest for automated tests; and |
||||||
|
- pure, source-specific parsers feeding a normalized import pipeline. |
||||||
|
|
||||||
|
Changes require explicit owner approval and a documented architectural decision. |
||||||
|
Bank-account imports use CSV; Banco do Brasil/Ourocard credit-card imports use |
||||||
|
TXT. OFX is not supported. |
||||||
|
|
||||||
|
The import flow should remain separate from database persistence: |
||||||
|
|
||||||
|
```text |
||||||
|
source document |
||||||
|
-> source-specific parser |
||||||
|
-> normalized candidate transactions |
||||||
|
-> validation and duplicate detection |
||||||
|
-> import preview |
||||||
|
-> committed transactions |
||||||
|
-> categorization and reporting |
||||||
|
``` |
||||||
|
|
||||||
|
## Privacy and security |
||||||
|
|
||||||
|
Financial documents, database exports, credentials, and local application data |
||||||
|
must never be committed to Git. The repository `.gitignore` excludes common |
||||||
|
financial source formats and local data directories, but contributors must |
||||||
|
still inspect staged changes before every commit. |
||||||
|
|
||||||
|
Development and application processes should use separate, least-privilege |
||||||
|
database accounts. Imports and material data changes should record their actor, |
||||||
|
source, timestamp, and outcome so human, application, and automated activity can |
||||||
|
be distinguished. |
||||||
|
|
||||||
|
Use only synthetic or thoroughly sanitized financial fixtures in automated |
||||||
|
tests and documentation. |
||||||
|
|
||||||
|
## Project status |
||||||
|
|
||||||
|
The local MVP is implemented: contributor foundation, application skeleton, |
||||||
|
financial domain, reviewed imports, owner screens, monthly reports, portable |
||||||
|
exports, and verified PostgreSQL backups. The local Django application uses |
||||||
|
PostgreSQL, explicit migrations, and isolated tests. See the |
||||||
|
[financial domain guide](docs/financial-domain.md) for supported writes and |
||||||
|
accounting conventions, and [reviewed imports](docs/imports.md) for the local |
||||||
|
CSV/Ourocard preview and commit workflow. |
||||||
|
|
||||||
|
Implemented milestones: |
||||||
|
|
||||||
|
1. Define the MVP and privacy model. |
||||||
|
2. Scaffold the application, database, migrations, and test suite. |
||||||
|
3. Implement and test the core financial domain model. |
||||||
|
4. Build the import-preview pipeline using sanitized fixtures. |
||||||
|
5. Add transaction management, categorization, and basic reports. |
||||||
|
|
||||||
|
## Development |
||||||
|
|
||||||
|
Follow the [local development guide](docs/development.md) for configuration, |
||||||
|
Docker Compose startup, migrations, tests, and database-role details, then use |
||||||
|
the [local owner workflows](docs/local-workflows.md) for daily operation and |
||||||
|
recovery. |
||||||
|
|
||||||
|
The readiness endpoint is available at `http://127.0.0.1:8000/health/` after |
||||||
|
startup. Do not place real financial files inside tracked directories. |
||||||
@ -0,0 +1,86 @@ |
|||||||
|
# Local development |
||||||
|
|
||||||
|
Requires Docker Desktop running Linux containers with Docker Compose. Host |
||||||
|
Python and uv are optional; all application commands run in containers. |
||||||
|
|
||||||
|
## First start (PowerShell) |
||||||
|
|
||||||
|
Copy `.env.example` to `.env` only if `.env` does not already exist. Replace |
||||||
|
all five values with independent random secrets. To generate one in PowerShell: |
||||||
|
|
||||||
|
```powershell |
||||||
|
[Convert]::ToHexString([System.Security.Cryptography.RandomNumberGenerator]::GetBytes(32)) |
||||||
|
``` |
||||||
|
|
||||||
|
Never share `.env` or output from `docker compose config` without `--quiet`. |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose config --quiet |
||||||
|
docker compose build |
||||||
|
docker compose up -d db |
||||||
|
docker compose run --rm manage migrate |
||||||
|
docker compose up -d web |
||||||
|
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||||
|
``` |
||||||
|
|
||||||
|
The health endpoint returns `{"status":"ok"}` when PostgreSQL is reachable, |
||||||
|
and HTTP 503 with `{"status":"unavailable"}` when database access fails. |
||||||
|
It does not verify schema currency. The dashboard is available at `/`. |
||||||
|
The server binds only to loopback on the host. Ports 8000 and 5432 must be |
||||||
|
available. PostgreSQL is exposed only at `127.0.0.1:5432` for local tools; use |
||||||
|
database `gizmoduck`, runtime role `gizmoduck_app`, and the ignored `.env` value |
||||||
|
of `RUNTIME_PASSWORD` for routine inspection. This is a local development server. |
||||||
|
|
||||||
|
## Verification |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose run --rm manage check |
||||||
|
docker compose run --rm manage makemigrations --check --dry-run |
||||||
|
docker compose run --rm manage migrate --check |
||||||
|
docker compose run --rm test |
||||||
|
docker compose run --rm permissions |
||||||
|
``` |
||||||
|
|
||||||
|
The test suite covers the complete synthetic local MVP workflow. The |
||||||
|
runtime-permission check exercises DML grants on both `finance_*` and |
||||||
|
`ingestion_*` tables. See [reviewed imports](imports.md) for the explicit local |
||||||
|
preview, decision, and commit workflow, and [local owner workflows](local-workflows.md) |
||||||
|
for categorization, reports, exports, backups, and recovery. |
||||||
|
|
||||||
|
The test service starts its own PostgreSQL container. Its dedicated non-superuser |
||||||
|
role can create the temporary pytest database. The test database uses tmpfs; |
||||||
|
stopping the test database discards only its synthetic test data. The permissions |
||||||
|
command checks real runtime grants on the migrated development schema, using |
||||||
|
synthetic group rows in rolled-back transactions (sequence gaps are normal). |
||||||
|
It verifies DML, sequence access, denied schema/table creation and role escalation, |
||||||
|
and absence of database creation and administrator privileges. |
||||||
|
|
||||||
|
Runtime `gizmoduck_app` has only DML and sequence access. Management commands use |
||||||
|
`gizmoduck_migrate`, which owns the schema. Bootstrap administrator credentials |
||||||
|
are passed only to the database service. Web startup does not run migrations. |
||||||
|
|
||||||
|
## Changes and lifecycle |
||||||
|
|
||||||
|
Rebuild with `docker compose build` after editing application or test code; |
||||||
|
then use `docker compose up -d web` to replace the web container. Dependencies |
||||||
|
are installed from `uv.lock` with `uv sync --frozen`. To intentionally update |
||||||
|
that lockfile, use uv with Python 3.13 and run `uv lock`, then rebuild and verify. |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose stop web db |
||||||
|
docker compose --profile test stop test-db |
||||||
|
docker compose --profile backup stop restore-db |
||||||
|
``` |
||||||
|
|
||||||
|
Development data persists in the project-scoped `postgres_data` Docker volume. |
||||||
|
Do not delete it to fix setup problems. Database initialization scripts run only |
||||||
|
when the volume is empty. Editing `.env` does not rotate credentials stored in |
||||||
|
an existing database: coordinate an explicit database password change and update |
||||||
|
the corresponding local environment value together. Schema privilege changes |
||||||
|
also require an explicit administrator/migration action on existing volumes. |
||||||
|
Preserve or back up valuable data before any maintenance. |
||||||
|
|
||||||
|
If a startup fails, first confirm Docker is running, configuration is complete, |
||||||
|
and ports 8000 and 5432 are free. Check service status with |
||||||
|
`docker compose ps`; database or application logs may contain sensitive details |
||||||
|
and must remain local. |
||||||
@ -0,0 +1,215 @@ |
|||||||
|
# Local owner workflows |
||||||
|
|
||||||
|
This guide covers the local MVP from setup through import, categorization, |
||||||
|
reporting, export, and verified backup. Commands assume PowerShell in the |
||||||
|
repository root. Financial source files, review JSON, exports, backups, and |
||||||
|
command output that contains financial details must remain local. |
||||||
|
|
||||||
|
## Start the application |
||||||
|
|
||||||
|
Follow the secret setup in [local development](development.md), then run: |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose config --quiet |
||||||
|
docker compose build |
||||||
|
docker compose up -d db |
||||||
|
docker compose run --rm manage migrate |
||||||
|
docker compose up -d web |
||||||
|
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||||
|
``` |
||||||
|
|
||||||
|
Open `http://127.0.0.1:8000/`. The application and PostgreSQL listen only on |
||||||
|
loopback. The web UI uses port `8000`; PostgreSQL uses `127.0.0.1:5432` for |
||||||
|
local database tools such as DBeaver. The database name is `gizmoduck`. Use |
||||||
|
`gizmoduck_app` with the `RUNTIME_PASSWORD` value from the ignored `.env` for |
||||||
|
routine inspection. The migration and bootstrap roles have broader privileges |
||||||
|
and are for maintenance only. |
||||||
|
|
||||||
|
The navigation provides Accounts, Transactions, Categories, Payees, Reports, |
||||||
|
and Exports. Create accounts, categories, and payees before assigning them. |
||||||
|
References may be archived, but historical transactions continue to show them. |
||||||
|
Transaction editing is limited to description, payee, and category. Amount, |
||||||
|
dates, account, kind, import provenance, transfers, refunds, statements, and |
||||||
|
installment identity are immutable. Corrections use a new transaction or the |
||||||
|
domain workflows described in [the financial domain guide](financial-domain.md). |
||||||
|
|
||||||
|
## Import reviewed transactions |
||||||
|
|
||||||
|
Create the destination account at `http://127.0.0.1:8000/accounts/new/`. In |
||||||
|
DBeaver, retrieve its UUID without exposing transaction data: |
||||||
|
|
||||||
|
```sql |
||||||
|
SELECT id, name, kind, currency, archived |
||||||
|
FROM finance_account |
||||||
|
ORDER BY name, id; |
||||||
|
``` |
||||||
|
|
||||||
|
Copy the source document into the ignored `imports/` directory. These commands |
||||||
|
use the tracked synthetic CSV; replace `source.csv` and the UUID for a real |
||||||
|
local import. Never copy a real document into `tests/fixtures/`. |
||||||
|
|
||||||
|
```powershell |
||||||
|
New-Item -ItemType Directory -Force imports | Out-Null |
||||||
|
Copy-Item tests/fixtures/synthetic/ingestion/bank.csv imports/source.csv |
||||||
|
git check-ignore imports/source.csv imports/mapping.json imports/review-1.json |
||||||
|
$account = '11111111-1111-4111-8111-111111111111' # Replace with the account UUID. |
||||||
|
@{"1"=$account} | ConvertTo-Json | Set-Content -Encoding utf8 imports/mapping.json |
||||||
|
$actor = @('--actor-kind', 'HUMAN', '--actor-identifier', 'local-owner') |
||||||
|
docker compose run --rm manage import_review preview ` |
||||||
|
--input source.csv --profile BANK_CSV_V1 --mapping mapping.json @actor |
||||||
|
``` |
||||||
|
|
||||||
|
Copy the printed batch UUID, export the review, and inspect every candidate, |
||||||
|
issue, and duplicate suggestion: |
||||||
|
|
||||||
|
```powershell |
||||||
|
$batch = '22222222-2222-4222-8222-222222222222' # Replace with the printed batch UUID. |
||||||
|
docker compose run --rm manage import_review export ` |
||||||
|
--batch $batch --output review-1.json @actor |
||||||
|
notepad imports/review-1.json |
||||||
|
``` |
||||||
|
|
||||||
|
Edit the top-level `decisions` array. Every row needs `CREATE`, `MATCH`, or |
||||||
|
`SKIP`; use the actual locators and current revision from the review. Apply the |
||||||
|
file, export a fresh review under a new name, and commit only when all issues are |
||||||
|
resolved: |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose run --rm manage import_review apply ` |
||||||
|
--batch $batch --input review-1.json @actor |
||||||
|
docker compose run --rm manage import_review export ` |
||||||
|
--batch $batch --output review-2.json @actor |
||||||
|
notepad imports/review-2.json |
||||||
|
$revision = 2 # Replace with the revision printed by apply/export. |
||||||
|
docker compose run --rm manage import_review commit ` |
||||||
|
--batch $batch --revision $revision @actor |
||||||
|
``` |
||||||
|
|
||||||
|
Only `commit` writes transactions and provenance. Preview, export, and apply are |
||||||
|
inert. See [reviewed imports](imports.md) for all profiles, decision fields, |
||||||
|
duplicate handling, refresh, cancellation, and Ourocard date contexts. |
||||||
|
|
||||||
|
## Categorize and report |
||||||
|
|
||||||
|
Open `http://127.0.0.1:8000/transactions/`, select an imported transaction, and |
||||||
|
choose **Edit annotations**. Assign an active category or payee and save. These |
||||||
|
POST forms use CSRF protection and record the fixed local human actor |
||||||
|
`HUMAN/local-owner` in the audit trail. |
||||||
|
|
||||||
|
Open `http://127.0.0.1:8000/reports/monthly/`, select one calendar month and one |
||||||
|
currency, then choose **Show report**. Reports use posting dates. Income is |
||||||
|
inflow; expenses increase spending; refunds reduce spending in their posting |
||||||
|
month. Transfers, including card-payment transfers, and opening balances are |
||||||
|
excluded from cash flow. A card purchase is counted once as an expense. Spending |
||||||
|
is grouped by expense category with a visible Uncategorized row, and account |
||||||
|
balances remain separate by account and currency. |
||||||
|
|
||||||
|
## Export portable data |
||||||
|
|
||||||
|
Use `http://127.0.0.1:8000/exports/` to download a ZIP, or create one under the |
||||||
|
ignored `exports/` directory: |
||||||
|
|
||||||
|
```powershell |
||||||
|
New-Item -ItemType Directory -Force exports | Out-Null |
||||||
|
git check-ignore exports/local-export.zip |
||||||
|
docker compose run --rm manage export_finances --output local-export.zip |
||||||
|
``` |
||||||
|
|
||||||
|
Choose a new plain `.zip` name for each run; exports never overwrite an existing |
||||||
|
file. The ZIP contains UTF-8 CSV files for accounts, payees, categories, card |
||||||
|
statements, installment plans, transactions, transfers, and source records. Its |
||||||
|
`manifest.json` records schema version, creation time, row counts, and SHA-256 |
||||||
|
checksums. Audit events and unfinished import reviews are excluded. Command |
||||||
|
output contains only the package name, format version, and row counts. |
||||||
|
|
||||||
|
## Back up and verify restoration |
||||||
|
|
||||||
|
Create a PostgreSQL 17 custom-format dump and checksum manifest under the |
||||||
|
ignored `backups/` directory: |
||||||
|
|
||||||
|
```powershell |
||||||
|
New-Item -ItemType Directory -Force backups | Out-Null |
||||||
|
git check-ignore backups/local-backup.dump backups/local-backup.dump.json |
||||||
|
docker compose run --rm manage backup_database --output local-backup.dump |
||||||
|
``` |
||||||
|
|
||||||
|
Choose a new plain `.dump` name for each backup. The command never overwrites an |
||||||
|
existing dump or manifest. Its output contains only format and table counts. |
||||||
|
The manifest records the source database, PostgreSQL client version, migrations, |
||||||
|
row counts, byte size, and SHA-256 digest. |
||||||
|
|
||||||
|
The supported verification command creates synthetic data in disposable |
||||||
|
`test-db`, backs it up, restores it into disposable `restore-db`, checks |
||||||
|
migrations and table counts, then clears both databases. It deliberately runs |
||||||
|
two complete cycles to prove cleanup and repeatability: |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose --profile backup run --rm verify-backup python manage.py test_backup_restore |
||||||
|
``` |
||||||
|
|
||||||
|
This acceptance check never restores the live `db/gizmoduck` database. Verify a |
||||||
|
named backup against the empty disposable restore database with: |
||||||
|
|
||||||
|
```powershell |
||||||
|
$backup = 'local-backup.dump' # Replace with the exact backup name. |
||||||
|
docker compose --profile backup run --rm verify-backup python manage.py verify_backup ` |
||||||
|
--input $backup --target-name gizmoduck_restore ` |
||||||
|
--target-host restore-db --target-user gizmoduck_restore |
||||||
|
docker compose --profile backup rm -sf restore-db |
||||||
|
``` |
||||||
|
|
||||||
|
The named verification checks its manifest and digest, restores it, and compares |
||||||
|
migrations and table counts. Removing `restore-db` afterward clears only the |
||||||
|
disposable tmpfs target so the next verification starts empty. The command |
||||||
|
rejects the configured live host and database pair. |
||||||
|
|
||||||
|
## Deliberate live recovery |
||||||
|
|
||||||
|
Live recovery replaces financial state. Perform it only after identifying the |
||||||
|
exact dump and its matching `.json` manifest. Keep the web service stopped until |
||||||
|
all restore checks pass. |
||||||
|
|
||||||
|
First stop writes and take a separate safety backup of the current live database: |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose stop web |
||||||
|
$safety = 'pre-restore-safety-20260908T120000Z.dump' # Use a new timestamped name. |
||||||
|
docker compose run --rm manage backup_database --output $safety |
||||||
|
``` |
||||||
|
|
||||||
|
Verify the intended named backup with the disposable procedure above before |
||||||
|
touching the live database. Then name both the backup and live target |
||||||
|
explicitly. The following PostgreSQL 17 command restores into `db/gizmoduck`; |
||||||
|
`--clean` replaces objects represented by the backup, and the single transaction |
||||||
|
rolls back on an error: |
||||||
|
|
||||||
|
```powershell |
||||||
|
$backup = 'local-backup.dump' # Confirm this exact file and matching manifest first. |
||||||
|
$targetHost = 'db' |
||||||
|
$targetDatabase = 'gizmoduck' |
||||||
|
docker compose run --rm -e BACKUP_FILE=$backup -e TARGET_HOST=$targetHost ` |
||||||
|
-e TARGET_DATABASE=$targetDatabase --entrypoint sh manage -c ` |
||||||
|
'PGPASSWORD="$DB_PASSWORD" pg_restore --exit-on-error --single-transaction --clean --if-exists --no-owner --no-acl --host "$TARGET_HOST" --port 5432 --username gizmoduck_migrate --dbname "$TARGET_DATABASE" "/app/backups/$BACKUP_FILE"' |
||||||
|
``` |
||||||
|
|
||||||
|
Do not restart the web process yet. Bring schema state forward if necessary, |
||||||
|
then validate Django and the live database connection: |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose run --rm manage migrate |
||||||
|
docker compose run --rm manage check |
||||||
|
docker compose run --rm manage migrate --check |
||||||
|
docker compose run --rm manage shell -c "from django.db import connection; c=connection.cursor(); c.execute('SELECT 1'); print(c.fetchone()[0])" |
||||||
|
``` |
||||||
|
|
||||||
|
Only after those checks succeed, restart the local application and verify its |
||||||
|
readiness endpoint: |
||||||
|
|
||||||
|
```powershell |
||||||
|
docker compose up -d web |
||||||
|
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||||
|
``` |
||||||
|
|
||||||
|
Keep the safety backup until the recovered data has been reviewed. If validation |
||||||
|
fails, leave `web` stopped and investigate locally; logs and database output may |
||||||
|
contain financial data. |
||||||
@ -0,0 +1,45 @@ |
|||||||
|
# Delivery roadmap |
||||||
|
|
||||||
|
Each substantial subsystem receives its own design and implementation plan. |
||||||
|
This document orders milestones; it is not a task tracker. |
||||||
|
|
||||||
|
## 1. Contributor foundation |
||||||
|
|
||||||
|
Establish agent instructions, MVP boundaries, binding architecture, and financial |
||||||
|
data policy. Complete when the project documents are linked, consistent, and |
||||||
|
Git exclusions protect local data while synthetic fixtures remain trackable. |
||||||
|
|
||||||
|
## 2. Runnable application skeleton |
||||||
|
|
||||||
|
Design and scaffold Django, PostgreSQL, Docker Compose, dependency tooling, |
||||||
|
environment configuration, database roles, migrations, and pytest. Complete |
||||||
|
when a clean checkout can follow documented setup, start the application, |
||||||
|
apply migrations, and pass a database-backed smoke test. Use no real data. |
||||||
|
|
||||||
|
## 3. Financial domain |
||||||
|
|
||||||
|
Design accounts, credit cards, transactions, payees, categories, transfers, |
||||||
|
refunds, installments, and provenance. Complete when migrations and tests |
||||||
|
demonstrate the agreed financial invariants and relationships. |
||||||
|
|
||||||
|
## 4. Reviewed imports |
||||||
|
|
||||||
|
Design the normalized candidate contract and separate parser, normalization, |
||||||
|
validation, duplicate detection, preview, and persistence stages. Implement |
||||||
|
bank-account CSV and Banco do Brasil/Ourocard TXT using synthetic or |
||||||
|
irreversibly sanitized fixtures. Complete when preview issues are understandable, |
||||||
|
reimport behavior is tested, and committing an import preserves provenance |
||||||
|
without unintended duplicates. OFX remains unsupported. |
||||||
|
|
||||||
|
## 5. Local MVP workflows |
||||||
|
|
||||||
|
Add transaction management, categorization, monthly cash-flow and spending |
||||||
|
summaries, exports, and backups. Complete when an owner can follow the full |
||||||
|
import-to-report workflow and verify backup restoration with synthetic data. |
||||||
|
|
||||||
|
**Status: complete.** The local owner workflow, portable export, PostgreSQL 17 |
||||||
|
backup, and repeatable disposable restore path are documented and covered by the |
||||||
|
complete synthetic acceptance suite. |
||||||
|
|
||||||
|
Bank APIs, investments, mobile clients, receipt recognition, forecasting, and |
||||||
|
AI categorization require future scope decisions. See [product scope](product.md). |
||||||
@ -0,0 +1,161 @@ |
|||||||
|
"""Synthetic acceptance workflow for the complete local MVP.""" |
||||||
|
|
||||||
|
import json |
||||||
|
from contextlib import contextmanager |
||||||
|
from decimal import Decimal |
||||||
|
from pathlib import Path |
||||||
|
from types import SimpleNamespace |
||||||
|
from zipfile import ZipFile |
||||||
|
|
||||||
|
import pytest |
||||||
|
from django.test import Client |
||||||
|
from django.urls import reverse |
||||||
|
|
||||||
|
from core.backup import ( |
||||||
|
DatabaseSnapshot, |
||||||
|
DatabaseTarget, |
||||||
|
create_backup, |
||||||
|
verify_backup, |
||||||
|
) |
||||||
|
from finance.exports import build_export |
||||||
|
from finance.models import SourceRecord, Transaction |
||||||
|
from finance.services import Actor, create_account, create_category |
||||||
|
from ingestion import services as import_services |
||||||
|
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.django_db(transaction=True) |
||||||
|
ACTOR = Actor(kind="AGENT", identifier="synthetic-mvp-acceptance") |
||||||
|
FIXTURE = Path("tests/fixtures/synthetic/ingestion/bank.csv") |
||||||
|
|
||||||
|
|
||||||
|
def test_synthetic_import_to_report_export_and_backup_workflow( |
||||||
|
tmp_path, settings, monkeypatch |
||||||
|
): |
||||||
|
account = create_account( |
||||||
|
name="Synthetic acceptance bank", |
||||||
|
kind="BANK", |
||||||
|
currency="BRL", |
||||||
|
actor=ACTOR, |
||||||
|
) |
||||||
|
category = create_category( |
||||||
|
name="Synthetic acceptance spending", |
||||||
|
kind="EXPENSE", |
||||||
|
actor=ACTOR, |
||||||
|
) |
||||||
|
batch = import_services.create_preview( |
||||||
|
data=FIXTURE.read_bytes(), |
||||||
|
profile="BANK_CSV_V1", |
||||||
|
section_mapping={"1": str(account.pk)}, |
||||||
|
actor=ACTOR, |
||||||
|
) |
||||||
|
assert Transaction.objects.count() == 0 |
||||||
|
|
||||||
|
decisions = [ |
||||||
|
{"locator": item.locator, "action": "CREATE"} |
||||||
|
for item in batch.items.order_by("locator") |
||||||
|
] |
||||||
|
batch = import_services.apply_decisions( |
||||||
|
batch_id=batch.pk, |
||||||
|
revision=batch.revision, |
||||||
|
decisions=decisions, |
||||||
|
actor=ACTOR, |
||||||
|
) |
||||||
|
batch = import_services.commit_preview( |
||||||
|
batch_id=batch.pk, |
||||||
|
revision=batch.revision, |
||||||
|
actor=ACTOR, |
||||||
|
) |
||||||
|
assert batch.result["created"] == 2 |
||||||
|
assert Transaction.objects.count() == SourceRecord.objects.count() == 2 |
||||||
|
|
||||||
|
expense = Transaction.objects.get(kind=Transaction.Kind.EXPENSE) |
||||||
|
browser = Client(enforce_csrf_checks=True) |
||||||
|
form = browser.get(reverse("transaction-annotate", args=[expense.pk])) |
||||||
|
csrf_token = form.cookies["csrftoken"].value |
||||||
|
response = browser.post( |
||||||
|
reverse("transaction-annotate", args=[expense.pk]), |
||||||
|
{ |
||||||
|
"csrfmiddlewaretoken": csrf_token, |
||||||
|
"description": expense.description, |
||||||
|
"payee": "", |
||||||
|
"category": str(category.pk), |
||||||
|
}, |
||||||
|
) |
||||||
|
assert response.status_code == 302 |
||||||
|
expense.refresh_from_db() |
||||||
|
assert expense.category_id == category.pk |
||||||
|
|
||||||
|
report = browser.get( |
||||||
|
reverse("monthly-report"), |
||||||
|
{"month": "2026-02", "currency": "BRL"}, |
||||||
|
) |
||||||
|
assert report.status_code == 200 |
||||||
|
assert report.context["summary"].inflow == Decimal("100.00") |
||||||
|
assert report.context["summary"].expense == Decimal("12.34") |
||||||
|
assert report.context["summary"].net_activity == Decimal("87.66") |
||||||
|
assert report.context["spending_rows"][0].category_name == category.name |
||||||
|
|
||||||
|
export = build_export() |
||||||
|
export_path = tmp_path / export.filename |
||||||
|
export_path.write_bytes(export.content) |
||||||
|
with ZipFile(export_path) as archive: |
||||||
|
manifest = json.loads(archive.read("manifest.json")) |
||||||
|
assert manifest["format"] == "gizmoduck-export" |
||||||
|
assert manifest["files"]["transactions.csv"]["rows"] == 2 |
||||||
|
assert manifest["files"]["source_records.csv"]["rows"] == 2 |
||||||
|
assert category.name.encode() in archive.read("categories.csv") |
||||||
|
|
||||||
|
settings.BACKUP_ROOT = tmp_path |
||||||
|
settings.BACKUP_VERIFY_ALLOWED = True |
||||||
|
snapshot = DatabaseSnapshot( |
||||||
|
migrations=(("finance", "0001_initial"), ("ingestion", "0001_initial")), |
||||||
|
row_counts={"finance_transaction": 2, "finance_sourcerecord": 2}, |
||||||
|
) |
||||||
|
|
||||||
|
@contextmanager |
||||||
|
def exported_snapshot(_target): |
||||||
|
yield "synthetic-mvp-snapshot", snapshot |
||||||
|
|
||||||
|
class Runner: |
||||||
|
def __init__(self): |
||||||
|
self.calls = [] |
||||||
|
|
||||||
|
def __call__(self, argv, **kwargs): |
||||||
|
self.calls.append(list(argv)) |
||||||
|
if argv[0] == "pg_dump" and argv != ["pg_dump", "--version"]: |
||||||
|
dump_name = next( |
||||||
|
value.removeprefix("--file=") |
||||||
|
for value in argv |
||||||
|
if value.startswith("--file=") |
||||||
|
) |
||||||
|
Path(dump_name).write_bytes(b"PGDMP synthetic acceptance") |
||||||
|
return SimpleNamespace(stdout=b"pg_dump (PostgreSQL) 17.6\n") |
||||||
|
|
||||||
|
monkeypatch.setattr("core.backup._exported_snapshot", exported_snapshot) |
||||||
|
backup_runner = Runner() |
||||||
|
backup = create_backup("synthetic-mvp.dump", runner=backup_runner) |
||||||
|
assert any(call[0] == "pg_dump" for call in backup_runner.calls) |
||||||
|
assert json.loads(backup.manifest_path.read_text(encoding="utf-8"))[ |
||||||
|
"row_counts" |
||||||
|
] == snapshot.row_counts |
||||||
|
|
||||||
|
target = DatabaseTarget( |
||||||
|
host="restore-db", |
||||||
|
port="5432", |
||||||
|
name="gizmoduck_restore", |
||||||
|
user="gizmoduck_restore", |
||||||
|
password="synthetic-restore-password", |
||||||
|
) |
||||||
|
observed = iter( |
||||||
|
[ |
||||||
|
DatabaseSnapshot(migrations=(), row_counts={}), |
||||||
|
snapshot, |
||||||
|
] |
||||||
|
) |
||||||
|
monkeypatch.setattr("core.backup._read_snapshot", lambda _target: next(observed)) |
||||||
|
restore_runner = Runner() |
||||||
|
result = verify_backup(backup.name, target, runner=restore_runner) |
||||||
|
|
||||||
|
assert result.row_counts == snapshot.row_counts |
||||||
|
assert restore_runner.calls[0][0] == "pg_restore" |
||||||
|
assert "--single-transaction" in restore_runner.calls[0] |
||||||
Loading…
Reference in new issue