Compare commits
18 Commits
6ae28aa5b1
...
3e6d4f8e81
| Author | SHA1 | Date |
|---|---|---|
|
|
3e6d4f8e81 | 2 months ago |
|
|
d01e8918c0 | 2 months ago |
|
|
06bfe5838a | 2 months ago |
|
|
4791abd043 | 2 months ago |
|
|
b83d9973b3 | 2 months ago |
|
|
bea0531a95 | 2 months ago |
|
|
0d8f378904 | 2 months ago |
|
|
135de1d5c8 | 2 months ago |
|
|
7538e739b9 | 2 months ago |
|
|
92bc3820ef | 2 months ago |
|
|
fc3aa51782 | 2 months ago |
|
|
b00c81779a | 2 months ago |
|
|
31f6ea4bd5 | 2 months ago |
|
|
c9331c0315 | 2 months ago |
|
|
6b44ad053e | 2 months ago |
|
|
ed008bee09 | 2 months ago |
|
|
dd58866dbe | 2 months ago |
|
|
d3eb9a17fc | 2 months ago |
113 changed files with 10069 additions and 0 deletions
@ -0,0 +1,18 @@
|
||||
* |
||||
!Dockerfile |
||||
!pyproject.toml |
||||
!uv.lock |
||||
!manage.py |
||||
!config/ |
||||
!config/** |
||||
!core/ |
||||
!core/** |
||||
!tests/ |
||||
!tests/** |
||||
**/__pycache__/ |
||||
**/*.pyc |
||||
.env* |
||||
!finance/ |
||||
!finance/** |
||||
!ingestion/ |
||||
!ingestion/** |
||||
@ -0,0 +1,6 @@
|
||||
# Copy to .env and replace all values with independently generated secrets. |
||||
DJANGO_SECRET_KEY=replace-with-random-secret |
||||
POSTGRES_PASSWORD=replace-with-bootstrap-secret |
||||
MIGRATION_PASSWORD=replace-with-migration-secret |
||||
RUNTIME_PASSWORD=replace-with-runtime-secret |
||||
TEST_PASSWORD=replace-with-test-secret |
||||
@ -0,0 +1,97 @@
|
||||
# Python bytecode and caches |
||||
__pycache__/ |
||||
*.py[cod] |
||||
*$py.class |
||||
.pytest_cache/ |
||||
.mypy_cache/ |
||||
.ruff_cache/ |
||||
.coverage |
||||
coverage.xml |
||||
htmlcov/ |
||||
|
||||
# Virtual environments |
||||
.venv/ |
||||
venv/ |
||||
env/ |
||||
ENV/ |
||||
|
||||
# Packaging and build output |
||||
build/ |
||||
dist/ |
||||
*.egg-info/ |
||||
|
||||
# Environment files and secrets |
||||
.env |
||||
.env.* |
||||
!.env.example |
||||
*.pem |
||||
*.key |
||||
|
||||
# Local databases and runtime data |
||||
*.sqlite3 |
||||
*.sqlite3-journal |
||||
*.db |
||||
data/ |
||||
uploads/ |
||||
media/ |
||||
|
||||
# Financial source files and database exports |
||||
documents/ |
||||
imports/ |
||||
exports/ |
||||
backups/ |
||||
*.ofx |
||||
*.qfx |
||||
*.sql.gz |
||||
dump.sql |
||||
dump-*.sql |
||||
**/dump.sql |
||||
**/dump-*.sql |
||||
|
||||
# Logs and temporary files |
||||
*.log |
||||
*.tmp |
||||
*.temp |
||||
*.swp |
||||
*~ |
||||
|
||||
# Jupyter |
||||
.ipynb_checkpoints/ |
||||
|
||||
# Editors and operating systems |
||||
.idea/ |
||||
.vscode/ |
||||
*.code-workspace |
||||
.DS_Store |
||||
Thumbs.db |
||||
Desktop.ini |
||||
|
||||
# Frontend dependencies and build artifacts |
||||
node_modules/ |
||||
.next/ |
||||
coverage/ |
||||
|
||||
# Docker overrides |
||||
compose.override.yaml |
||||
docker-compose.override.yml |
||||
|
||||
# Financial documents and database dumps |
||||
*.csv |
||||
*.txt |
||||
*.pdf |
||||
*.sql |
||||
*.dump |
||||
*.backup |
||||
*.sql.bz2 |
||||
*.sql.xz |
||||
*.sql.zip |
||||
*.zip |
||||
|
||||
# Explicitly synthetic fixtures and dependency manifests |
||||
!tests/fixtures/synthetic/**/*.csv |
||||
!tests/fixtures/synthetic/**/*.txt |
||||
!tests/fixtures/synthetic/**/*.pdf |
||||
!requirements*.txt |
||||
|
||||
# Isolated worktrees |
||||
.worktrees/ |
||||
@ -0,0 +1,40 @@
|
||||
# Contributor instructions |
||||
|
||||
Before substantive work, read [product scope](docs/product.md), |
||||
[architecture](docs/architecture.md), [data policy](docs/data-policy.md), and |
||||
[roadmap](docs/roadmap.md). This file is the authoritative agent entry point. |
||||
|
||||
## Binding decisions |
||||
|
||||
- Gizmoduck is the clean successor to Robopato. |
||||
- Use Python, Django, PostgreSQL, migrations from the beginning, Docker Compose |
||||
for local infrastructure, and pytest for automated tests. |
||||
- Import bank accounts from CSV and Banco do Brasil/Ourocard credit cards from |
||||
TXT. OFX is not supported. |
||||
- Keep parsing, normalization, validation, duplicate detection, preview, and |
||||
persistence separate. |
||||
- Obtain owner approval before changing these constraints or product scope. |
||||
|
||||
## Working rules |
||||
|
||||
- Preserve user changes and avoid unrelated edits. |
||||
- Make small, cohesive changes with verification appropriate to their risk. |
||||
- Use isolated Git worktrees for concurrent implementation that could overlap. |
||||
- Design and plan substantial subsystems before implementing them. |
||||
- Report assumptions, checks performed, and unresolved risks accurately. |
||||
- Do not perform destructive operations, pushes, merges, releases, or history |
||||
rewrites without explicit authorization. |
||||
- Focused local commits are allowed after verification. Inspect both staged |
||||
paths and staged content for real financial data and secrets before every |
||||
commit. When uncertain whether data is safe, stop and ask the owner. |
||||
- Agent-authored commits use repository-local `Codex <codex@localhost>`; |
||||
human-authored commits retain the human contributor's identity. |
||||
|
||||
## Financial data |
||||
|
||||
Agents may read every file in Robopato and copy real documents for local |
||||
development, parser validation, migration, and compatibility testing. Store |
||||
real data only in ignored local directories or local databases. Never commit |
||||
raw data, identifying excerpts, snapshots, logs, screenshots, reports, or dumps. |
||||
Committed fixtures must be synthetic or irreversibly sanitized. Follow the |
||||
[full data policy](docs/data-policy.md) before handling source documents. |
||||
@ -0,0 +1,26 @@
|
||||
FROM python:3.13-slim |
||||
COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /usr/local/bin/uv |
||||
RUN apt-get update \ |
||||
&& apt-get install -y --no-install-recommends ca-certificates curl \ |
||||
&& install -d /usr/share/postgresql-common/pgdg \ |
||||
&& curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ |
||||
-o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ |
||||
&& . /etc/os-release \ |
||||
&& echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" \ |
||||
> /etc/apt/sources.list.d/pgdg.list \ |
||||
&& apt-get update \ |
||||
&& apt-get install -y --no-install-recommends postgresql-client-17 \ |
||||
&& rm -rf /var/lib/apt/lists/* |
||||
WORKDIR /app |
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PATH="/app/.venv/bin:$PATH" |
||||
COPY pyproject.toml uv.lock ./ |
||||
RUN uv sync --frozen --no-install-project |
||||
COPY config ./config |
||||
COPY core ./core |
||||
COPY finance ./finance |
||||
COPY ingestion ./ingestion |
||||
COPY tests ./tests |
||||
COPY manage.py ./ |
||||
RUN useradd --uid 10001 --create-home app |
||||
USER app |
||||
CMD ["python", "manage.py", "runserver", "0.0.0.0:8000", "--noreload"] |
||||
@ -0,0 +1,102 @@
|
||||
# Gizmoduck |
||||
|
||||
Gizmoduck is a privacy-conscious personal finance application for importing, |
||||
organizing, and understanding financial activity. It is a clean successor to |
||||
the Robopato prototype, carrying forward what was learned from parsing Banco do |
||||
Brasil account statements and Ourocard invoices without inheriting the |
||||
prototype's notebook-driven architecture or historical data. |
||||
|
||||
## Goals |
||||
|
||||
The initial product should make it possible to: |
||||
|
||||
- manage bank accounts and credit cards; |
||||
- import supported statement and invoice formats; |
||||
- preview and validate an import before committing it; |
||||
- detect duplicates and preserve import provenance; |
||||
- organize transactions by payee and category; |
||||
- represent transfers, refunds, and installments correctly; |
||||
- inspect monthly cash flow and spending summaries; and |
||||
- export and back up user-owned data. |
||||
|
||||
The first release is intended to be a reliable local application. Bank APIs, |
||||
investment tracking, mobile clients, receipt recognition, forecasting, and AI |
||||
categorization are possible later additions, not MVP requirements. |
||||
|
||||
## Project documentation |
||||
|
||||
Start with [contributor instructions](AGENTS.md), then read the |
||||
[product scope](docs/product.md), [architecture](docs/architecture.md), |
||||
[data policy](docs/data-policy.md), and [delivery roadmap](docs/roadmap.md). |
||||
Owners can follow the copy-paste [local workflow guide](docs/local-workflows.md) |
||||
from reviewed import through reports, export, and verified backup restoration. |
||||
|
||||
## Architecture |
||||
|
||||
The approved technical choices are: |
||||
|
||||
- Python and Django for the application; |
||||
- PostgreSQL for persistent storage; |
||||
- schema migrations from the first release; |
||||
- Docker Compose for local infrastructure; |
||||
- pytest for automated tests; and |
||||
- pure, source-specific parsers feeding a normalized import pipeline. |
||||
|
||||
Changes require explicit owner approval and a documented architectural decision. |
||||
Bank-account imports use CSV; Banco do Brasil/Ourocard credit-card imports use |
||||
TXT. OFX is not supported. |
||||
|
||||
The import flow should remain separate from database persistence: |
||||
|
||||
```text |
||||
source document |
||||
-> source-specific parser |
||||
-> normalized candidate transactions |
||||
-> validation and duplicate detection |
||||
-> import preview |
||||
-> committed transactions |
||||
-> categorization and reporting |
||||
``` |
||||
|
||||
## Privacy and security |
||||
|
||||
Financial documents, database exports, credentials, and local application data |
||||
must never be committed to Git. The repository `.gitignore` excludes common |
||||
financial source formats and local data directories, but contributors must |
||||
still inspect staged changes before every commit. |
||||
|
||||
Development and application processes should use separate, least-privilege |
||||
database accounts. Imports and material data changes should record their actor, |
||||
source, timestamp, and outcome so human, application, and automated activity can |
||||
be distinguished. |
||||
|
||||
Use only synthetic or thoroughly sanitized financial fixtures in automated |
||||
tests and documentation. |
||||
|
||||
## Project status |
||||
|
||||
The local MVP is implemented: contributor foundation, application skeleton, |
||||
financial domain, reviewed imports, owner screens, monthly reports, portable |
||||
exports, and verified PostgreSQL backups. The local Django application uses |
||||
PostgreSQL, explicit migrations, and isolated tests. See the |
||||
[financial domain guide](docs/financial-domain.md) for supported writes and |
||||
accounting conventions, and [reviewed imports](docs/imports.md) for the local |
||||
CSV/Ourocard preview and commit workflow. |
||||
|
||||
Implemented milestones: |
||||
|
||||
1. Define the MVP and privacy model. |
||||
2. Scaffold the application, database, migrations, and test suite. |
||||
3. Implement and test the core financial domain model. |
||||
4. Build the import-preview pipeline using sanitized fixtures. |
||||
5. Add transaction management, categorization, and basic reports. |
||||
|
||||
## Development |
||||
|
||||
Follow the [local development guide](docs/development.md) for configuration, |
||||
Docker Compose startup, migrations, tests, and database-role details, then use |
||||
the [local owner workflows](docs/local-workflows.md) for daily operation and |
||||
recovery. |
||||
|
||||
The readiness endpoint is available at `http://127.0.0.1:8000/health/` after |
||||
startup. Do not place real financial files inside tracked directories. |
||||
@ -0,0 +1,119 @@
|
||||
name: gizmoduck |
||||
x-app: &app |
||||
build: . |
||||
image: gizmoduck-local |
||||
x-runtime: &runtime |
||||
DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:?Set DJANGO_SECRET_KEY in .env} |
||||
DB_NAME: gizmoduck |
||||
DB_HOST: db |
||||
DB_USER: gizmoduck_app |
||||
DB_PASSWORD: ${RUNTIME_PASSWORD:?Set RUNTIME_PASSWORD in .env} |
||||
services: |
||||
db: |
||||
image: postgres:17 |
||||
environment: |
||||
POSTGRES_DB: gizmoduck |
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} |
||||
MIGRATION_PASSWORD: ${MIGRATION_PASSWORD:?Set MIGRATION_PASSWORD in .env} |
||||
RUNTIME_PASSWORD: ${RUNTIME_PASSWORD:?Set RUNTIME_PASSWORD in .env} |
||||
volumes: |
||||
- postgres_data:/var/lib/postgresql/data |
||||
- ./docker/postgres/init.sh:/docker-entrypoint-initdb.d/10-roles.sh:ro |
||||
healthcheck: |
||||
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres -d gizmoduck"] |
||||
interval: 2s |
||||
timeout: 3s |
||||
retries: 30 |
||||
ports: |
||||
- "127.0.0.1:5432:5432" |
||||
web: |
||||
<<: *app |
||||
environment: *runtime |
||||
ports: ["127.0.0.1:8000:8000"] |
||||
depends_on: |
||||
db: {condition: service_healthy} |
||||
manage: |
||||
<<: *app |
||||
profiles: ["tools"] |
||||
environment: |
||||
<<: *runtime |
||||
DB_USER: gizmoduck_migrate |
||||
DB_PASSWORD: ${MIGRATION_PASSWORD:?Set MIGRATION_PASSWORD in .env} |
||||
volumes: |
||||
- ./imports:/app/imports |
||||
- ./exports:/app/exports |
||||
- ./backups:/app/backups |
||||
entrypoint: ["python", "manage.py"] |
||||
command: ["check"] |
||||
depends_on: |
||||
db: {condition: service_healthy} |
||||
test-db: |
||||
image: postgres:17 |
||||
profiles: ["test", "backup"] |
||||
environment: |
||||
POSTGRES_DB: gizmoduck_tests |
||||
POSTGRES_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} |
||||
tmpfs: [/var/lib/postgresql/data] |
||||
volumes: |
||||
- ./docker/postgres/test-init.sh:/docker-entrypoint-initdb.d/10-test.sh:ro |
||||
healthcheck: |
||||
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres -d gizmoduck_tests"] |
||||
interval: 2s |
||||
timeout: 3s |
||||
retries: 30 |
||||
test: |
||||
<<: *app |
||||
profiles: ["test"] |
||||
environment: |
||||
DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:?Set DJANGO_SECRET_KEY in .env} |
||||
DB_HOST: test-db |
||||
DB_NAME: gizmoduck_tests |
||||
DB_USER: gizmoduck_test |
||||
DB_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} |
||||
command: ["pytest", "-q", "-p", "no:cacheprovider"] |
||||
depends_on: |
||||
test-db: {condition: service_healthy} |
||||
permissions: |
||||
<<: *app |
||||
profiles: ["tools"] |
||||
environment: *runtime |
||||
command: ["python", "tests/check_permissions.py"] |
||||
depends_on: |
||||
db: {condition: service_healthy} |
||||
restore-db: |
||||
image: postgres:17 |
||||
profiles: ["backup"] |
||||
environment: |
||||
POSTGRES_DB: gizmoduck_restore |
||||
POSTGRES_USER: gizmoduck_restore |
||||
POSTGRES_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} |
||||
tmpfs: [/var/lib/postgresql/data] |
||||
healthcheck: |
||||
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U gizmoduck_restore -d gizmoduck_restore"] |
||||
interval: 2s |
||||
timeout: 3s |
||||
retries: 30 |
||||
verify-backup: |
||||
<<: *app |
||||
profiles: ["backup"] |
||||
environment: |
||||
DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:?Set DJANGO_SECRET_KEY in .env} |
||||
DB_HOST: restore-db |
||||
DB_NAME: gizmoduck_restore |
||||
DB_USER: gizmoduck_restore |
||||
DB_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} |
||||
BACKUP_ROOT: /app/backups |
||||
BACKUP_VERIFY_ALLOWED: "1" |
||||
LIVE_DB_HOST: db |
||||
LIVE_DB_NAME: gizmoduck |
||||
BACKUP_SOURCE_HOST: test-db |
||||
BACKUP_SOURCE_NAME: gizmoduck_tests |
||||
BACKUP_SOURCE_USER: gizmoduck_test |
||||
BACKUP_SOURCE_PASSWORD: ${TEST_PASSWORD:?Set TEST_PASSWORD in .env} |
||||
volumes: |
||||
- ./backups:/app/backups |
||||
depends_on: |
||||
restore-db: {condition: service_healthy} |
||||
test-db: {condition: service_healthy} |
||||
volumes: |
||||
postgres_data: |
||||
@ -0,0 +1,4 @@
|
||||
import os |
||||
from django.core.asgi import get_asgi_application |
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") |
||||
application = get_asgi_application() |
||||
@ -0,0 +1,60 @@
|
||||
import os |
||||
from django.core.exceptions import ImproperlyConfigured |
||||
|
||||
|
||||
def required(name): |
||||
value = os.environ.get(name) |
||||
if not value or value.startswith("replace-"): |
||||
raise ImproperlyConfigured(f"Set {name} in your local environment.") |
||||
return value |
||||
|
||||
|
||||
SECRET_KEY = required("DJANGO_SECRET_KEY") |
||||
DEBUG = False |
||||
ALLOWED_HOSTS = ["localhost", "127.0.0.1"] |
||||
ROOT_URLCONF = "config.urls" |
||||
INSTALLED_APPS = [ |
||||
"django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", |
||||
"django.contrib.messages", "django.contrib.staticfiles", "core", "finance", "ingestion", |
||||
] |
||||
MIDDLEWARE = [ |
||||
"django.middleware.security.SecurityMiddleware", |
||||
"django.contrib.sessions.middleware.SessionMiddleware", |
||||
"django.middleware.common.CommonMiddleware", |
||||
"django.middleware.csrf.CsrfViewMiddleware", |
||||
"django.contrib.messages.middleware.MessageMiddleware", |
||||
] |
||||
TEMPLATES = [{ |
||||
"BACKEND": "django.template.backends.django.DjangoTemplates", |
||||
"DIRS": [], |
||||
"APP_DIRS": True, |
||||
"OPTIONS": {"context_processors": [ |
||||
"django.template.context_processors.request", |
||||
"django.contrib.messages.context_processors.messages", |
||||
"core.context.navigation", |
||||
]}, |
||||
}] |
||||
SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies" |
||||
STATIC_URL = "static/" |
||||
LOCAL_SERVE_STATIC = os.environ.get("LOCAL_SERVE_STATIC", "true").casefold() in {"1", "true", "yes"} |
||||
DATABASES = {"default": { |
||||
"ENGINE": "django.db.backends.postgresql", |
||||
"NAME": required("DB_NAME"), |
||||
"USER": required("DB_USER"), |
||||
"PASSWORD": required("DB_PASSWORD"), |
||||
"HOST": required("DB_HOST"), |
||||
"PORT": "5432", |
||||
"OPTIONS": {"connect_timeout": 3}, |
||||
}} |
||||
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" |
||||
USE_TZ = True |
||||
TIME_ZONE = "UTC" |
||||
|
||||
IMPORT_ROOT = "/app/imports" |
||||
EXPORT_ROOT = os.environ.get("EXPORT_ROOT", "/app/exports") |
||||
BACKUP_ROOT = os.environ.get("BACKUP_ROOT", "/app/backups") |
||||
BACKUP_VERIFY_ALLOWED = os.environ.get("BACKUP_VERIFY_ALLOWED", "").casefold() in { |
||||
"1", "true", "yes" |
||||
} |
||||
LIVE_DATABASE_HOST = os.environ.get("LIVE_DB_HOST") |
||||
LIVE_DATABASE_NAME = os.environ.get("LIVE_DB_NAME") |
||||
@ -0,0 +1,30 @@
|
||||
from django.conf import settings |
||||
from django.contrib.staticfiles.views import serve as serve_static |
||||
from django.urls import path, re_path |
||||
from core import views |
||||
|
||||
urlpatterns = [ |
||||
path("", views.home, name="home"), |
||||
path("health/", views.health, name="health"), |
||||
path("reports/monthly/", views.monthly_report, name="monthly-report"), |
||||
path("exports/", views.export_index, name="export-index"), |
||||
path("exports/download/", views.export_download, name="export-download"), |
||||
path("transactions/", views.transaction_list, name="transaction-list"), |
||||
path("transactions/new/", views.transaction_create, name="transaction-create"), |
||||
path("transactions/<uuid:transaction_id>/", views.transaction_detail, name="transaction-detail"), |
||||
path("transactions/<uuid:transaction_id>/annotate/", views.transaction_annotate, name="transaction-annotate"), |
||||
path("accounts/", views.account_list, name="account-list"), |
||||
path("accounts/new/", views.account_create, name="account-create"), |
||||
path("accounts/<uuid:reference_id>/archive/", views.account_archive, name="account-archive"), |
||||
path("categories/", views.category_list, name="category-list"), |
||||
path("categories/new/", views.category_create, name="category-create"), |
||||
path("categories/<uuid:reference_id>/archive/", views.category_archive, name="category-archive"), |
||||
path("payees/", views.payee_list, name="payee-list"), |
||||
path("payees/new/", views.payee_create, name="payee-create"), |
||||
path("payees/<uuid:reference_id>/archive/", views.payee_archive, name="payee-archive"), |
||||
] |
||||
|
||||
if settings.LOCAL_SERVE_STATIC: |
||||
urlpatterns.append( |
||||
re_path(r"^static/(?P<path>.*)$", serve_static, {"insecure": True}, name="local-static") |
||||
) |
||||
@ -0,0 +1,4 @@
|
||||
import os |
||||
from django.core.wsgi import get_wsgi_application |
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") |
||||
application = get_wsgi_application() |
||||
@ -0,0 +1,699 @@
|
||||
import json |
||||
import os |
||||
import re |
||||
import subprocess |
||||
import tempfile |
||||
from contextlib import contextmanager |
||||
from dataclasses import dataclass |
||||
from datetime import datetime, timezone as datetime_timezone |
||||
from hashlib import sha256 |
||||
from pathlib import Path |
||||
|
||||
import psycopg |
||||
from psycopg import sql |
||||
from django.conf import settings |
||||
|
||||
|
||||
BACKUP_FORMAT = "gizmoduck-postgresql-backup" |
||||
BACKUP_VERSION = 1 |
||||
APPLICATION_VERSION = "0.1.0" |
||||
|
||||
_PLAIN_DUMP_NAME = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9._-]{0,198}\.dump\Z") |
||||
_TABLE_NAME = re.compile(r"\A[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?\Z") |
||||
_MANIFEST_KEYS = { |
||||
"application_version", |
||||
"byte_size", |
||||
"created_at", |
||||
"database_name", |
||||
"format", |
||||
"migrations", |
||||
"postgresql_client_version", |
||||
"row_counts", |
||||
"sha256", |
||||
"version", |
||||
} |
||||
|
||||
|
||||
class BackupError(Exception): |
||||
def __init__(self, code): |
||||
self.code = code |
||||
super().__init__(code) |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class DatabaseTarget: |
||||
host: str |
||||
port: str |
||||
name: str |
||||
user: str |
||||
password: str |
||||
|
||||
@classmethod |
||||
def from_default(cls, database): |
||||
return cls( |
||||
host=str(database.get("HOST", "")), |
||||
port=str(database.get("PORT") or "5432"), |
||||
name=str(database.get("NAME", "")), |
||||
user=str(database.get("USER", "")), |
||||
password=str(database.get("PASSWORD", "")), |
||||
) |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class DatabaseSnapshot: |
||||
migrations: tuple[tuple[str, str], ...] |
||||
row_counts: dict[str, int] |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class BackupResult: |
||||
name: str |
||||
path: Path |
||||
manifest_path: Path |
||||
version: int |
||||
created_at: datetime |
||||
postgresql_client_version: str |
||||
row_counts: dict[str, int] |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class VerificationResult: |
||||
name: str |
||||
version: int |
||||
row_counts: dict[str, int] |
||||
|
||||
|
||||
def create_backup(name=None, *, runner=None, _target=None): |
||||
runner = runner or subprocess.run |
||||
root = _backup_root() |
||||
if name is None: |
||||
now = datetime.now(datetime_timezone.utc) |
||||
name = f"gizmoduck-backup-{now:%Y%m%dT%H%M%SZ}.dump" |
||||
else: |
||||
now = datetime.now(datetime_timezone.utc) |
||||
_validate_name(name) |
||||
|
||||
destination = root / name |
||||
manifest_destination = root / f"{name}.json" |
||||
target = _target or DatabaseTarget.from_default(settings.DATABASES["default"]) |
||||
dump_path = None |
||||
manifest_path = None |
||||
try: |
||||
_ensure_available(destination, manifest_destination) |
||||
dump_fd, dump_name = tempfile.mkstemp( |
||||
prefix=".gizmoduck-backup-", suffix=".dump.tmp", dir=root |
||||
) |
||||
dump_path = Path(dump_name) |
||||
os.close(dump_fd) |
||||
with _exported_snapshot(target) as (snapshot_id, snapshot): |
||||
runner( |
||||
_pg_dump_argv(target, dump_path, snapshot_id), |
||||
env=_database_env(target), |
||||
check=True, |
||||
stdout=None, |
||||
capture_output=True, |
||||
) |
||||
with dump_path.open("rb") as stream: |
||||
os.fsync(stream.fileno()) |
||||
|
||||
content_digest, byte_size = _file_digest(dump_path) |
||||
version_process = runner( |
||||
["pg_dump", "--version"], |
||||
env=_database_env(target), |
||||
check=True, |
||||
stdout=None, |
||||
capture_output=True, |
||||
) |
||||
client_version = _safe_client_version(getattr(version_process, "stdout", b"")) |
||||
manifest = { |
||||
"application_version": APPLICATION_VERSION, |
||||
"byte_size": byte_size, |
||||
"created_at": now.isoformat(), |
||||
"database_name": target.name, |
||||
"format": BACKUP_FORMAT, |
||||
"migrations": [list(item) for item in snapshot.migrations], |
||||
"postgresql_client_version": client_version, |
||||
"row_counts": snapshot.row_counts, |
||||
"sha256": content_digest, |
||||
"version": BACKUP_VERSION, |
||||
} |
||||
manifest_fd, manifest_name = tempfile.mkstemp( |
||||
prefix=".gizmoduck-backup-", suffix=".json.tmp", dir=root |
||||
) |
||||
manifest_path = Path(manifest_name) |
||||
with os.fdopen(manifest_fd, "w", encoding="utf-8", newline="\n") as stream: |
||||
json.dump(manifest, stream, ensure_ascii=False, separators=(",", ":")) |
||||
stream.write("\n") |
||||
stream.flush() |
||||
os.fsync(stream.fileno()) |
||||
|
||||
_publish_pair( |
||||
root, |
||||
((manifest_path, manifest_destination), (dump_path, destination)), |
||||
) |
||||
manifest_path = None |
||||
dump_path = None |
||||
return BackupResult( |
||||
name=name, |
||||
path=destination, |
||||
manifest_path=manifest_destination, |
||||
version=BACKUP_VERSION, |
||||
created_at=now, |
||||
postgresql_client_version=client_version, |
||||
row_counts=dict(snapshot.row_counts), |
||||
) |
||||
except BackupError: |
||||
raise |
||||
except Exception: |
||||
raise BackupError("backup_failed") from None |
||||
finally: |
||||
cleanup_failed = False |
||||
for path in (manifest_path, dump_path): |
||||
if not _unlink_private(path): |
||||
cleanup_failed = True |
||||
if cleanup_failed: |
||||
raise BackupError("cleanup_failed") from None |
||||
|
||||
|
||||
def verify_backup(name, target, *, runner=None): |
||||
runner = runner or subprocess.run |
||||
_validate_name(name) |
||||
live = DatabaseTarget( |
||||
host=str(getattr(settings, "LIVE_DATABASE_HOST", None) or settings.DATABASES["default"].get("HOST", "")), |
||||
port=str(settings.DATABASES["default"].get("PORT") or "5432"), |
||||
name=str(getattr(settings, "LIVE_DATABASE_NAME", None) or settings.DATABASES["default"].get("NAME", "")), |
||||
user="", |
||||
password="", |
||||
) |
||||
if _same_database(live, target): |
||||
raise BackupError("live_target") |
||||
if not getattr(settings, "BACKUP_VERIFY_ALLOWED", False): |
||||
raise BackupError("verification_not_allowed") |
||||
|
||||
try: |
||||
root = _backup_root() |
||||
dump_path = root / name |
||||
manifest_path = root / f"{name}.json" |
||||
manifest = _load_manifest(manifest_path) |
||||
if not dump_path.is_file() or dump_path.is_symlink(): |
||||
raise BackupError("missing_backup") |
||||
digest, byte_size = _file_digest(dump_path) |
||||
if manifest["byte_size"] != byte_size: |
||||
raise BackupError("size_mismatch") |
||||
if manifest["sha256"] != digest: |
||||
raise BackupError("digest_mismatch") |
||||
before = _read_snapshot(target) |
||||
if before.migrations or before.row_counts: |
||||
raise BackupError("target_not_empty") |
||||
runner( |
||||
_pg_restore_argv(target, dump_path), |
||||
env=_database_env(target), |
||||
check=True, |
||||
stdout=None, |
||||
capture_output=True, |
||||
) |
||||
after = _read_snapshot(target) |
||||
except BackupError: |
||||
raise |
||||
except Exception: |
||||
raise BackupError("verification_failed") from None |
||||
|
||||
expected_migrations = tuple(tuple(item) for item in manifest["migrations"]) |
||||
if after.migrations != expected_migrations: |
||||
raise BackupError("migration_mismatch") |
||||
if after.row_counts != manifest["row_counts"]: |
||||
raise BackupError("row_count_mismatch") |
||||
return VerificationResult(name=name, version=BACKUP_VERSION, row_counts=after.row_counts) |
||||
|
||||
|
||||
def _pg_dump_argv(target, dump_path, snapshot_id): |
||||
return [ |
||||
"pg_dump", |
||||
"--format=custom", |
||||
"--no-owner", |
||||
"--no-acl", |
||||
f"--snapshot={snapshot_id}", |
||||
f"--file={dump_path}", |
||||
"--host", |
||||
target.host, |
||||
"--port", |
||||
target.port, |
||||
"--username", |
||||
target.user, |
||||
"--dbname", |
||||
target.name, |
||||
] |
||||
|
||||
|
||||
def _pg_restore_argv(target, dump_path): |
||||
return [ |
||||
"pg_restore", |
||||
"--exit-on-error", |
||||
"--single-transaction", |
||||
"--no-owner", |
||||
"--no-acl", |
||||
"--host", |
||||
target.host, |
||||
"--port", |
||||
target.port, |
||||
"--username", |
||||
target.user, |
||||
"--dbname", |
||||
target.name, |
||||
str(dump_path), |
||||
] |
||||
|
||||
|
||||
def _database_env(target): |
||||
env = os.environ.copy() |
||||
env["PGPASSWORD"] = target.password |
||||
return env |
||||
|
||||
|
||||
def _backup_root(): |
||||
try: |
||||
root = Path(settings.BACKUP_ROOT).resolve(strict=True) |
||||
except (AttributeError, OSError, RuntimeError, TypeError): |
||||
raise BackupError("invalid_root") from None |
||||
if not root.is_dir(): |
||||
raise BackupError("invalid_root") |
||||
return root |
||||
|
||||
|
||||
def _validate_name(name): |
||||
if not isinstance(name, str) or not _PLAIN_DUMP_NAME.fullmatch(name): |
||||
raise BackupError("invalid_name") |
||||
|
||||
|
||||
def _ensure_available(*paths): |
||||
try: |
||||
if any(path.exists() or path.is_symlink() for path in paths): |
||||
raise BackupError("existing_destination") |
||||
except BackupError: |
||||
raise |
||||
except Exception: |
||||
raise BackupError("publication_failed") from None |
||||
|
||||
|
||||
def _publish_no_clobber(source, destination): |
||||
try: |
||||
os.link(source, destination) |
||||
except FileExistsError: |
||||
raise BackupError("existing_destination") from None |
||||
except OSError: |
||||
raise BackupError("publication_failed") from None |
||||
|
||||
|
||||
def _publish_pair(root, pairs): |
||||
publications = [] |
||||
try: |
||||
for source, destination in pairs: |
||||
_publish_no_clobber(source, destination) |
||||
publications.append((destination, source)) |
||||
_fsync_directory(root) |
||||
except Exception: |
||||
_remove_owned_publications(publications) |
||||
for source, _destination in pairs: |
||||
_unlink_private(source) |
||||
raise BackupError("publication_failed") from None |
||||
|
||||
cleanup_failed = False |
||||
for source, _destination in pairs: |
||||
if not _unlink_private(source): |
||||
cleanup_failed = True |
||||
if cleanup_failed: |
||||
raise BackupError("cleanup_failed") |
||||
|
||||
|
||||
def _remove_owned_publications(publications): |
||||
for destination, source in reversed(publications): |
||||
try: |
||||
if source.exists() and destination.exists() and os.path.samefile( |
||||
destination, source |
||||
): |
||||
destination.unlink() |
||||
except OSError: |
||||
pass |
||||
|
||||
|
||||
def _unlink_private(path): |
||||
if path is None: |
||||
return True |
||||
try: |
||||
path.unlink(missing_ok=True) |
||||
return True |
||||
except OSError: |
||||
return False |
||||
|
||||
|
||||
def _fsync_directory(root): |
||||
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) |
||||
descriptor = os.open(root, flags) |
||||
try: |
||||
os.fsync(descriptor) |
||||
finally: |
||||
os.close(descriptor) |
||||
|
||||
|
||||
def _file_digest(path): |
||||
digest = sha256() |
||||
size = 0 |
||||
with path.open("rb") as stream: |
||||
while chunk := stream.read(1024 * 1024): |
||||
digest.update(chunk) |
||||
size += len(chunk) |
||||
return digest.hexdigest(), size |
||||
|
||||
|
||||
def _safe_client_version(output): |
||||
if isinstance(output, bytes): |
||||
output = output.decode("ascii", errors="replace") |
||||
version = str(output).strip() |
||||
if not _is_pg17_version(version): |
||||
raise BackupError("client_version_failed") |
||||
return version |
||||
|
||||
|
||||
def _is_pg17_version(version): |
||||
return ( |
||||
isinstance(version, str) |
||||
and version.startswith("pg_dump (PostgreSQL) 17.") |
||||
and len(version) <= 80 |
||||
) |
||||
|
||||
|
||||
def _load_manifest(path): |
||||
try: |
||||
if path.is_symlink(): |
||||
raise BackupError("invalid_manifest") |
||||
manifest = json.loads(path.read_text(encoding="utf-8")) |
||||
except BackupError: |
||||
raise |
||||
except (OSError, UnicodeError, json.JSONDecodeError): |
||||
raise BackupError("invalid_manifest") from None |
||||
if not isinstance(manifest, dict) or set(manifest) != _MANIFEST_KEYS: |
||||
raise BackupError("invalid_manifest") |
||||
if ( |
||||
manifest["format"] != BACKUP_FORMAT |
||||
or type(manifest["version"]) is not int |
||||
or manifest["version"] != BACKUP_VERSION |
||||
): |
||||
raise BackupError("invalid_manifest") |
||||
if ( |
||||
not isinstance(manifest["application_version"], str) |
||||
or not manifest["application_version"] |
||||
or not isinstance(manifest["created_at"], str) |
||||
or not isinstance(manifest["database_name"], str) |
||||
or not manifest["database_name"] |
||||
or not _is_pg17_version(manifest["postgresql_client_version"]) |
||||
or type(manifest["byte_size"]) is not int |
||||
or manifest["byte_size"] < 0 |
||||
or not isinstance(manifest["sha256"], str) |
||||
or not re.fullmatch(r"[0-9a-f]{64}", manifest["sha256"]) |
||||
or not isinstance(manifest["migrations"], list) |
||||
or not isinstance(manifest["row_counts"], dict) |
||||
): |
||||
raise BackupError("invalid_manifest") |
||||
try: |
||||
created_at = datetime.fromisoformat(manifest["created_at"]) |
||||
except ValueError: |
||||
raise BackupError("invalid_manifest") from None |
||||
if created_at.tzinfo is None: |
||||
raise BackupError("invalid_manifest") |
||||
migrations = manifest["migrations"] |
||||
if any( |
||||
not isinstance(item, list) |
||||
or len(item) != 2 |
||||
or not all(isinstance(value, str) and value for value in item) |
||||
for item in migrations |
||||
): |
||||
raise BackupError("invalid_manifest") |
||||
if any( |
||||
not isinstance(table, str) |
||||
or not _TABLE_NAME.fullmatch(table) |
||||
or type(count) is not int |
||||
or count < 0 |
||||
for table, count in manifest["row_counts"].items() |
||||
): |
||||
raise BackupError("invalid_manifest") |
||||
return manifest |
||||
|
||||
|
||||
@contextmanager |
||||
def _exported_snapshot(target): |
||||
with psycopg.connect( |
||||
host=target.host, |
||||
port=target.port, |
||||
dbname=target.name, |
||||
user=target.user, |
||||
password=target.password, |
||||
connect_timeout=3, |
||||
) as database: |
||||
with database.cursor() as cursor: |
||||
cursor.execute( |
||||
"SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY" |
||||
) |
||||
cursor.execute("SELECT pg_export_snapshot()") |
||||
snapshot_id = cursor.fetchone()[0] |
||||
snapshot = _read_snapshot_cursor(cursor) |
||||
yield snapshot_id, snapshot |
||||
|
||||
|
||||
def _read_snapshot(target): |
||||
with psycopg.connect( |
||||
host=target.host, |
||||
port=target.port, |
||||
dbname=target.name, |
||||
user=target.user, |
||||
password=target.password, |
||||
connect_timeout=3, |
||||
) as database: |
||||
with database.cursor() as cursor: |
||||
return _read_snapshot_cursor(cursor) |
||||
|
||||
|
||||
def _read_snapshot_cursor(cursor): |
||||
cursor.execute( |
||||
""" |
||||
SELECT table_schema, table_name |
||||
FROM information_schema.tables |
||||
WHERE table_type = 'BASE TABLE' |
||||
AND table_schema NOT IN ('pg_catalog', 'information_schema', 'pg_toast') |
||||
AND table_schema NOT LIKE 'pg\\_%' ESCAPE '\\' |
||||
ORDER BY table_schema, table_name |
||||
""" |
||||
) |
||||
tables = [(row[0], row[1]) for row in cursor.fetchall()] |
||||
migrations = () |
||||
if ("public", "django_migrations") in tables: |
||||
cursor.execute("SELECT app, name FROM django_migrations ORDER BY app, name") |
||||
migrations = tuple((row[0], row[1]) for row in cursor.fetchall()) |
||||
row_counts = {} |
||||
for schema, table in tables: |
||||
cursor.execute( |
||||
sql.SQL("SELECT count(*) FROM {}.{}").format( |
||||
sql.Identifier(schema), sql.Identifier(table) |
||||
) |
||||
) |
||||
key = table if schema == "public" else f"{schema}.{table}" |
||||
row_counts[key] = cursor.fetchone()[0] |
||||
return DatabaseSnapshot(migrations=migrations, row_counts=row_counts) |
||||
|
||||
|
||||
def _same_database(first, second): |
||||
return ( |
||||
first.host.strip().casefold(), |
||||
first.name.strip().casefold(), |
||||
) == ( |
||||
second.host.strip().casefold(), |
||||
second.name.strip().casefold(), |
||||
) |
||||
|
||||
|
||||
def run_synthetic_restore_verification(): |
||||
"""Exercise backup and restore only against disposable synthetic databases.""" |
||||
from datetime import date |
||||
from decimal import Decimal |
||||
from uuid import uuid4 |
||||
|
||||
from django.core.management import call_command |
||||
from django.db import connections |
||||
|
||||
from finance.models import Account, AuditEvent, Category, Payee, SourceRecord, Transaction |
||||
from ingestion.models import ImportBatch, ImportItem |
||||
|
||||
if not getattr(settings, "BACKUP_VERIFY_ALLOWED", False): |
||||
raise BackupError("verification_not_allowed") |
||||
required = ( |
||||
"BACKUP_SOURCE_HOST", |
||||
"BACKUP_SOURCE_NAME", |
||||
"BACKUP_SOURCE_USER", |
||||
"BACKUP_SOURCE_PASSWORD", |
||||
) |
||||
if any(not os.environ.get(name) for name in required): |
||||
raise BackupError("source_not_configured") |
||||
|
||||
restore_target = DatabaseTarget.from_default(settings.DATABASES["default"]) |
||||
source_target = DatabaseTarget( |
||||
host=os.environ["BACKUP_SOURCE_HOST"], |
||||
port=os.environ.get("BACKUP_SOURCE_PORT", "5432"), |
||||
name=os.environ["BACKUP_SOURCE_NAME"], |
||||
user=os.environ["BACKUP_SOURCE_USER"], |
||||
password=os.environ["BACKUP_SOURCE_PASSWORD"], |
||||
) |
||||
live_target = DatabaseTarget( |
||||
host=str(getattr(settings, "LIVE_DATABASE_HOST", None) or ""), |
||||
port="5432", |
||||
name=str(getattr(settings, "LIVE_DATABASE_NAME", None) or ""), |
||||
user="", |
||||
password="", |
||||
) |
||||
if _same_database(source_target, restore_target): |
||||
raise BackupError("source_is_target") |
||||
if live_target.host and ( |
||||
_same_database(source_target, live_target) |
||||
or _same_database(restore_target, live_target) |
||||
): |
||||
raise BackupError("live_target") |
||||
try: |
||||
if _read_snapshot(source_target).row_counts: |
||||
raise BackupError("source_not_empty") |
||||
restore_before = _read_snapshot(restore_target) |
||||
if restore_before.migrations or restore_before.row_counts: |
||||
raise BackupError("target_not_empty") |
||||
except BackupError: |
||||
raise |
||||
except Exception: |
||||
raise BackupError("integration_verification_failed") from None |
||||
|
||||
default_connection = connections["default"] |
||||
original_database = dict(default_connection.settings_dict) |
||||
artifact = None |
||||
try: |
||||
default_connection.close() |
||||
_configure_connection(default_connection, source_target) |
||||
call_command("migrate", interactive=False, verbosity=0) |
||||
|
||||
account = Account.objects.create( |
||||
name="Synthetic Restore Bank", kind=Account.Kind.BANK, currency="BRL" |
||||
) |
||||
payee = Payee.objects.create(name="Synthetic Restore Merchant") |
||||
category = Category.objects.create( |
||||
name="Synthetic Restore Expense", kind=Category.Kind.EXPENSE |
||||
) |
||||
transaction = Transaction.objects.create( |
||||
account=account, |
||||
amount=Decimal("-12.34"), |
||||
posting_date=date(2026, 9, 8), |
||||
description="Synthetic restore purchase", |
||||
kind=Transaction.Kind.EXPENSE, |
||||
payee=payee, |
||||
category=category, |
||||
) |
||||
SourceRecord.objects.create( |
||||
transaction=transaction, |
||||
document_hash="a" * 64, |
||||
source_format=SourceRecord.Format.BANK_CSV, |
||||
parser_version="synthetic-v1", |
||||
item_locator="row:1", |
||||
) |
||||
AuditEvent.objects.create( |
||||
actor_kind=AuditEvent.ActorKind.AGENT, |
||||
actor_identifier="synthetic-backup-verification", |
||||
operation="synthetic_backup_verification", |
||||
target_id=transaction.id, |
||||
outcome=AuditEvent.Outcome.SUCCESS, |
||||
) |
||||
batch = ImportBatch.objects.create( |
||||
document_hash="b" * 64, |
||||
source_format="BANK_CSV", |
||||
profile="SYNTHETIC", |
||||
encoding="utf-8", |
||||
actor_kind="AGENT", |
||||
actor_identifier="synthetic-backup-verification", |
||||
) |
||||
ImportItem.objects.create( |
||||
batch=batch, |
||||
locator="row:1", |
||||
candidate={"description": "Synthetic restore candidate"}, |
||||
) |
||||
before = _read_snapshot(source_target) |
||||
artifact = create_backup( |
||||
f"synthetic-restore-check-{uuid4().hex}.dump", _target=source_target |
||||
) |
||||
|
||||
default_connection.close() |
||||
default_connection.settings_dict.clear() |
||||
default_connection.settings_dict.update(original_database) |
||||
result = verify_backup(artifact.name, restore_target) |
||||
call_command("check", databases=["default"], verbosity=0) |
||||
if _read_snapshot(source_target) != before: |
||||
raise BackupError("source_changed") |
||||
return result |
||||
except BackupError: |
||||
raise |
||||
except Exception: |
||||
raise BackupError("integration_verification_failed") from None |
||||
finally: |
||||
cleanup_failed = False |
||||
try: |
||||
default_connection.close() |
||||
default_connection.settings_dict.clear() |
||||
default_connection.settings_dict.update(original_database) |
||||
except Exception: |
||||
cleanup_failed = True |
||||
try: |
||||
_cleanup_synthetic_resources(artifact, source_target, restore_target) |
||||
except BackupError: |
||||
cleanup_failed = True |
||||
if cleanup_failed: |
||||
raise BackupError("cleanup_failed") from None |
||||
|
||||
|
||||
def _configure_connection(connection, target): |
||||
connection.settings_dict.update( |
||||
{ |
||||
"HOST": target.host, |
||||
"PORT": target.port, |
||||
"NAME": target.name, |
||||
"USER": target.user, |
||||
"PASSWORD": target.password, |
||||
} |
||||
) |
||||
|
||||
|
||||
def _clear_disposable_database(target): |
||||
try: |
||||
with psycopg.connect( |
||||
host=target.host, |
||||
port=target.port, |
||||
dbname=target.name, |
||||
user=target.user, |
||||
password=target.password, |
||||
connect_timeout=3, |
||||
autocommit=True, |
||||
) as database: |
||||
with database.cursor() as cursor: |
||||
cursor.execute("SET lock_timeout = '5s'") |
||||
cursor.execute("SET statement_timeout = '15s'") |
||||
cursor.execute("DROP SCHEMA public CASCADE") |
||||
cursor.execute("CREATE SCHEMA public") |
||||
except Exception: |
||||
raise BackupError("cleanup_failed") from None |
||||
|
||||
|
||||
def _cleanup_synthetic_resources(artifact, source_target, restore_target): |
||||
failed = False |
||||
if artifact is not None: |
||||
for path in (artifact.path, artifact.manifest_path): |
||||
try: |
||||
path.unlink(missing_ok=True) |
||||
except Exception: |
||||
failed = True |
||||
for target in (source_target, restore_target): |
||||
try: |
||||
_clear_disposable_database(target) |
||||
except BackupError: |
||||
failed = True |
||||
if failed: |
||||
raise BackupError("cleanup_failed") |
||||
@ -0,0 +1,10 @@
|
||||
def navigation(_request): |
||||
return {"navigation": ( |
||||
("home", "Home"), |
||||
("account-list", "Accounts"), |
||||
("transaction-list", "Transactions"), |
||||
("category-list", "Categories"), |
||||
("payee-list", "Payees"), |
||||
("monthly-report", "Reports"), |
||||
("export-index", "Exports"), |
||||
)} |
||||
@ -0,0 +1,118 @@
|
||||
from datetime import date |
||||
import re |
||||
from uuid import UUID |
||||
|
||||
from django import forms |
||||
|
||||
from finance.models import Account, Category, Payee, Transaction |
||||
from finance.queries import active_accounts, active_categories, active_payees |
||||
|
||||
|
||||
_MONTH_RE = re.compile(r"(?P<year>\d{4})-(?P<month>0[1-9]|1[0-2])", re.ASCII) |
||||
|
||||
|
||||
def _reference_choices(model, *, include=None): |
||||
rows = {Account: active_accounts, Category: active_categories, Payee: active_payees}[model]() |
||||
if include is not None: |
||||
rows = model.objects.filter(pk=include.pk) | rows |
||||
rows = rows.order_by("name", "id") |
||||
return [(str(row.pk), row.name) for row in rows] |
||||
|
||||
|
||||
def _uuid(value): |
||||
return None if value == "" else UUID(value) |
||||
|
||||
|
||||
class ManualTransactionForm(forms.Form): |
||||
account = forms.TypedChoiceField(coerce=UUID) |
||||
amount = forms.DecimalField(max_digits=18, decimal_places=2) |
||||
posting_date = forms.DateField(widget=forms.DateInput(attrs={"type": "date"})) |
||||
occurrence_date = forms.DateField(required=False, widget=forms.DateInput(attrs={"type": "date"})) |
||||
kind = forms.ChoiceField(choices=Transaction.Kind.choices) |
||||
description = forms.CharField(required=False, widget=forms.Textarea(attrs={"rows": 3})) |
||||
payee = forms.TypedChoiceField(required=False, empty_value=None, coerce=_uuid) |
||||
category = forms.TypedChoiceField(required=False, empty_value=None, coerce=_uuid) |
||||
|
||||
def __init__(self, *args, **kwargs): |
||||
super().__init__(*args, **kwargs) |
||||
self.fields["account"].choices = _reference_choices(Account) |
||||
self.fields["payee"].choices = [("", "None"), *_reference_choices(Payee)] |
||||
self.fields["category"].choices = [("", "Uncategorized"), *_reference_choices(Category)] |
||||
|
||||
def clean(self): |
||||
values = super().clean() |
||||
amount = values.get("amount") |
||||
kind = values.get("kind") |
||||
if amount is not None and kind == Transaction.Kind.EXPENSE and amount >= 0: |
||||
self.add_error("amount", "Expense amounts must be negative.") |
||||
if amount is not None and kind in {Transaction.Kind.INCOME, Transaction.Kind.REFUND} and amount <= 0: |
||||
self.add_error("amount", "Income and refund amounts must be positive.") |
||||
return values |
||||
|
||||
|
||||
class AnnotationForm(forms.Form): |
||||
description = forms.CharField(required=False, widget=forms.Textarea(attrs={"rows": 3})) |
||||
payee = forms.TypedChoiceField(required=False, empty_value=None, coerce=_uuid) |
||||
category = forms.TypedChoiceField(required=False, empty_value=None, coerce=_uuid) |
||||
|
||||
def __init__(self, *args, transaction, **kwargs): |
||||
super().__init__(*args, **kwargs) |
||||
self.transaction = transaction |
||||
self.fields["payee"].choices = [("", "None"), *_reference_choices(Payee, include=transaction.payee)] |
||||
self.fields["category"].choices = [("", "Uncategorized"), *_reference_choices(Category, include=transaction.category)] |
||||
if not self.is_bound: |
||||
self.initial = { |
||||
"description": transaction.description, |
||||
"payee": transaction.payee_id, |
||||
"category": transaction.category_id, |
||||
} |
||||
|
||||
def changes(self): |
||||
values = self.cleaned_data |
||||
changes = {} |
||||
for form_name, model_name in (("description", "description"), ("payee", "payee_id"), ("category", "category_id")): |
||||
if values[form_name] != getattr(self.transaction, model_name): |
||||
changes[model_name] = values[form_name] |
||||
return changes |
||||
|
||||
|
||||
class AccountForm(forms.Form): |
||||
name = forms.CharField(max_length=255) |
||||
kind = forms.ChoiceField(choices=Account.Kind.choices) |
||||
currency = forms.RegexField(regex=r"^[A-Z]{3}$", max_length=3) |
||||
|
||||
|
||||
class CategoryForm(forms.Form): |
||||
name = forms.CharField(max_length=255) |
||||
kind = forms.ChoiceField(choices=Category.Kind.choices) |
||||
|
||||
|
||||
class PayeeForm(forms.Form): |
||||
name = forms.CharField(max_length=255) |
||||
|
||||
|
||||
class ReportSelectionForm(forms.Form): |
||||
month = forms.CharField( |
||||
error_messages={"required": "invalid_month"}, |
||||
widget=forms.TextInput(attrs={"type": "month"}), |
||||
) |
||||
currency = forms.ChoiceField( |
||||
error_messages={"required": "invalid_currency", "invalid_choice": "invalid_currency"} |
||||
) |
||||
|
||||
def __init__(self, *args, currencies, **kwargs): |
||||
super().__init__(*args, **kwargs) |
||||
self.fields["currency"].choices = [(currency, currency) for currency in currencies] |
||||
|
||||
def clean_month(self): |
||||
value = self.cleaned_data["month"] |
||||
match = _MONTH_RE.fullmatch(value) |
||||
if match is None: |
||||
raise forms.ValidationError("invalid_month", code="invalid_month") |
||||
try: |
||||
month = date(int(match["year"]), int(match["month"]), 1) |
||||
except ValueError as exc: |
||||
raise forms.ValidationError("invalid_month", code="invalid_month") from exc |
||||
if month.year == date.max.year and month.month == 12: |
||||
raise forms.ValidationError("invalid_month", code="invalid_month") |
||||
return month |
||||
@ -0,0 +1,19 @@
|
||||
from django.core.management.base import BaseCommand, CommandError |
||||
|
||||
from core.backup import BackupError, create_backup |
||||
|
||||
|
||||
class Command(BaseCommand): |
||||
help = "Create a local PostgreSQL backup and checksum manifest." |
||||
|
||||
def add_arguments(self, parser): |
||||
parser.add_argument("--output", dest="output") |
||||
|
||||
def handle(self, *args, **options): |
||||
try: |
||||
result = create_backup(options["output"]) |
||||
except BackupError as error: |
||||
raise CommandError(error.code) from None |
||||
self.stdout.write(f"backup={result.name} format={result.version}") |
||||
for table, count in sorted(result.row_counts.items()): |
||||
self.stdout.write(f"table={table} rows={count}") |
||||
@ -0,0 +1,17 @@
|
||||
from django.core.management.base import BaseCommand, CommandError |
||||
|
||||
from core.backup import BackupError, run_synthetic_restore_verification |
||||
|
||||
|
||||
class Command(BaseCommand): |
||||
help = "Run the synthetic backup and disposable restore acceptance check." |
||||
|
||||
def handle(self, *args, **options): |
||||
try: |
||||
result = run_synthetic_restore_verification() |
||||
result = run_synthetic_restore_verification() |
||||
except BackupError as error: |
||||
raise CommandError(error.code) from None |
||||
self.stdout.write(f"backup={result.name} format={result.version} verified") |
||||
for table, count in sorted(result.row_counts.items()): |
||||
self.stdout.write(f"table={table} rows={count}") |
||||
@ -0,0 +1,31 @@
|
||||
from django.conf import settings |
||||
from django.core.management.base import BaseCommand, CommandError |
||||
|
||||
from core.backup import BackupError, DatabaseTarget, verify_backup |
||||
|
||||
|
||||
class Command(BaseCommand): |
||||
help = "Restore a backup into an explicitly allowed disposable database and verify it." |
||||
|
||||
def add_arguments(self, parser): |
||||
parser.add_argument("--input", dest="input", required=True) |
||||
parser.add_argument("--target-name", required=True) |
||||
parser.add_argument("--target-host", required=True) |
||||
parser.add_argument("--target-user", required=True) |
||||
|
||||
def handle(self, *args, **options): |
||||
default = settings.DATABASES["default"] |
||||
target = DatabaseTarget( |
||||
host=options["target_host"], |
||||
port=str(default.get("PORT") or "5432"), |
||||
name=options["target_name"], |
||||
user=options["target_user"], |
||||
password=str(default.get("PASSWORD", "")), |
||||
) |
||||
try: |
||||
result = verify_backup(options["input"], target) |
||||
except BackupError as error: |
||||
raise CommandError(error.code) from None |
||||
self.stdout.write(f"backup={result.name} format={result.version} verified") |
||||
for table, count in sorted(result.row_counts.items()): |
||||
self.stdout.write(f"table={table} rows={count}") |
||||
@ -0,0 +1,26 @@
|
||||
:root { color-scheme: light; font-family: system-ui, sans-serif; color: #17221b; background: #f4f7f4; } |
||||
* { box-sizing: border-box; } |
||||
body { margin: 0; } |
||||
header { display: flex; align-items: center; gap: 2rem; padding: 1rem max(1rem, calc((100% - 72rem) / 2)); background: #173d2a; color: white; } |
||||
header a { color: white; } |
||||
.brand { font-size: 1.25rem; font-weight: 700; text-decoration: none; } |
||||
nav { display: flex; flex-wrap: wrap; gap: 1rem; } |
||||
main { max-width: 72rem; margin: auto; padding: 2rem 1rem; } |
||||
.heading { display: flex; align-items: center; justify-content: space-between; gap: 1rem; } |
||||
.filters { display: grid; grid-template-columns: repeat(auto-fit, minmax(10rem, 1fr)); gap: 1rem; align-items: end; padding: 1rem; background: white; border-radius: .5rem; } |
||||
label { display: block; font-weight: 650; margin-bottom: .3rem; } |
||||
input, select, textarea, button, .button { width: 100%; padding: .65rem; border: 1px solid #7b8d82; border-radius: .3rem; font: inherit; } |
||||
button, .button { width: auto; background: #236642; color: white; border-color: #236642; cursor: pointer; text-decoration: none; } |
||||
.secondary { background: white; color: #294b38; } |
||||
:focus-visible { outline: 3px solid #de9f22; outline-offset: 2px; } |
||||
.field { margin-bottom: 1rem; max-width: 36rem; } |
||||
.table-wrap { overflow-x: auto; margin-top: 1rem; } |
||||
table { width: 100%; border-collapse: collapse; background: white; } |
||||
th, td { padding: .75rem; border-bottom: 1px solid #d7dfd9; text-align: left; } |
||||
.number { text-align: right; font-variant-numeric: tabular-nums; } |
||||
.details { display: grid; grid-template-columns: max-content 1fr; gap: .5rem 1.5rem; } |
||||
.details dt { font-weight: 700; } |
||||
.details dd { margin: 0; } |
||||
.error, .errorlist, .error-summary { color: #8d1b16; } |
||||
.error-summary, .message { padding: .75rem; background: white; border-left: .3rem solid currentColor; } |
||||
@media (max-width: 40rem) { header, .heading { align-items: flex-start; flex-direction: column; } .details { grid-template-columns: 1fr; } } |
||||
@ -0,0 +1,22 @@
|
||||
{% load static %} |
||||
<!doctype html> |
||||
<html lang="en"> |
||||
<head> |
||||
<meta charset="utf-8"> |
||||
<meta name="viewport" content="width=device-width, initial-scale=1"> |
||||
<title>{% block title %}Gizmoduck{% endblock %}</title> |
||||
<link rel="stylesheet" href="{% static 'core/app.css' %}"> |
||||
</head> |
||||
<body> |
||||
<header> |
||||
<a class="brand" href="{% url 'home' %}">Gizmoduck</a> |
||||
<nav aria-label="Main navigation"> |
||||
{% for route, label in navigation %}<a href="{% url route %}">{{ label }}</a>{% endfor %} |
||||
</nav> |
||||
</header> |
||||
<main> |
||||
{% if messages %}<div aria-live="polite">{% for message in messages %}<p class="message {{ message.tags }}">{{ message }}</p>{% endfor %}</div>{% endif %} |
||||
{% block content %}{% endblock %} |
||||
</main> |
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,29 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}Dashboard · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<div class="heading"><h1>Dashboard</h1><a class="button secondary" href="{% url 'monthly-report' %}?month={{ selection_form.month.value }}&currency={{ selection_form.currency.value }}">Detailed monthly report</a></div> |
||||
<form method="get" class="filters"> |
||||
<div><label for="id_month">Month</label>{{ selection_form.month }}{% for error in selection_form.month.errors %}<p class="error">{{ error }}</p>{% endfor %}</div> |
||||
<div><label for="id_currency">Currency</label>{{ selection_form.currency }}{% for error in selection_form.currency.errors %}<p class="error">{{ error }}</p>{% endfor %}</div> |
||||
<button type="submit">Show month</button> |
||||
</form> |
||||
{% if selection_form.errors %}<p class="error-summary" role="alert">Review the invalid report selection above.</p>{% endif %} |
||||
{% if summary %} |
||||
<h2>Cash flow</h2> |
||||
<dl class="details"> |
||||
<dt>Inflow</dt><dd class="number">{{ summary.inflow|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
<dt>Expense</dt><dd class="number">{{ summary.expense|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
<dt>Refund</dt><dd class="number">{{ summary.refund|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
<dt>Net spending{% if summary.net_spending < 0 %} (negative){% endif %}</dt><dd class="number">{{ summary.net_spending|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
<dt>Net activity</dt><dd class="number">{{ summary.net_activity|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
</dl> |
||||
<h2>Spending by category</h2> |
||||
{% if spending_rows %}<div class="table-wrap"><table><thead><tr><th>Category</th><th class="number">Expense</th><th class="number">Refund</th><th class="number">Net spending</th></tr></thead><tbody> |
||||
{% for row in spending_rows %}<tr><td>{{ row.category_name }}</td><td class="number">{{ row.expense|floatformat:"2g" }}</td><td class="number">{{ row.refund|floatformat:"2g" }}</td><td class="number">{{ row.net_spending|floatformat:"2g" }} {{ summary.currency }}</td></tr>{% endfor %} |
||||
</tbody></table></div>{% else %}<p>No spending activity for this month.</p>{% endif %} |
||||
<h2>Account balances</h2> |
||||
{% if balances %}<div class="table-wrap"><table><thead><tr><th>Account</th><th class="number">Balance</th></tr></thead><tbody> |
||||
{% for balance in balances %}<tr><td>{{ balance.account_name }}</td><td class="number">{% if balance.balance < 0 %}<span>Balance (negative): </span>{% endif %}{{ balance.balance|floatformat:"2g" }} {{ summary.currency }}</td></tr>{% endfor %} |
||||
</tbody></table></div>{% else %}<p>No accounts use {{ summary.currency }}.</p>{% endif %} |
||||
{% endif %} |
||||
{% endblock %} |
||||
@ -0,0 +1,12 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}Export financial data · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<h1>Export financial data</h1> |
||||
<p>Download a versioned ZIP package containing UTF-8 CSV files and a checksum manifest.</p> |
||||
<p>The export includes accounts, payees, categories, card statements, installment plans, transactions, transfers, and source records. It preserves stable identifiers and relationships so the data remains portable.</p> |
||||
<p>The download is built from one consistent database snapshot. It does not include audit events or unfinished import reviews.</p> |
||||
<form method="post" action="{% url 'export-download' %}"> |
||||
{% csrf_token %} |
||||
<button type="submit">Download export</button> |
||||
</form> |
||||
{% endblock %} |
||||
@ -0,0 +1,7 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}{{ title }} · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<h1>{{ title }}</h1> |
||||
{% if form.errors %}<div class="error-summary" role="alert"><strong>Review the highlighted fields.</strong>{{ form.non_field_errors }}</div>{% endif %} |
||||
<form method="post">{% csrf_token %}{% for field in form %}<div class="field">{{ field.label_tag }}{{ field }}{{ field.errors }}</div>{% endfor %}<button type="submit">Save</button></form> |
||||
{% endblock %} |
||||
@ -0,0 +1,8 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}{{ title }} · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<div class="heading"><h1>{{ title }}</h1><a class="button" href="{% url create_url %}">Create</a></div> |
||||
{% if rows %}<div class="table-wrap"><table><thead><tr><th>Name</th><th>Type</th><th>Status</th><th>Action</th></tr></thead><tbody> |
||||
{% for row in rows %}<tr><td>{{ row.name }}</td><td>{{ row.get_kind_display|default:"—" }}{% if row.currency %} · {{ row.currency }}{% endif %}</td><td>{% if row.archived %}Archived{% else %}Active{% endif %}</td><td>{% if not row.archived %}<form method="post" action="{% url archive_url row.pk %}">{% csrf_token %}<button type="submit" class="secondary">Archive</button></form>{% else %}Historical{% endif %}</td></tr>{% endfor %} |
||||
</tbody></table></div>{% else %}<p>No entries yet.</p>{% endif %} |
||||
{% endblock %} |
||||
@ -0,0 +1,29 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}Monthly report · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<div class="heading"><h1>Monthly report</h1><a class="button secondary" href="{% url 'home' %}?month={{ selection_form.month.value }}&currency={{ selection_form.currency.value }}">Dashboard</a></div> |
||||
<form method="get" class="filters"> |
||||
<div><label for="id_month">Month</label>{{ selection_form.month }}{% for error in selection_form.month.errors %}<p class="error">{{ error }}</p>{% endfor %}</div> |
||||
<div><label for="id_currency">Currency</label>{{ selection_form.currency }}{% for error in selection_form.currency.errors %}<p class="error">{{ error }}</p>{% endfor %}</div> |
||||
<button type="submit">Show report</button> |
||||
</form> |
||||
{% if selection_form.errors %}<p class="error-summary" role="alert">Review the invalid report selection above.</p>{% endif %} |
||||
{% if summary %} |
||||
<h2>Cash flow</h2> |
||||
<dl class="details"> |
||||
<dt>Inflow</dt><dd class="number">{{ summary.inflow|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
<dt>Expense</dt><dd class="number">{{ summary.expense|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
<dt>Refund</dt><dd class="number">{{ summary.refund|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
<dt>Net spending{% if summary.net_spending < 0 %} (negative){% endif %}</dt><dd class="number">{{ summary.net_spending|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
<dt>Net activity</dt><dd class="number">{{ summary.net_activity|floatformat:"2g" }} {{ summary.currency }}</dd> |
||||
</dl> |
||||
<h2>Spending by category</h2> |
||||
{% if spending_rows %}<div class="table-wrap"><table><thead><tr><th>Category</th><th class="number">Expense</th><th class="number">Refund</th><th class="number">Net spending</th></tr></thead><tbody> |
||||
{% for row in spending_rows %}<tr><td>{{ row.category_name }}</td><td class="number">{{ row.expense|floatformat:"2g" }}</td><td class="number">{{ row.refund|floatformat:"2g" }}</td><td class="number">{{ row.net_spending|floatformat:"2g" }} {{ summary.currency }}</td></tr>{% endfor %} |
||||
</tbody></table></div>{% else %}<p>No spending activity for this month.</p>{% endif %} |
||||
<h2>Account balances</h2> |
||||
{% if balances %}<div class="table-wrap"><table><thead><tr><th>Account</th><th class="number">Balance</th></tr></thead><tbody> |
||||
{% for balance in balances %}<tr><td>{{ balance.account_name }}</td><td class="number">{% if balance.balance < 0 %}<span>Balance (negative): </span>{% endif %}{{ balance.balance|floatformat:"2g" }} {{ summary.currency }}</td></tr>{% endfor %} |
||||
</tbody></table></div>{% else %}<p>No accounts use {{ summary.currency }}.</p>{% endif %} |
||||
{% endif %} |
||||
{% endblock %} |
||||
@ -0,0 +1,22 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}Transaction · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<div class="heading"><h1>Transaction</h1><a class="button" href="{% url 'transaction-annotate' transaction.pk %}">Edit annotations</a></div> |
||||
<dl class="details"> |
||||
<dt>Description</dt><dd>{{ transaction.description|default:"(No description)" }}</dd> |
||||
<dt>Posting date</dt><dd>{{ transaction.posting_date|date:"Y-m-d" }}</dd> |
||||
<dt>Occurrence date</dt><dd>{{ transaction.occurrence_date|date:"Y-m-d"|default:"—" }}</dd> |
||||
<dt>Amount</dt><dd>{{ transaction.amount }} {{ transaction.account.currency }}</dd> |
||||
<dt>Account</dt><dd>{{ transaction.account.name }}</dd> |
||||
<dt>Kind</dt><dd>{{ transaction.get_kind_display }}</dd> |
||||
<dt>Payee</dt><dd>{{ transaction.payee.name|default:"—" }}</dd> |
||||
<dt>Category</dt><dd>{{ transaction.category.name|default:"Uncategorized" }}</dd> |
||||
{% if transaction.statement %}<dt>Statement</dt><dd>{{ transaction.statement.period_start|date:"Y-m-d" }} to {{ transaction.statement.period_end|date:"Y-m-d" }}; due {{ transaction.statement.due_date|date:"Y-m-d" }}</dd>{% endif %} |
||||
{% if transaction.installment_plan %}<dt>Installment plan</dt><dd>{{ transaction.installment_plan.description|default:"(No description)" }} · {{ transaction.installment_number }} of {{ transaction.installment_plan.total_installments }}</dd>{% endif %} |
||||
{% if transaction.refund_of %}<dt>Refund of</dt><dd><a href="{% url 'transaction-detail' transaction.refund_of.pk %}">{{ transaction.refund_of.description|default:"(No description)" }}</a></dd>{% endif %} |
||||
{% if transaction.outgoing_transfer %}<dt>Transfer counterpart</dt><dd>Incoming to <a href="{% url 'transaction-detail' transaction.outgoing_transfer.incoming.pk %}">{{ transaction.outgoing_transfer.incoming.account.name }}</a></dd>{% elif transaction.incoming_transfer %}<dt>Transfer counterpart</dt><dd>Outgoing from <a href="{% url 'transaction-detail' transaction.incoming_transfer.outgoing.pk %}">{{ transaction.incoming_transfer.outgoing.account.name }}</a></dd>{% endif %} |
||||
</dl> |
||||
{% if transaction.refunds.all %}<h2>Refunds</h2><ul>{% for refund in transaction.refunds.all %}<li><a href="{% url 'transaction-detail' refund.pk %}">{{ refund.description|default:"(No description)" }}</a> · {{ refund.amount }} {{ refund.account.currency }}</li>{% endfor %}</ul>{% endif %} |
||||
<h2>Source provenance</h2> |
||||
{% if transaction.source_records.all %}<div class="table-wrap"><table><thead><tr><th>Format</th><th>Parser</th><th>Location</th></tr></thead><tbody>{% for source in transaction.source_records.all %}<tr><td>{{ source.source_format }}</td><td>{{ source.parser_version }}</td><td>{{ source.item_locator }}</td></tr>{% endfor %}</tbody></table></div>{% else %}<p>This transaction was entered manually.</p>{% endif %} |
||||
{% endblock %} |
||||
@ -0,0 +1,10 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}{{ title }} · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<h1>{{ title }}</h1> |
||||
{% if form.errors %}<div class="error-summary" role="alert"><strong>Review the highlighted fields.</strong>{{ form.non_field_errors }}</div>{% endif %} |
||||
<form method="post">{% csrf_token %} |
||||
{% for field in form %}<div class="field">{{ field.label_tag }}{{ field }}{% if field.help_text %}<small>{{ field.help_text }}</small>{% endif %}{{ field.errors }}</div>{% endfor %} |
||||
<button type="submit">Save</button> |
||||
</form> |
||||
{% endblock %} |
||||
@ -0,0 +1,23 @@
|
||||
{% extends "core/base.html" %} |
||||
{% block title %}Transactions · Gizmoduck{% endblock %} |
||||
{% block content %} |
||||
<div class="heading"><h1>Transactions</h1><a class="button" href="{% url 'transaction-create' %}">New transaction</a></div> |
||||
<form method="get" class="filters"> |
||||
<div><label for="date_from">From</label><input id="date_from" name="date_from" type="date" value="{{ request.GET.date_from }}">{% if filter_errors.date_from %}<p class="error">{{ filter_errors.date_from }}</p>{% endif %}</div> |
||||
<div><label for="date_to">To</label><input id="date_to" name="date_to" type="date" value="{{ request.GET.date_to }}">{% if filter_errors.date_to %}<p class="error">{{ filter_errors.date_to }}</p>{% endif %}</div> |
||||
<div><label for="account">Account</label><select id="account" name="account"><option value="">All</option>{% for item in accounts %}<option value="{{ item.pk }}"{% if request.GET.account == item.pk|stringformat:'s' %} selected{% endif %}>{{ item.name }}{% if item.archived %} (Archived){% endif %}</option>{% endfor %}</select>{% if filter_errors.account %}<p class="error">{{ filter_errors.account }}</p>{% endif %}</div> |
||||
<div><label for="kind">Kind</label><select id="kind" name="kind"><option value="">All</option>{% for value, label in kinds %}<option value="{{ value }}"{% if request.GET.kind == value %} selected{% endif %}>{{ label }}</option>{% endfor %}</select></div> |
||||
<div><label for="payee">Payee</label><select id="payee" name="payee"><option value="">All</option>{% for item in payees %}<option value="{{ item.pk }}"{% if request.GET.payee == item.pk|stringformat:'s' %} selected{% endif %}>{{ item.name }}</option>{% endfor %}</select></div> |
||||
<div><label for="category">Category</label><select id="category" name="category"><option value="">All</option>{% for item in categories %}<option value="{{ item.pk }}"{% if request.GET.category == item.pk|stringformat:'s' %} selected{% endif %}>{{ item.name }}</option>{% endfor %}</select></div> |
||||
<div><label for="currency">Currency</label><input id="currency" name="currency" maxlength="3" value="{{ request.GET.currency }}">{% if filter_errors.currency %}<p class="error">{{ filter_errors.currency }}</p>{% endif %}</div> |
||||
<div><label for="uncategorized">Category status</label><select id="uncategorized" name="uncategorized"><option value="">All</option><option value="yes"{% if request.GET.uncategorized == 'yes' %} selected{% endif %}>Uncategorized</option><option value="no"{% if request.GET.uncategorized == 'no' %} selected{% endif %}>Categorized</option></select></div> |
||||
<button type="submit">Filter</button> |
||||
</form> |
||||
{% if filter_errors %}<p class="error" role="alert">Review the invalid filters above.</p>{% endif %} |
||||
{% if page.object_list %} |
||||
<div class="table-wrap"><table><thead><tr><th>Date</th><th>Description</th><th>Account</th><th>Kind</th><th>Category</th><th class="number">Amount</th></tr></thead><tbody> |
||||
{% for row in page %}<tr><td>{{ row.posting_date|date:"Y-m-d" }}</td><td><a href="{% url 'transaction-detail' row.pk %}">{{ row.description|default:"(No description)" }}</a></td><td>{{ row.account.name }}</td><td>{{ row.get_kind_display }}</td><td>{{ row.category.name|default:"Uncategorized" }}</td><td class="number">{{ row.amount }} {{ row.account.currency }}</td></tr>{% endfor %} |
||||
</tbody></table></div> |
||||
{% else %}<p>No transactions match these filters.</p>{% endif %} |
||||
{% if page.has_other_pages %}<nav aria-label="Pagination">{% if page.has_previous %}<a href="?{% if pagination_query %}{{ pagination_query }}&{% endif %}page={{ page.previous_page_number }}">Previous</a>{% endif %}<span>Page {{ page.number }} of {{ page.paginator.num_pages }}</span>{% if page.has_next %}<a href="?{% if pagination_query %}{{ pagination_query }}&{% endif %}page={{ page.next_page_number }}">Next</a>{% endif %}</nav>{% endif %} |
||||
{% endblock %} |
||||
@ -0,0 +1,250 @@
|
||||
from django.contrib import messages |
||||
from django.core.paginator import Paginator |
||||
from django.db import DatabaseError, connection |
||||
from django.http import HttpResponse, JsonResponse |
||||
from django.shortcuts import get_object_or_404, redirect, render |
||||
from django.utils import timezone |
||||
from django.views.decorators.http import require_POST |
||||
|
||||
from finance.models import Account, Category, Payee, Transaction |
||||
from finance.exports import build_export |
||||
from finance.queries import ( |
||||
QueryInputError, TransactionFilters, filtered_transactions, |
||||
listed_accounts, listed_categories, listed_payees, |
||||
) |
||||
from finance.reports import account_balances, available_currencies, monthly_summary, spending_by_category |
||||
from finance.services import ( |
||||
Actor, DomainError, annotate_transaction, archive, create_account, |
||||
create_category, create_payee, record_transaction, |
||||
) |
||||
|
||||
from .forms import ( |
||||
AccountForm, AnnotationForm, CategoryForm, ManualTransactionForm, PayeeForm, |
||||
ReportSelectionForm, |
||||
) |
||||
|
||||
|
||||
LOCAL_ACTOR = Actor(kind="HUMAN", identifier="local-owner") |
||||
DOMAIN_MESSAGES = { |
||||
"archived_reference": "That reference is archived and cannot be selected.", |
||||
"category_kind_mismatch": "The category does not match the transaction kind.", |
||||
"conflict": "A record with these values already exists.", |
||||
"invalid_amount": "Enter a valid amount with at most two decimal places.", |
||||
"invalid_name": "Enter a name.", |
||||
"not_found": "The requested record was not found.", |
||||
"validation_error": "The values do not satisfy the financial rules.", |
||||
"zero_amount": "The amount cannot be zero.", |
||||
} |
||||
UNKNOWN_DOMAIN_MESSAGE = "Unable to save this change." |
||||
|
||||
|
||||
def health(request): |
||||
try: |
||||
with connection.cursor() as cursor: |
||||
cursor.execute("SELECT 1") |
||||
cursor.fetchone() |
||||
except DatabaseError: |
||||
return JsonResponse({"status": "unavailable"}, status=503) |
||||
return JsonResponse({"status": "ok"}) |
||||
|
||||
|
||||
def home(request): |
||||
return _monthly_report(request, "core/dashboard.html") |
||||
|
||||
|
||||
def monthly_report(request): |
||||
return _monthly_report(request, "core/reports/monthly.html") |
||||
|
||||
|
||||
def export_index(request): |
||||
return render(request, "core/exports.html") |
||||
|
||||
|
||||
@require_POST |
||||
def export_download(request): |
||||
artifact = build_export() |
||||
response = HttpResponse(artifact.content, content_type="application/zip") |
||||
response["Content-Disposition"] = f'attachment; filename="{artifact.filename}"' |
||||
return response |
||||
|
||||
|
||||
def transaction_list(request): |
||||
filter_errors = {} |
||||
try: |
||||
filters = TransactionFilters.from_mapping(request.GET) |
||||
rows = filtered_transactions(filters) |
||||
except QueryInputError as exc: |
||||
filters = TransactionFilters() |
||||
rows = Transaction.objects.none() |
||||
filter_errors = {field: _query_message(code) for field, code in exc.errors.items()} |
||||
page = Paginator(rows, 50).get_page(request.GET.get("page")) |
||||
pagination_parameters = request.GET.copy() |
||||
pagination_parameters.pop("page", None) |
||||
return render(request, "core/transactions/list.html", { |
||||
"page": page, |
||||
"filters": filters, |
||||
"filter_errors": filter_errors, |
||||
"accounts": listed_accounts(), |
||||
"categories": listed_categories(), |
||||
"payees": listed_payees(), |
||||
"kinds": Transaction.Kind.choices, |
||||
"pagination_query": pagination_parameters.urlencode(), |
||||
}) |
||||
|
||||
|
||||
def transaction_detail(request, transaction_id): |
||||
row = get_object_or_404( |
||||
Transaction.objects.select_related( |
||||
"account", "payee", "category", "statement", "refund_of", |
||||
"installment_plan", "outgoing_transfer__incoming__account", |
||||
"incoming_transfer__outgoing__account", |
||||
).prefetch_related("source_records", "refunds"), |
||||
pk=transaction_id, |
||||
) |
||||
return render(request, "core/transactions/detail.html", {"transaction": row}) |
||||
|
||||
|
||||
def transaction_create(request): |
||||
form = ManualTransactionForm(request.POST or None) |
||||
if request.method == "POST" and form.is_valid(): |
||||
values = form.cleaned_data |
||||
try: |
||||
row = record_transaction( |
||||
account_id=values["account"], amount=values["amount"], |
||||
posting_date=values["posting_date"], occurrence_date=values["occurrence_date"], |
||||
kind=values["kind"], description=values["description"], |
||||
payee_id=values["payee"], category_id=values["category"], actor=LOCAL_ACTOR, |
||||
) |
||||
except DomainError as exc: |
||||
form.add_error(None, _domain_message(exc)) |
||||
else: |
||||
messages.success(request, "Transaction recorded.") |
||||
return redirect("transaction-detail", transaction_id=row.pk) |
||||
return render(request, "core/transactions/form.html", {"form": form, "title": "New transaction"}) |
||||
|
||||
|
||||
def transaction_annotate(request, transaction_id): |
||||
row = get_object_or_404(Transaction.objects.select_related("payee", "category"), pk=transaction_id) |
||||
form = AnnotationForm(request.POST or None, transaction=row) |
||||
if request.method == "POST" and form.is_valid(): |
||||
changes = form.changes() |
||||
if not changes: |
||||
return redirect("transaction-detail", transaction_id=row.pk) |
||||
try: |
||||
annotate_transaction(transaction_id=row.pk, actor=LOCAL_ACTOR, **changes) |
||||
except DomainError as exc: |
||||
form.add_error(None, _domain_message(exc)) |
||||
else: |
||||
messages.success(request, "Transaction annotations updated.") |
||||
return redirect("transaction-detail", transaction_id=row.pk) |
||||
return render(request, "core/transactions/form.html", { |
||||
"form": form, "title": "Edit transaction annotations", "transaction": row, |
||||
}) |
||||
|
||||
|
||||
def account_list(request): |
||||
return _reference_list(request, listed_accounts(), "Accounts", "account-create", "account-archive") |
||||
|
||||
|
||||
def category_list(request): |
||||
return _reference_list(request, listed_categories(), "Categories", "category-create", "category-archive") |
||||
|
||||
|
||||
def payee_list(request): |
||||
return _reference_list(request, listed_payees(), "Payees", "payee-create", "payee-archive") |
||||
|
||||
|
||||
def account_create(request): |
||||
return _reference_create(request, AccountForm, create_account, "New account", "account-list") |
||||
|
||||
|
||||
def category_create(request): |
||||
return _reference_create(request, CategoryForm, create_category, "New category", "category-list") |
||||
|
||||
|
||||
def payee_create(request): |
||||
return _reference_create(request, PayeeForm, create_payee, "New payee", "payee-list") |
||||
|
||||
|
||||
@require_POST |
||||
def account_archive(request, reference_id): |
||||
return _reference_archive(request, Account, reference_id, "account-list") |
||||
|
||||
|
||||
@require_POST |
||||
def category_archive(request, reference_id): |
||||
return _reference_archive(request, Category, reference_id, "category-list") |
||||
|
||||
|
||||
@require_POST |
||||
def payee_archive(request, reference_id): |
||||
return _reference_archive(request, Payee, reference_id, "payee-list") |
||||
|
||||
|
||||
def _reference_list(request, rows, title, create_url, archive_url): |
||||
return render(request, "core/references/list.html", { |
||||
"title": title, |
||||
"rows": rows, |
||||
"create_url": create_url, |
||||
"archive_url": archive_url, |
||||
}) |
||||
|
||||
|
||||
def _reference_create(request, form_class, service, title, redirect_name): |
||||
form = form_class(request.POST or None) |
||||
if request.method == "POST" and form.is_valid(): |
||||
try: |
||||
service(actor=LOCAL_ACTOR, **form.cleaned_data) |
||||
except DomainError as exc: |
||||
form.add_error(None, _domain_message(exc)) |
||||
else: |
||||
messages.success(request, f"{title.removeprefix('New ')} created.") |
||||
return redirect(redirect_name) |
||||
return render(request, "core/references/form.html", {"form": form, "title": title}) |
||||
|
||||
|
||||
def _reference_archive(request, model, reference_id, redirect_name): |
||||
item = get_object_or_404(model, pk=reference_id) |
||||
try: |
||||
archive(item, actor=LOCAL_ACTOR) |
||||
except DomainError as exc: |
||||
messages.error(request, _domain_message(exc)) |
||||
else: |
||||
messages.success(request, "Reference archived.") |
||||
return redirect(redirect_name) |
||||
|
||||
|
||||
def _domain_message(error): |
||||
return DOMAIN_MESSAGES.get(error.code, UNKNOWN_DOMAIN_MESSAGE) |
||||
|
||||
|
||||
def _query_message(code): |
||||
return { |
||||
"invalid_date": "Enter a valid date.", |
||||
"before_date_from": "The end date must be on or after the start date.", |
||||
"invalid_uuid": "Select a valid reference.", |
||||
"invalid_choice": "Select a valid choice.", |
||||
"invalid_currency": "Enter a three-letter uppercase currency code.", |
||||
}.get(code, "Invalid filter.") |
||||
|
||||
|
||||
def _monthly_report(request, template_name): |
||||
currencies = available_currencies() or ("BRL",) |
||||
parameters = { |
||||
"month": request.GET.get("month", timezone.localdate().strftime("%Y-%m")), |
||||
"currency": request.GET.get("currency", currencies[0]), |
||||
} |
||||
form = ReportSelectionForm(parameters, currencies=currencies) |
||||
context = {"selection_form": form} |
||||
status = 200 |
||||
if form.is_valid(): |
||||
month = form.cleaned_data["month"] |
||||
currency = form.cleaned_data["currency"] |
||||
context.update({ |
||||
"summary": monthly_summary(month, currency), |
||||
"spending_rows": spending_by_category(month, currency), |
||||
"balances": account_balances(month, currency), |
||||
}) |
||||
else: |
||||
status = 400 |
||||
return render(request, template_name, context, status=status) |
||||
@ -0,0 +1,13 @@
|
||||
#!/bin/sh |
||||
set -eu |
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --set=migration_password="$MIGRATION_PASSWORD" --set=runtime_password="$RUNTIME_PASSWORD" <<'SQL' |
||||
CREATE ROLE gizmoduck_migrate LOGIN PASSWORD :'migration_password' NOSUPERUSER NOCREATEDB NOCREATEROLE; |
||||
CREATE ROLE gizmoduck_app LOGIN PASSWORD :'runtime_password' NOSUPERUSER NOCREATEDB NOCREATEROLE; |
||||
REVOKE ALL ON DATABASE gizmoduck FROM PUBLIC; |
||||
GRANT CONNECT ON DATABASE gizmoduck TO gizmoduck_migrate, gizmoduck_app; |
||||
REVOKE ALL ON SCHEMA public FROM PUBLIC; |
||||
ALTER SCHEMA public OWNER TO gizmoduck_migrate; |
||||
GRANT USAGE ON SCHEMA public TO gizmoduck_app; |
||||
ALTER DEFAULT PRIVILEGES FOR ROLE gizmoduck_migrate IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO gizmoduck_app; |
||||
ALTER DEFAULT PRIVILEGES FOR ROLE gizmoduck_migrate IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO gizmoduck_app; |
||||
SQL |
||||
@ -0,0 +1,6 @@
|
||||
#!/bin/sh |
||||
set -eu |
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --set=test_password="$POSTGRES_PASSWORD" <<'SQL' |
||||
CREATE ROLE gizmoduck_test LOGIN PASSWORD :'test_password' NOSUPERUSER CREATEDB NOCREATEROLE; |
||||
ALTER DATABASE gizmoduck_tests OWNER TO gizmoduck_test; |
||||
SQL |
||||
@ -0,0 +1,55 @@
|
||||
# Architecture |
||||
|
||||
## Binding stack |
||||
|
||||
Use Python and Django for the application, PostgreSQL for persistent storage, |
||||
schema migrations from the beginning, Docker Compose for local infrastructure, |
||||
and pytest for automated tests. Changes require explicit owner approval and a |
||||
documented architectural decision. |
||||
|
||||
Dependency versions and tooling will be selected in the application-skeleton |
||||
design. This foundation does not prescribe a database schema or UI framework. |
||||
|
||||
## Import boundaries |
||||
|
||||
```text |
||||
source document |
||||
-> source-specific parsing |
||||
-> normalization |
||||
-> validation |
||||
-> duplicate detection |
||||
-> preview |
||||
-> explicit commit and persistence |
||||
-> categorization and reporting |
||||
``` |
||||
|
||||
Bank-account parsers consume CSV; Banco do Brasil/Ourocard card parsers consume |
||||
TXT. OFX is unsupported. Parsers are pure transformations of supplied input; |
||||
they do not query or write the database. Normalization maps parser output into |
||||
a common candidate representation without committing transactions. |
||||
|
||||
Validation reports issues for review. Duplicate detection compares candidates |
||||
with the relevant import and persisted records through a separate boundary. |
||||
Preview displays candidates and issues without committing financial records. |
||||
Persistence owns database writes, migrations, and commit integrity. Parser |
||||
grammars, candidate contracts, duplicate rules, and transaction boundaries are |
||||
specified and tested when the import subsystem is designed. |
||||
|
||||
## Data and accountability |
||||
|
||||
Preserve import provenance. Imports and material data changes record actor, |
||||
source, timestamp, and outcome so human, application, and automated activity can |
||||
be distinguished. Keep sensitive audit details in local storage governed by |
||||
the [data policy](data-policy.md). |
||||
|
||||
Use separate least-privilege database accounts for development and application |
||||
processes. Credentials belong in local environment configuration, never in |
||||
tracked source. The skeleton design must define role provisioning and migration |
||||
permissions. Schema changes must use migrations, not manual database drift. |
||||
|
||||
## Delivery boundaries |
||||
|
||||
The financial domain model, UI, importer grammar, duplicate algorithm, deployment |
||||
model, and backup implementation each require design before substantial work. |
||||
Follow the [roadmap](roadmap.md) for sequencing and [product scope](product.md) |
||||
for acceptance boundaries. |
||||
@ -0,0 +1,52 @@
|
||||
# Financial data policy |
||||
|
||||
## Authorized local use |
||||
|
||||
Agents may read every file in the Robopato repository and copy real financial |
||||
documents for local development, parser validation, migration, and compatibility |
||||
testing. This permission does not authorize committing that material or sending |
||||
it to external services. |
||||
|
||||
Real financial data may exist only in ignored local directories and local |
||||
databases. Use these repository-root directories: |
||||
|
||||
| Location | Purpose | |
||||
| --- | --- | |
||||
| `data/` | Local working data and database storage | |
||||
| `documents/` | Source documents copied from Robopato | |
||||
| `imports/` | Local import inputs and intermediate results | |
||||
| `uploads/` and `media/` | Application-managed local files | |
||||
| `exports/` | User exports and generated reports | |
||||
| `backups/` | Database and application backups | |
||||
|
||||
Verify the destination with `git check-ignore` before copying financial data. |
||||
Keep local credentials in ignored `.env` files; `.env.example` may contain only |
||||
non-secret examples. Keep sensitive execution output out of shared transcripts. |
||||
|
||||
## Git boundary |
||||
|
||||
Never add real financial data to Git. This includes raw documents, database |
||||
dumps, copied excerpts, snapshots, logs, screenshots, generated reports, and |
||||
documentation containing identifying transaction data. Ignoring a file does |
||||
not remove data that was already tracked. Never use force-add to bypass this |
||||
policy. |
||||
|
||||
Committed tests and examples must use synthetic or irreversibly sanitized data. |
||||
Use `tests/fixtures/synthetic/` for explicitly synthetic CSV, TXT, and PDF |
||||
fixtures. A fixture exception in `.gitignore` is not evidence that its contents |
||||
are safe. Sanitization must remove identifying information and prevent recovery |
||||
of the original sensitive data; changing a name alone is insufficient. |
||||
|
||||
Before every commit, inspect `git diff --cached --name-only` and |
||||
`git diff --cached`, including newly added fixtures, for financial data and |
||||
secrets. When uncertain whether material is safe, stop and ask the owner. |
||||
|
||||
## Runtime handling |
||||
|
||||
Use separate least-privilege development and application database accounts. |
||||
Record actor, source, timestamp, and outcome for imports and material changes. |
||||
Treat audit records, errors containing source rows, and test output derived from |
||||
real documents as financial data. Use synthetic data for shareable diagnostics. |
||||
|
||||
If sensitive material is accidentally tracked, stop propagation and notify the |
||||
owner. Do not push it or rewrite history without explicit authorization. |
||||
@ -0,0 +1,86 @@
|
||||
# Local development |
||||
|
||||
Requires Docker Desktop running Linux containers with Docker Compose. Host |
||||
Python and uv are optional; all application commands run in containers. |
||||
|
||||
## First start (PowerShell) |
||||
|
||||
Copy `.env.example` to `.env` only if `.env` does not already exist. Replace |
||||
all five values with independent random secrets. To generate one in PowerShell: |
||||
|
||||
```powershell |
||||
[Convert]::ToHexString([System.Security.Cryptography.RandomNumberGenerator]::GetBytes(32)) |
||||
``` |
||||
|
||||
Never share `.env` or output from `docker compose config` without `--quiet`. |
||||
|
||||
```powershell |
||||
docker compose config --quiet |
||||
docker compose build |
||||
docker compose up -d db |
||||
docker compose run --rm manage migrate |
||||
docker compose up -d web |
||||
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||
``` |
||||
|
||||
The health endpoint returns `{"status":"ok"}` when PostgreSQL is reachable, |
||||
and HTTP 503 with `{"status":"unavailable"}` when database access fails. |
||||
It does not verify schema currency. The dashboard is available at `/`. |
||||
The server binds only to loopback on the host. Ports 8000 and 5432 must be |
||||
available. PostgreSQL is exposed only at `127.0.0.1:5432` for local tools; use |
||||
database `gizmoduck`, runtime role `gizmoduck_app`, and the ignored `.env` value |
||||
of `RUNTIME_PASSWORD` for routine inspection. This is a local development server. |
||||
|
||||
## Verification |
||||
|
||||
```powershell |
||||
docker compose run --rm manage check |
||||
docker compose run --rm manage makemigrations --check --dry-run |
||||
docker compose run --rm manage migrate --check |
||||
docker compose run --rm test |
||||
docker compose run --rm permissions |
||||
``` |
||||
|
||||
The test suite covers the complete synthetic local MVP workflow. The |
||||
runtime-permission check exercises DML grants on both `finance_*` and |
||||
`ingestion_*` tables. See [reviewed imports](imports.md) for the explicit local |
||||
preview, decision, and commit workflow, and [local owner workflows](local-workflows.md) |
||||
for categorization, reports, exports, backups, and recovery. |
||||
|
||||
The test service starts its own PostgreSQL container. Its dedicated non-superuser |
||||
role can create the temporary pytest database. The test database uses tmpfs; |
||||
stopping the test database discards only its synthetic test data. The permissions |
||||
command checks real runtime grants on the migrated development schema, using |
||||
synthetic group rows in rolled-back transactions (sequence gaps are normal). |
||||
It verifies DML, sequence access, denied schema/table creation and role escalation, |
||||
and absence of database creation and administrator privileges. |
||||
|
||||
Runtime `gizmoduck_app` has only DML and sequence access. Management commands use |
||||
`gizmoduck_migrate`, which owns the schema. Bootstrap administrator credentials |
||||
are passed only to the database service. Web startup does not run migrations. |
||||
|
||||
## Changes and lifecycle |
||||
|
||||
Rebuild with `docker compose build` after editing application or test code; |
||||
then use `docker compose up -d web` to replace the web container. Dependencies |
||||
are installed from `uv.lock` with `uv sync --frozen`. To intentionally update |
||||
that lockfile, use uv with Python 3.13 and run `uv lock`, then rebuild and verify. |
||||
|
||||
```powershell |
||||
docker compose stop web db |
||||
docker compose --profile test stop test-db |
||||
docker compose --profile backup stop restore-db |
||||
``` |
||||
|
||||
Development data persists in the project-scoped `postgres_data` Docker volume. |
||||
Do not delete it to fix setup problems. Database initialization scripts run only |
||||
when the volume is empty. Editing `.env` does not rotate credentials stored in |
||||
an existing database: coordinate an explicit database password change and update |
||||
the corresponding local environment value together. Schema privilege changes |
||||
also require an explicit administrator/migration action on existing volumes. |
||||
Preserve or back up valuable data before any maintenance. |
||||
|
||||
If a startup fails, first confirm Docker is running, configuration is complete, |
||||
and ports 8000 and 5432 are free. Check service status with |
||||
`docker compose ps`; database or application logs may contain sensitive details |
||||
and must remain local. |
||||
@ -0,0 +1,110 @@
|
||||
# Financial domain developer guide |
||||
|
||||
Use `finance.services` for application writes. The ORM remains useful for reads; |
||||
`Model.save`, `QuerySet.update`, bulk creation, and direct SQL are not supported |
||||
application mutation APIs. Database checks enforce row-local rules, while the |
||||
services enforce cross-record currency, account, archive, and relationship rules. |
||||
There are no finance HTTP endpoints or editable admin models yet. |
||||
|
||||
## Recording activity |
||||
|
||||
```python |
||||
from datetime import date |
||||
from decimal import Decimal |
||||
from finance.services import Actor, create_account, record_transaction |
||||
|
||||
actor = Actor("HUMAN", "local-owner") |
||||
account = create_account(name="Example bank", kind="BANK", actor=actor) |
||||
row = record_transaction( |
||||
account_id=account.pk, amount=Decimal("-12.50"), kind="EXPENSE", |
||||
posting_date=date(2026, 9, 6), description="Synthetic example", actor=actor, |
||||
) |
||||
``` |
||||
|
||||
Examples are synthetic and are not automatically loaded. Service operations |
||||
must be called outside an existing `transaction.atomic` block. Each owns its |
||||
outermost transaction, commits its success audit with the mutation, and writes |
||||
a sanitized failure audit after rollback. Composite operations use private |
||||
helpers within one transaction. Invalid actor identities or caller-owned |
||||
transactions are rejected before mutation and cannot produce a trusted, |
||||
independently committed audit. Database failures are allowed to propagate. |
||||
|
||||
Actor identifiers are caller-supplied accountability labels, not authentication. |
||||
A future authenticated interface must derive them from trusted identity rather |
||||
than accept arbitrary client input. Never use raw source payloads as identifiers. |
||||
`DomainError.code` is a safe machine-readable code; consumers must not expose |
||||
underlying database exceptions. Failure events have no target for creations that |
||||
rolled back. Audits contain field names, not transaction values or source text. |
||||
|
||||
## Public operations |
||||
|
||||
- `create_account`, `create_payee`, `create_category`: create reference data. |
||||
- `archive(entity, actor=...)`: archive an Account, Payee, or Category. |
||||
- `create_statement`: group observed card activity with optional quoted balance. |
||||
- `create_plan`: describe installment count without generating future charges. |
||||
- `record_transaction`: post one observed movement. |
||||
- `create_transfer`: atomically create outgoing/incoming legs and their link. |
||||
- `match_transfer`: link two existing unmatched legs without posting more money. |
||||
- `attach_source`: attach unique document/item provenance to a transaction. |
||||
- `annotate_transaction`: update only description, payee, and category. |
||||
|
||||
Each create/match/attach operation returns its saved model instance. All related |
||||
inputs are UUIDs named `account_id`, `category_id`, etc. All public writes require |
||||
`actor=Actor(kind, identifier)` with HUMAN, APPLICATION, or AGENT kind. |
||||
|
||||
## Financial conventions |
||||
|
||||
Use Decimal, decimal strings, or integers for money; floats, booleans, non-finite |
||||
values, zero movement, more than two fractional digits, and out-of-range amounts |
||||
are rejected before database rounding. Amounts have 18 total digits and two |
||||
fractional digits. Statement metadata may be zero. |
||||
|
||||
Positive amounts improve the owner's position; negative amounts reduce it. |
||||
Purchases on a credit card are negative. Paying a card is a negative bank leg |
||||
and a positive card leg, both TRANSFER. Refunds are positive separate records. |
||||
Opening entries are unique per account and excluded from income/spending. |
||||
|
||||
Balances are `Sum("amount")` per account. Never aggregate balances across currency |
||||
codes. Net spending is the negated sum of EXPENSE and REFUND amounts within one |
||||
currency and the desired date range. Transfers and opening entries are excluded. |
||||
No FX conversions or combined-currency summaries are implemented. |
||||
|
||||
Unmatched TRANSFER entries are expected when only one account has been imported. |
||||
Matching validates currency, distinct accounts, opposite equal amounts, and |
||||
unique use of each leg. Account and transaction locks are acquired in UUID order; |
||||
reference locks precede transaction locks to avoid opposing lock order. |
||||
|
||||
Linked refunds target an expense in the same currency, possibly on another |
||||
account; partial and over-refunds are allowed. Installment membership requires |
||||
the same account, a one-based number within the plan total, and a unique |
||||
(plan, number). Posting installment 3/10 does not create any other installment. |
||||
Statement balances are quoted metadata and need not equal partially imported rows. |
||||
|
||||
Accounts' currencies, posted financial fields, and plan definitions are immutable |
||||
through the supported API. Only transaction annotations can change. Existing |
||||
archived assignments remain on history; they cannot be newly assigned. Referenced |
||||
records are protected from deletion. Financial correction/reversal workflows are |
||||
not exposed in this milestone. |
||||
|
||||
Source identity is (document SHA-256, item locator). Changing a parser version |
||||
does not allow the same physical item to be attached twice. Multiple distinct |
||||
source items can explicitly describe the same transaction. Semantic duplicate |
||||
detection and file parsing belong to the next milestone. |
||||
|
||||
## Verification |
||||
|
||||
Follow [local setup](development.md), then run: |
||||
|
||||
```powershell |
||||
docker compose build |
||||
docker compose run --rm manage migrate |
||||
docker compose run --rm test |
||||
docker compose run --rm permissions |
||||
docker compose run --rm manage makemigrations --check --dry-run |
||||
docker compose run --rm manage migrate --check |
||||
``` |
||||
|
||||
The full PostgreSQL suite includes direct constraint violations, protected |
||||
history, precision rejection, audit rollback, relationships, and concurrent |
||||
transfer matching with separate database sessions. The permissions command |
||||
checks grants on all finance tables and rolls back synthetic runtime account DML. |
||||
@ -0,0 +1,165 @@
|
||||
# Reviewed local imports |
||||
|
||||
Imports require a saved preview and explicit row decisions before financial |
||||
records are written. The `import_review` management command supports `preview`, |
||||
`export`, `apply`, `refresh`, `commit`, and `cancel`. Every operation requires |
||||
`--actor-kind HUMAN|APPLICATION|AGENT` and `--actor-identifier`. |
||||
|
||||
Keep source documents, mappings and review JSON in the ignored repository-root |
||||
`imports/` directory. The management container mounts it at `/app/imports`. |
||||
Command paths may be absolute under that root or relative to it; traversal and |
||||
symlink escapes are rejected. Output files must not already exist: choose a new |
||||
filename for every export. Input documents and JSON files are limited to 10 MiB. |
||||
Review files contain financial descriptions and must never enter Git, logs or |
||||
external services. See [data policy](data-policy.md). |
||||
|
||||
## PowerShell workflow |
||||
|
||||
Run the setup and migrations in [development](development.md) first. Create the |
||||
destination bank account or card through the existing financial services; use |
||||
its UUID below. These names and UUIDs are illustrative, synthetic examples. |
||||
|
||||
```powershell |
||||
New-Item -ItemType Directory -Force imports | Out-Null |
||||
git check-ignore imports/source.csv imports/mapping.json imports/review-1.json |
||||
@' |
||||
{"1":"11111111-1111-4111-8111-111111111111"} |
||||
'@ | Set-Content -Encoding utf8 imports/mapping.json |
||||
$actor = @('--actor-kind', 'HUMAN', '--actor-identifier', 'local-owner') |
||||
docker compose run --rm manage import_review preview ` |
||||
--input source.csv --profile BANK_CSV_V1 --mapping mapping.json @actor |
||||
``` |
||||
|
||||
Place the source CSV in `imports/source.csv` before previewing it. Substitute |
||||
the batch UUID printed by `preview`, then export its review: |
||||
|
||||
```powershell |
||||
$batch = '22222222-2222-4222-8222-222222222222' |
||||
docker compose run --rm manage import_review export ` |
||||
--batch $batch --output review-1.json @actor |
||||
notepad imports/review-1.json |
||||
``` |
||||
|
||||
Read `items`, including candidate amounts/dates, validation issues and duplicate |
||||
suggestions. Edit only the top-level `decisions` array. Preserve every other |
||||
field, including each item's stored `decision`. Alternatively save a minimal |
||||
object with exactly `batch_id`, integer `revision`, and `decisions`. Unknown |
||||
keys, repeated JSON keys, repeated locators, foreign batch items, stale |
||||
revisions and changes to source evidence are rejected. |
||||
|
||||
```json |
||||
{ |
||||
"batch_id": "22222222-2222-4222-8222-222222222222", |
||||
"revision": 1, |
||||
"decisions": [ |
||||
{"locator": "line:2", "action": "CREATE"}, |
||||
{"locator": "line:3", "action": "SKIP", "reason": "USER_EXCLUDED"} |
||||
] |
||||
} |
||||
``` |
||||
|
||||
Use the actual locators in the exported file. Apply the complete decision set, |
||||
then export again to inspect the resulting issues and latest revision: |
||||
|
||||
```powershell |
||||
docker compose run --rm manage import_review apply ` |
||||
--batch $batch --input review-1.json @actor |
||||
docker compose run --rm manage import_review export ` |
||||
--batch $batch --output review-2.json @actor |
||||
notepad imports/review-2.json |
||||
$revision = 2 # Replace with the revision printed by apply/export. |
||||
docker compose run --rm manage import_review commit ` |
||||
--batch $batch --revision $revision @actor |
||||
``` |
||||
|
||||
Only `commit` writes financial transactions, source provenance and new |
||||
installment plans. It posts all accepted rows atomically. Repeating a successful |
||||
commit with its latest revision returns the recorded result. If live account, |
||||
reference or duplicate information has materially changed, refresh and review |
||||
again. Refresh increments the revision and clears all decisions: |
||||
|
||||
```powershell |
||||
docker compose run --rm manage import_review refresh ` |
||||
--batch $batch --revision $revision @actor |
||||
docker compose run --rm manage import_review export ` |
||||
--batch $batch --output review-refreshed.json @actor |
||||
``` |
||||
|
||||
To abandon a preview, use its current revision: |
||||
|
||||
```powershell |
||||
docker compose run --rm manage import_review cancel ` |
||||
--batch $batch --revision $revision @actor |
||||
``` |
||||
|
||||
Cancellation creates no financial entries and prevents later commitment. |
||||
Terminal output contains batch identifiers, state, revision and counts; inspect |
||||
the local review file for financial details. Errors expose only safe codes. |
||||
|
||||
## Decisions and duplicate handling |
||||
|
||||
Every candidate needs one explicit decision. Applying a file replaces the |
||||
complete decision set. The supported actions are: |
||||
|
||||
- `CREATE`: create a transaction. Optional annotations are `kind`, `category_id`, |
||||
`payee_id`, `statement_id`, `refund_of_id`, `installment_plan_id`, and `new_plan`. |
||||
Possible duplicates require `"acknowledge_duplicate": true` after review. |
||||
- `MATCH`: attach the source to the existing `transaction_id` selected from |
||||
compatible duplicate suggestions. The immutable financial fields and signature |
||||
must still agree. Existing annotations are preserved. |
||||
- `SKIP`: omit the item with a `reason` code: `USER_EXCLUDED`, `MALFORMED`, |
||||
`UNRECOGNIZED`, `ALREADY_IMPORTED`, or `SECTION_EXCLUDED`. Malformed rows and |
||||
unknown lines require explicit skipping; file-level corruption cannot be |
||||
overridden. Exact already-imported source items cannot be created or matched |
||||
again; acknowledge them with `SKIP` / `ALREADY_IMPORTED`. |
||||
|
||||
For example, an installment decision can include |
||||
`"new_plan": "synthetic-plan-a"`. This is a batch-scoped plan name, not an |
||||
existing UUID; the account and installment count come from the candidate. |
||||
Reuse the same name only for rows belonging to the same plan, or select an |
||||
existing `installment_plan_id`. Plan identity is never inferred from merchant |
||||
names, and missing installments are never generated. Refunds can remain unlinked |
||||
or use an explicit compatible `refund_of_id`. No missing statements or transfer |
||||
counterparts are created automatically. |
||||
|
||||
Section mapping JSON maps ordinal string keys such as `"1"` and `"2"` to account |
||||
UUIDs, or `null` to exclude a section. Map every section explicitly; several |
||||
sections may share one account. Owner names are never database lookup keys. |
||||
Correct mappings or immutable dates/amounts by creating a new preview from a |
||||
corrected input. They cannot be rewritten in decisions. |
||||
|
||||
## Supported compatibility profiles |
||||
|
||||
All profiles decode UTF-8 (including BOM), then Latin-1 without replacement |
||||
characters. NUL bytes, files over 10 MiB and more than 20,000 candidates are |
||||
rejected. Initial profiles require BRL destination accounts. Source dates become |
||||
both occurrence and posting dates. OFX and PDF are unsupported. |
||||
|
||||
| Profile | Accepted layout | |
||||
| --- | --- | |
||||
| `BANK_CSV_V1` | Bank account CSV, comma-delimited with CSV quoting, at least six header columns; date at column 0 as `DD/MM/YYYY`, description at 2, signed decimal-point amount at 5. Each record must match the header width. | |
||||
| `OUROCARD_FULL_V1` | Card TXT, `DD.MM.YYYY`, 37-character description/location field, two-character country field, debit and credit in Brazilian decimal notation. A separate `PGTO DEBITO CONTA` layout is recognized. | |
||||
| `OUROCARD_PARTIAL_V1` | Card TXT, `DD/MM`, 43-character description/location field, country, debit and credit; requires an explicit date interval. | |
||||
|
||||
CSV header aliases are case/accent insensitive: date accepts `Data`, |
||||
`Data movimento`, `Data do movimento`, or `Data lancamento`; description accepts |
||||
`Historico`, `Descricao`, or `Descricao do lancamento`; amount accepts `Valor`, |
||||
`Valor lancamento`, or `Valor do lancamento`. Other columns are retained only as |
||||
layout positions. Recognized balance rows are excluded and counted. |
||||
|
||||
Card amounts use credit minus debit. Nonzero amounts in both columns block |
||||
creation pending explicit review; known account-debit payment lines propose an |
||||
unmatched positive transfer. Card credits propose refunds, bank credits income, |
||||
and debits expenses. `PARC n/total` supplies installment metadata. Section |
||||
headings establish ordinal mappings; unknown nonblank text remains visible for |
||||
acknowledgement, and transaction-like malformed lines are never silently lost. |
||||
|
||||
Yearless dates require both `--context-start` and `--context-end` as ISO dates |
||||
in an inclusive interval of at most 366 days. A date must resolve uniquely; |
||||
the importer never guesses the current year. For example: |
||||
|
||||
```powershell |
||||
docker compose run --rm manage import_review preview ` |
||||
--input card.txt --profile OUROCARD_PARTIAL_V1 --mapping card-mapping.json ` |
||||
--context-start 2025-12-01 --context-end 2026-01-31 @actor |
||||
``` |
||||
@ -0,0 +1,215 @@
|
||||
# Local owner workflows |
||||
|
||||
This guide covers the local MVP from setup through import, categorization, |
||||
reporting, export, and verified backup. Commands assume PowerShell in the |
||||
repository root. Financial source files, review JSON, exports, backups, and |
||||
command output that contains financial details must remain local. |
||||
|
||||
## Start the application |
||||
|
||||
Follow the secret setup in [local development](development.md), then run: |
||||
|
||||
```powershell |
||||
docker compose config --quiet |
||||
docker compose build |
||||
docker compose up -d db |
||||
docker compose run --rm manage migrate |
||||
docker compose up -d web |
||||
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||
``` |
||||
|
||||
Open `http://127.0.0.1:8000/`. The application and PostgreSQL listen only on |
||||
loopback. The web UI uses port `8000`; PostgreSQL uses `127.0.0.1:5432` for |
||||
local database tools such as DBeaver. The database name is `gizmoduck`. Use |
||||
`gizmoduck_app` with the `RUNTIME_PASSWORD` value from the ignored `.env` for |
||||
routine inspection. The migration and bootstrap roles have broader privileges |
||||
and are for maintenance only. |
||||
|
||||
The navigation provides Accounts, Transactions, Categories, Payees, Reports, |
||||
and Exports. Create accounts, categories, and payees before assigning them. |
||||
References may be archived, but historical transactions continue to show them. |
||||
Transaction editing is limited to description, payee, and category. Amount, |
||||
dates, account, kind, import provenance, transfers, refunds, statements, and |
||||
installment identity are immutable. Corrections use a new transaction or the |
||||
domain workflows described in [the financial domain guide](financial-domain.md). |
||||
|
||||
## Import reviewed transactions |
||||
|
||||
Create the destination account at `http://127.0.0.1:8000/accounts/new/`. In |
||||
DBeaver, retrieve its UUID without exposing transaction data: |
||||
|
||||
```sql |
||||
SELECT id, name, kind, currency, archived |
||||
FROM finance_account |
||||
ORDER BY name, id; |
||||
``` |
||||
|
||||
Copy the source document into the ignored `imports/` directory. These commands |
||||
use the tracked synthetic CSV; replace `source.csv` and the UUID for a real |
||||
local import. Never copy a real document into `tests/fixtures/`. |
||||
|
||||
```powershell |
||||
New-Item -ItemType Directory -Force imports | Out-Null |
||||
Copy-Item tests/fixtures/synthetic/ingestion/bank.csv imports/source.csv |
||||
git check-ignore imports/source.csv imports/mapping.json imports/review-1.json |
||||
$account = '11111111-1111-4111-8111-111111111111' # Replace with the account UUID. |
||||
@{"1"=$account} | ConvertTo-Json | Set-Content -Encoding utf8 imports/mapping.json |
||||
$actor = @('--actor-kind', 'HUMAN', '--actor-identifier', 'local-owner') |
||||
docker compose run --rm manage import_review preview ` |
||||
--input source.csv --profile BANK_CSV_V1 --mapping mapping.json @actor |
||||
``` |
||||
|
||||
Copy the printed batch UUID, export the review, and inspect every candidate, |
||||
issue, and duplicate suggestion: |
||||
|
||||
```powershell |
||||
$batch = '22222222-2222-4222-8222-222222222222' # Replace with the printed batch UUID. |
||||
docker compose run --rm manage import_review export ` |
||||
--batch $batch --output review-1.json @actor |
||||
notepad imports/review-1.json |
||||
``` |
||||
|
||||
Edit the top-level `decisions` array. Every row needs `CREATE`, `MATCH`, or |
||||
`SKIP`; use the actual locators and current revision from the review. Apply the |
||||
file, export a fresh review under a new name, and commit only when all issues are |
||||
resolved: |
||||
|
||||
```powershell |
||||
docker compose run --rm manage import_review apply ` |
||||
--batch $batch --input review-1.json @actor |
||||
docker compose run --rm manage import_review export ` |
||||
--batch $batch --output review-2.json @actor |
||||
notepad imports/review-2.json |
||||
$revision = 2 # Replace with the revision printed by apply/export. |
||||
docker compose run --rm manage import_review commit ` |
||||
--batch $batch --revision $revision @actor |
||||
``` |
||||
|
||||
Only `commit` writes transactions and provenance. Preview, export, and apply are |
||||
inert. See [reviewed imports](imports.md) for all profiles, decision fields, |
||||
duplicate handling, refresh, cancellation, and Ourocard date contexts. |
||||
|
||||
## Categorize and report |
||||
|
||||
Open `http://127.0.0.1:8000/transactions/`, select an imported transaction, and |
||||
choose **Edit annotations**. Assign an active category or payee and save. These |
||||
POST forms use CSRF protection and record the fixed local human actor |
||||
`HUMAN/local-owner` in the audit trail. |
||||
|
||||
Open `http://127.0.0.1:8000/reports/monthly/`, select one calendar month and one |
||||
currency, then choose **Show report**. Reports use posting dates. Income is |
||||
inflow; expenses increase spending; refunds reduce spending in their posting |
||||
month. Transfers, including card-payment transfers, and opening balances are |
||||
excluded from cash flow. A card purchase is counted once as an expense. Spending |
||||
is grouped by expense category with a visible Uncategorized row, and account |
||||
balances remain separate by account and currency. |
||||
|
||||
## Export portable data |
||||
|
||||
Use `http://127.0.0.1:8000/exports/` to download a ZIP, or create one under the |
||||
ignored `exports/` directory: |
||||
|
||||
```powershell |
||||
New-Item -ItemType Directory -Force exports | Out-Null |
||||
git check-ignore exports/local-export.zip |
||||
docker compose run --rm manage export_finances --output local-export.zip |
||||
``` |
||||
|
||||
Choose a new plain `.zip` name for each run; exports never overwrite an existing |
||||
file. The ZIP contains UTF-8 CSV files for accounts, payees, categories, card |
||||
statements, installment plans, transactions, transfers, and source records. Its |
||||
`manifest.json` records schema version, creation time, row counts, and SHA-256 |
||||
checksums. Audit events and unfinished import reviews are excluded. Command |
||||
output contains only the package name, format version, and row counts. |
||||
|
||||
## Back up and verify restoration |
||||
|
||||
Create a PostgreSQL 17 custom-format dump and checksum manifest under the |
||||
ignored `backups/` directory: |
||||
|
||||
```powershell |
||||
New-Item -ItemType Directory -Force backups | Out-Null |
||||
git check-ignore backups/local-backup.dump backups/local-backup.dump.json |
||||
docker compose run --rm manage backup_database --output local-backup.dump |
||||
``` |
||||
|
||||
Choose a new plain `.dump` name for each backup. The command never overwrites an |
||||
existing dump or manifest. Its output contains only format and table counts. |
||||
The manifest records the source database, PostgreSQL client version, migrations, |
||||
row counts, byte size, and SHA-256 digest. |
||||
|
||||
The supported verification command creates synthetic data in disposable |
||||
`test-db`, backs it up, restores it into disposable `restore-db`, checks |
||||
migrations and table counts, then clears both databases. It deliberately runs |
||||
two complete cycles to prove cleanup and repeatability: |
||||
|
||||
```powershell |
||||
docker compose --profile backup run --rm verify-backup python manage.py test_backup_restore |
||||
``` |
||||
|
||||
This acceptance check never restores the live `db/gizmoduck` database. Verify a |
||||
named backup against the empty disposable restore database with: |
||||
|
||||
```powershell |
||||
$backup = 'local-backup.dump' # Replace with the exact backup name. |
||||
docker compose --profile backup run --rm verify-backup python manage.py verify_backup ` |
||||
--input $backup --target-name gizmoduck_restore ` |
||||
--target-host restore-db --target-user gizmoduck_restore |
||||
docker compose --profile backup rm -sf restore-db |
||||
``` |
||||
|
||||
The named verification checks its manifest and digest, restores it, and compares |
||||
migrations and table counts. Removing `restore-db` afterward clears only the |
||||
disposable tmpfs target so the next verification starts empty. The command |
||||
rejects the configured live host and database pair. |
||||
|
||||
## Deliberate live recovery |
||||
|
||||
Live recovery replaces financial state. Perform it only after identifying the |
||||
exact dump and its matching `.json` manifest. Keep the web service stopped until |
||||
all restore checks pass. |
||||
|
||||
First stop writes and take a separate safety backup of the current live database: |
||||
|
||||
```powershell |
||||
docker compose stop web |
||||
$safety = 'pre-restore-safety-20260908T120000Z.dump' # Use a new timestamped name. |
||||
docker compose run --rm manage backup_database --output $safety |
||||
``` |
||||
|
||||
Verify the intended named backup with the disposable procedure above before |
||||
touching the live database. Then name both the backup and live target |
||||
explicitly. The following PostgreSQL 17 command restores into `db/gizmoduck`; |
||||
`--clean` replaces objects represented by the backup, and the single transaction |
||||
rolls back on an error: |
||||
|
||||
```powershell |
||||
$backup = 'local-backup.dump' # Confirm this exact file and matching manifest first. |
||||
$targetHost = 'db' |
||||
$targetDatabase = 'gizmoduck' |
||||
docker compose run --rm -e BACKUP_FILE=$backup -e TARGET_HOST=$targetHost ` |
||||
-e TARGET_DATABASE=$targetDatabase --entrypoint sh manage -c ` |
||||
'PGPASSWORD="$DB_PASSWORD" pg_restore --exit-on-error --single-transaction --clean --if-exists --no-owner --no-acl --host "$TARGET_HOST" --port 5432 --username gizmoduck_migrate --dbname "$TARGET_DATABASE" "/app/backups/$BACKUP_FILE"' |
||||
``` |
||||
|
||||
Do not restart the web process yet. Bring schema state forward if necessary, |
||||
then validate Django and the live database connection: |
||||
|
||||
```powershell |
||||
docker compose run --rm manage migrate |
||||
docker compose run --rm manage check |
||||
docker compose run --rm manage migrate --check |
||||
docker compose run --rm manage shell -c "from django.db import connection; c=connection.cursor(); c.execute('SELECT 1'); print(c.fetchone()[0])" |
||||
``` |
||||
|
||||
Only after those checks succeed, restart the local application and verify its |
||||
readiness endpoint: |
||||
|
||||
```powershell |
||||
docker compose up -d web |
||||
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||
``` |
||||
|
||||
Keep the safety backup until the recovered data has been reviewed. If validation |
||||
fails, leave `web` stopped and investigate locally; logs and database output may |
||||
contain financial data. |
||||
@ -0,0 +1,35 @@
|
||||
# Product scope |
||||
|
||||
Gizmoduck is a privacy-conscious personal finance application for an individual |
||||
managing their own accounts and credit cards locally. It succeeds Robopato, |
||||
retaining lessons from Banco do Brasil statements and Ourocard invoices without |
||||
carrying forward notebook-driven architecture or importing historical data by |
||||
default. |
||||
|
||||
## MVP |
||||
|
||||
- Manage bank accounts and credit cards. |
||||
- Import bank-account CSV and Banco do Brasil/Ourocard credit-card TXT. |
||||
- Preview parsed transactions, validation issues, and suspected duplicates |
||||
before committing an import. |
||||
- Preserve provenance and detect duplicates across imports. |
||||
- Organize transactions by payee and category. |
||||
- Represent transfers, refunds, and installments correctly. |
||||
- Inspect monthly cash flow and spending summaries. |
||||
- Export and back up user-owned data. |
||||
|
||||
A successful MVP supports the local path from source file through reviewed |
||||
import to understandable transactions and summaries, with tests using synthetic |
||||
or irreversibly sanitized fixtures. A failed or cancelled preview must not |
||||
commit financial transactions. Exact importer grammars and financial modeling |
||||
rules require subsystem designs before implementation. |
||||
|
||||
## Exclusions |
||||
|
||||
OFX is not supported. Bank APIs, investment tracking, mobile clients, receipt |
||||
recognition, forecasting, and AI categorization are outside the MVP. Hosted |
||||
multi-user operation is not part of the initial local release. |
||||
|
||||
The owner must approve scope changes. Technical choices are defined in |
||||
[architecture](architecture.md); financial data handling follows the |
||||
[data policy](data-policy.md). |
||||
@ -0,0 +1,45 @@
|
||||
# Delivery roadmap |
||||
|
||||
Each substantial subsystem receives its own design and implementation plan. |
||||
This document orders milestones; it is not a task tracker. |
||||
|
||||
## 1. Contributor foundation |
||||
|
||||
Establish agent instructions, MVP boundaries, binding architecture, and financial |
||||
data policy. Complete when the project documents are linked, consistent, and |
||||
Git exclusions protect local data while synthetic fixtures remain trackable. |
||||
|
||||
## 2. Runnable application skeleton |
||||
|
||||
Design and scaffold Django, PostgreSQL, Docker Compose, dependency tooling, |
||||
environment configuration, database roles, migrations, and pytest. Complete |
||||
when a clean checkout can follow documented setup, start the application, |
||||
apply migrations, and pass a database-backed smoke test. Use no real data. |
||||
|
||||
## 3. Financial domain |
||||
|
||||
Design accounts, credit cards, transactions, payees, categories, transfers, |
||||
refunds, installments, and provenance. Complete when migrations and tests |
||||
demonstrate the agreed financial invariants and relationships. |
||||
|
||||
## 4. Reviewed imports |
||||
|
||||
Design the normalized candidate contract and separate parser, normalization, |
||||
validation, duplicate detection, preview, and persistence stages. Implement |
||||
bank-account CSV and Banco do Brasil/Ourocard TXT using synthetic or |
||||
irreversibly sanitized fixtures. Complete when preview issues are understandable, |
||||
reimport behavior is tested, and committing an import preserves provenance |
||||
without unintended duplicates. OFX remains unsupported. |
||||
|
||||
## 5. Local MVP workflows |
||||
|
||||
Add transaction management, categorization, monthly cash-flow and spending |
||||
summaries, exports, and backups. Complete when an owner can follow the full |
||||
import-to-report workflow and verify backup restoration with synthetic data. |
||||
|
||||
**Status: complete.** The local owner workflow, portable export, PostgreSQL 17 |
||||
backup, and repeatable disposable restore path are documented and covered by the |
||||
complete synthetic acceptance suite. |
||||
|
||||
Bank APIs, investments, mobile clients, receipt recognition, forecasting, and |
||||
AI categorization require future scope decisions. See [product scope](product.md). |
||||
@ -0,0 +1,60 @@
|
||||
# Agent Foundation Implementation Plan |
||||
|
||||
> **For agentic workers:** Use superpowers:executing-plans to implement this plan inline, task by task. |
||||
|
||||
**Goal:** Establish the approved contributor guidance and privacy boundaries. |
||||
|
||||
**Architecture:** Keep AGENTS.md concise and authoritative. Separate product, |
||||
architecture, data policy, and delivery sequence into linked documents. |
||||
|
||||
**Tech Stack:** Markdown and Git; the application will use Python, Django, |
||||
PostgreSQL, Docker Compose, and pytest. |
||||
|
||||
**Spec:** [Approved foundation design](../specs/2026-09-06-agent-foundation-design.md). |
||||
|
||||
## Global constraints |
||||
|
||||
- The project is named Gizmoduck and is the clean successor to Robopato. |
||||
- The backend uses Python and Django. |
||||
- Persistent storage uses PostgreSQL with migrations from the beginning. |
||||
- Local infrastructure uses Docker Compose. |
||||
- Automated tests use pytest. |
||||
- Bank-account transactions are imported from CSV. |
||||
- Credit-card transactions are imported from Banco do Brasil/Ourocard TXT. |
||||
- OFX is not supported. |
||||
- Parsing, normalization, validation, duplicate detection, preview, and |
||||
persistence remain separate stages. |
||||
- Real financial data and secrets must never enter Git. |
||||
- Preserve existing README and ignore-file content while making targeted updates. |
||||
|
||||
## Task 1: Publish contributor and project guidance |
||||
|
||||
**Files:** Create AGENTS.md, docs/product.md, docs/architecture.md, |
||||
docs/data-policy.md, docs/roadmap.md; update README.md. |
||||
|
||||
**Interfaces:** Consumes the approved spec; produces linked contributor policy. |
||||
|
||||
- [x] Write AGENTS.md with binding constraints, reading order, operating rules, |
||||
local commit identity, and links to the four project documents. |
||||
- [x] Write product scope and exclusions, architecture boundaries, permitted |
||||
Robopato access and local-data locations, and milestone acceptance criteria. |
||||
- [x] Link the documents from README and align its architecture language with |
||||
the approved binding choices. |
||||
- [x] Check that every relative Markdown link resolves and review all documents |
||||
against the approved spec for contradictions and unresolved policy choices. |
||||
|
||||
## Task 2: Verify data exclusions and readiness |
||||
|
||||
**Files:** Update .gitignore only where approved protections are missing. |
||||
|
||||
**Interfaces:** Consumes data-policy locations; produces Git exclusion rules. |
||||
|
||||
- [x] Exclude raw financial CSV/TXT/PDF and database dump formats, keeping |
||||
explicitly synthetic fixture paths trackable. Preserve existing rules. |
||||
- [x] Use git check-ignore with representative paths under each local-data |
||||
directory, secret and dump formats, and synthetic fixtures. Confirm raw data |
||||
is ignored and synthetic fixtures remain trackable without creating real data. |
||||
- [x] Run git diff --check; verify repository-local user.name and user.email |
||||
equal Codex and codex@localhost; inspect changed files for secrets/data. |
||||
- [x] Report the completed foundation and next application-skeleton milestone. |
||||
Leave existing user edits and new documents available for review locally. |
||||
@ -0,0 +1,39 @@
|
||||
# Application Skeleton Implementation Plan |
||||
|
||||
**Goal:** Execute the approved application skeleton design. |
||||
**Spec:** [Design](../specs/2026-09-06-application-skeleton-design.md). |
||||
**Architecture:** Compose services separate runtime, migration, and test access. |
||||
**Stack:** Python 3.13, Django 5.2, PostgreSQL 17, uv and pytest. |
||||
|
||||
- [x] Create locked dependency configuration, Docker image, and minimal Django configuration. Add health success/failure tests and observe failure before implementing the endpoint. |
||||
- [x] Provision development database roles using initialization scripts; run migrations explicitly. Run pytest on an isolated PostgreSQL test service. |
||||
- [x] Add runtime permission checks using rolled-back synthetic rows and denied DDL/role escalation. Verify the real database grants. |
||||
- [x] Document setup and existing-volume behavior, verify HTTP startup, migration drift, system checks, and secret exclusions. Review all changes before completion. |
||||
|
||||
Files: config/, core/, tests/, docker/postgres/, manage.py, pyproject.toml, |
||||
uv.lock, Dockerfile, .dockerignore, compose.yaml, .env.example and README.md. |
||||
No financial models or real data. Application startup never runs migrations. |
||||
Verification commands: docker compose config --quiet; docker compose build; |
||||
docker compose run --rm manage migrate; docker compose run --rm test; |
||||
docker compose run --rm permissions; docker compose run --rm manage check; |
||||
docker compose run --rm manage makemigrations --check --dry-run; |
||||
docker compose run --rm manage migrate --check; request localhost:8000/health/. |
||||
|
||||
## Verification results (2026-09-06) |
||||
|
||||
- Locked Docker build succeeded; Django 5.2.17 installed under Python 3.13. |
||||
- Three pytest tests passed against isolated PostgreSQL, including HTTP success, |
||||
generic database failure, and missing-secret validation. |
||||
- Fourteen built-in migrations applied on a fresh development volume; system |
||||
checks passed and no pending or missing migrations were detected. |
||||
- Runtime permission checks passed for DML, sequences, denied DDL, role |
||||
escalation, and privileged role attributes. |
||||
- Actual HTTP readiness returned 200; a separate container with an unreachable |
||||
database returned generic 503 without connection details. |
||||
- Independent review identified a temporary-server readiness race. Both database |
||||
healthchecks now require TCP. Fresh-volume migrations, permissions and tests |
||||
passed again in the isolated gizmoduck-skeleton-verify Compose project. |
||||
- Verification-project containers are stopped; its synthetic database volume is |
||||
retained. The main local application remains running. |
||||
- Documentation links, whitespace, and candidate files were checked for local |
||||
secret values. Existing foundation changes remain uncommitted with this work. |
||||
@ -0,0 +1,41 @@
|
||||
# Financial Domain Implementation Plan |
||||
|
||||
**Spec:** [Approved domain design](../specs/2026-09-06-financial-domain-design.md) |
||||
**Goal:** Deliver the financial models, constraints, service boundary and PostgreSQL verification. |
||||
**Architecture:** finance/models for persistence; finance/services for audited atomic writes; tests/finance for synthetic acceptance and concurrency checks. |
||||
|
||||
## Interfaces |
||||
|
||||
Models: Account(name,kind,currency,archived), Payee(name,archived), Category(name,kind,archived), CardStatement(account,period_start,period_end,due_date,statement_balance), InstallmentPlan(account,description,total_installments), Transaction(account,amount,posting_date,occurrence_date,description,kind,payee,category,statement,refund_of,installment_plan,installment_number), Transfer(outgoing,incoming), SourceRecord(transaction,document_hash,source_format,parser_version,item_locator), AuditEvent(actor_kind,actor_identifier,operation,target_id,outcome,changed_fields,error_code). All identifiers UUID; timestamps created_at/updated_at except immutable provenance/audit created_at. |
||||
|
||||
Public services take actor=Actor(kind,identifier). Create operations take model scalar fields, related UUID arguments named account_id etc. create_transfer takes outgoing_account_id, incoming_account_id, positive amount, outgoing_date, incoming_date. match_transfer takes outgoing_id/incoming_id. annotate_transaction takes transaction_id and description/payee_id/category_id only. archive takes entity. attach_source takes transaction_id and source metadata. DomainError.code is stable and sanitized. |
||||
|
||||
- [x] Task 1: Implement models and PostgreSQL constraints; create migrations. Owned files finance/models and finance/migrations. Check constraint rejection using direct ORM writes, including sign, uniqueness, and protected deletion. |
||||
- [x] Task 2: Implement services for ordinary posting, reference data, statements, plans and audit. Owned files finance/services. Validate Decimal precision before persistence, serialize references and record success atomically; failures recorded after rollback. |
||||
- [x] Task 3: Implement transfer/refund/installment/provenance operations and synthetic acceptance tests, including concurrent matching in separate PostgreSQL connections. Tests written before service implementation and run red, then green. |
||||
- [x] Task 4: Integrate finance in settings and Docker context, verify fresh/existing migrations and runtime privileges; review code and document developer contracts. Run full pytest, check, makemigrations --check, migrate --check and HTTP health. |
||||
|
||||
Ruling: public write operations own their outermost database transaction. Reject invocation inside an existing atomic block so failure audit events can persist independently after rollback. Composite operations call private helpers in one public transaction, never public services recursively. |
||||
Ruling: archived reference data cannot be newly assigned, but existing historical assignments can remain when only a description is updated. |
||||
Ruling: retain the existing working directory and branch because all previous work is uncommitted here; workers own disjoint files, no overlapping implementation. |
||||
|
||||
|
||||
## Completion evidence |
||||
|
||||
- 83 PostgreSQL tests passed, including database constraints, relationship |
||||
validation, audit rollback, source identity and two-session concurrent matching. |
||||
- Initial service run exposed audit field suffix mismatch (79 passed/1 failed); |
||||
normalized audit field names and reran successfully. |
||||
- Review found Decimal-context rounding in transfer negation. Both new |
||||
regressions failed before the fix; copy_negate() corrected creation and |
||||
matching. Scoped re-review reported no remaining findings. |
||||
- Finance migration applied to the existing skeleton database and a fresh |
||||
gizmoduck-domain-verify database. Runtime grants and synthetic account DML |
||||
passed on both. The verification database container is stopped; its synthetic |
||||
volume is retained. No real financial data was used. |
||||
- Django checks passed with no migration drift or pending migrations. |
||||
- Candidate Git files checked for local secrets; migration/source files remain |
||||
trackable. Documentation links and whitespace verified. |
||||
- Financial APIs and transaction-boundary limits documented in |
||||
docs/financial-domain.md. No financial HTTP UI or import parser added. |
||||
- Changes remain local and uncommitted alongside the prior foundation/skeleton. |
||||
@ -0,0 +1,48 @@
|
||||
# Reviewed Imports Implementation Plan |
||||
|
||||
**Spec:** [Approved design](../specs/2026-09-06-reviewed-imports-design.md) |
||||
**Goal:** Ship pure format parsing, persisted previews, explicit decisions, and atomic audited imports through local commands. |
||||
**Architecture:** ingestion/parsers.py + normalization.py -> validation.py + duplicates.py -> services.py lifecycle -> management command. Finance internal posting helpers are reused inside one outer transaction. |
||||
|
||||
## Interfaces |
||||
|
||||
Profiles BANK_CSV_V1, OUROCARD_FULL_V1, OUROCARD_PARTIAL_V1. |
||||
parse_document(data: bytes, profile: str, context: dict | None) returns immutable ParseResult(encoding,profile,items,skipped) or raises ParseError(code). |
||||
normalize(result) returns tuple of immutable Candidate with to_dict(): locator, section, posting_date ISO or null, occurrence_date ISO or null, description, amount decimal string or null, kind, installment_number, installment_total, errors list of safe codes. |
||||
|
||||
Public services require actor=finance.services.Actor: |
||||
create_preview(data,profile,section_mapping,context=None) -> ImportBatch; |
||||
export_review(batch_id) -> dict; apply_decisions(batch_id,revision,decisions) -> ImportBatch; |
||||
refresh_preview(batch_id,revision) -> ImportBatch; commit_preview(batch_id,revision) -> ImportBatch; |
||||
cancel_preview(batch_id,revision) -> ImportBatch. DomainError.code for failures. |
||||
Decisions list entries: locator + action CREATE/MATCH/SKIP, optional acknowledge_duplicate bool, transaction_id for MATCH, reason for SKIP (USER_EXCLUDED, MALFORMED, UNRECOGNIZED, ALREADY_IMPORTED, SECTION_EXCLUDED), kind/category_id/payee_id/statement_id/refund_of_id/installment_plan_id/new_plan for CREATE. new_plan is a batch-scoped string name; count/account derived from candidate. |
||||
section_mapping dict keys numeric ordinal strings (default section '1'), values account UUID strings or null (skip section explicitly). Multi-section no implicit mapping. |
||||
Export JSON: batch_id, revision, state, profile, encoding, summary, items (locator,candidate,issues,duplicates,decision), decisions list template. Command decision file accepts ONLY batch_id/revision/decisions, or entire exported review with read-only metadata verified against current export. |
||||
|
||||
- [x] Task 1: Pure parser/normalization, synthetic fixtures and edge tests. Owned parser files and tests/test_import_parsers.py. |
||||
- [x] Task 2: Batch/item models, duplicate/validation snapshots and lifecycle services. Persist no finance objects until commit. Revalidate locked snapshots and revision; preserve idempotency and independent failure audits. |
||||
- [x] Task 3: Management command and service/concurrency tests; demonstrate explicit review/edit/apply/commit with synthetic inputs. |
||||
- [x] Task 4: Docker/settings/migrations/runtime grants, full PostgreSQL verification, fresh/existing DB migration, review and developer docs. |
||||
|
||||
Ruling: creation rejects file-level parser errors before staging; malformed individual rows remain reviewable but require an explicit SKIP. Section mappings may be corrected by creating a new preview; candidate bytes/context are immutable. |
||||
Ruling: refresh increments revision and clears all decisions, requiring renewed consent after live state changes. Applying decisions replaces the complete decision set, never incrementally changes implicit defaults. |
||||
Ruling: successful repeated commit returns the recorded batch without a second success audit; latest committed revision must match the request. |
||||
|
||||
## Completion evidence |
||||
|
||||
- All 138 tests passed against PostgreSQL. Coverage includes the three parser |
||||
profiles, malformed and unknown lines, immutable preview evidence, explicit |
||||
decisions, duplicate matching, rollback, stale reviews, and concurrent commits. |
||||
- Parser reviewability regressions failed before unknown/malformed source labels |
||||
were retained in local candidates, then passed after the fix. Generic issue |
||||
codes remain free of source text. |
||||
- The ingestion migration applied to both the existing database and a fresh |
||||
`gizmoduck-import-verify` database. Django system checks, migration drift, and |
||||
pending-migration checks passed. |
||||
- Runtime DML and sequence grants passed for both finance and ingestion tables; |
||||
DDL and role escalation remained denied. |
||||
- The documented command flow ran end to end on the isolated database: preview, |
||||
export, edit, apply, commit, and repeated idempotent commit. Exactly one |
||||
transaction and one provenance record were created from synthetic input. |
||||
- The rehearsal files were removed afterward. The isolated verification |
||||
database container is stopped and its synthetic volume is retained. |
||||
@ -0,0 +1,453 @@
|
||||
# Local MVP Workflows Implementation Plan |
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. |
||||
|
||||
**Goal:** Deliver a local server-rendered workflow for managing and categorizing transactions, viewing monthly reports, exporting user-owned data, and creating and verifying PostgreSQL backups. |
||||
|
||||
**Architecture:** `finance.queries` and `finance.reports` own read behavior, existing audited finance services own every mutation, and `core` owns HTTP forms, views, URLs, and templates. A versioned export service is shared by browser and command entry points; backup commands wrap PostgreSQL 17 client tools and verify restores only against a separate explicitly marked database. |
||||
|
||||
**Tech Stack:** Python 3.13, Django 5.2, PostgreSQL 17, psycopg 3, Django templates/forms, pytest and pytest-django, Docker Compose. |
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-09-07-local-mvp-workflows-design.md` |
||||
|
||||
## Global Constraints |
||||
|
||||
- All monthly reports use `posting_date`, an inclusive calendar month, and exactly one currency. |
||||
- `INCOME` is inflow; `EXPENSE` is spending; `REFUND` reduces spending in its posting month; `TRANSFER` and `OPENING` are excluded from cash flow and spending. |
||||
- Account balances include every entry through month end, including openings and transfer legs, and are never summed across currencies. |
||||
- Browser writes must call audited `finance.services` with `Actor(kind="HUMAN", identifier="local-owner")`; views never save financial models directly. |
||||
- Transactions and source evidence cannot be deleted or have immutable financial fields rewritten. |
||||
- All mutation forms use POST and Django CSRF protection. |
||||
- Export and backup artifacts must remain under ignored `exports/` and `backups/`, never overwrite existing files, and never publish partial files. |
||||
- Terminal output, HTTP errors, tests, and committed fixtures must contain only synthetic data and safe error codes. |
||||
- PostgreSQL backup clients must use major version 17 and receive passwords through environment variables, never command arguments. |
||||
- OFX, bank APIs, investments, forecasting, receipt recognition, mobile clients, hosted multi-user operation, and automatic or AI categorization remain excluded. |
||||
|
||||
--- |
||||
|
||||
### Task 1: Query, indexing, and report services |
||||
|
||||
**Files:** |
||||
- Create: `finance/queries.py` |
||||
- Create: `finance/reports.py` |
||||
- Create: `finance/migrations/0002_transaction_report_indexes.py` |
||||
- Test: `tests/finance/test_queries.py` |
||||
- Test: `tests/finance/test_reports.py` |
||||
|
||||
**Interfaces:** |
||||
- Consumes: `finance.models.Account`, `Category`, `Payee`, and `Transaction`. |
||||
- Produces: `TransactionFilters.from_mapping(mapping)`, `filtered_transactions(filters)`, `available_currencies()`, `monthly_summary(month, currency)`, `spending_by_category(month, currency)`, and `account_balances(month, currency)`. |
||||
- `month` is a `datetime.date` whose day must be `1`; currency is three uppercase ASCII letters. |
||||
- Report return types are frozen dataclasses containing `Decimal` values. |
||||
|
||||
- [ ] **Step 1: Write failing query tests** |
||||
|
||||
```python |
||||
def test_filtered_transactions_combines_filters_and_orders_stably(): |
||||
filters = TransactionFilters.from_mapping({ |
||||
"date_from": "2026-01-01", "date_to": "2026-01-31", |
||||
"currency": "BRL", "uncategorized": "yes", |
||||
}) |
||||
rows = list(filtered_transactions(filters)) |
||||
assert [row.pk for row in rows] == [newer.pk, older.pk] |
||||
|
||||
def test_invalid_filter_is_explicit(): |
||||
with pytest.raises(QueryInputError) as caught: |
||||
TransactionFilters.from_mapping({"date_from": "31/01/2026"}) |
||||
assert caught.value.errors == {"date_from": "invalid_date"} |
||||
``` |
||||
|
||||
- [ ] **Step 2: Run the query tests and confirm imports fail because `finance.queries` does not exist** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/finance/test_queries.py` |
||||
|
||||
- [ ] **Step 3: Implement validated filters and deterministic query ordering** |
||||
|
||||
Implement a frozen `TransactionFilters` dataclass with optional dates and UUIDs, kind, currency, and tri-state uncategorized value. Parse ISO dates and UUIDs once, return field error codes through `QueryInputError.errors`, use `select_related`, and order by `-posting_date`, `-id`. |
||||
|
||||
- [ ] **Step 4: Run query tests and confirm they pass** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/finance/test_queries.py` |
||||
|
||||
- [ ] **Step 5: Write failing report-semantic tests** |
||||
|
||||
```python |
||||
def test_monthly_summary_counts_card_purchase_once_and_refund_in_posting_month(): |
||||
result = monthly_summary(date(2026, 1, 1), "BRL") |
||||
assert result.inflow == Decimal("1000.00") |
||||
assert result.expense == Decimal("300.00") |
||||
assert result.refund == Decimal("50.00") |
||||
assert result.net_spending == Decimal("250.00") |
||||
assert result.net_activity == Decimal("750.00") |
||||
|
||||
def test_spending_has_uncategorized_bucket_and_never_combines_currency(): |
||||
rows = spending_by_category(date(2026, 1, 1), "BRL") |
||||
assert [(row.category_name, row.net_spending) for row in rows] == [ |
||||
("Synthetic Food", Decimal("80.00")), |
||||
("Uncategorized", Decimal("20.00")), |
||||
] |
||||
``` |
||||
|
||||
- [ ] **Step 6: Run report tests and confirm failure because report functions are absent** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/finance/test_reports.py` |
||||
|
||||
- [ ] **Step 7: Implement monthly summaries, category spending, balances, and currency choices** |
||||
|
||||
Use conditional PostgreSQL aggregates with explicit `Decimal("0.00")` defaults. Validate month and currency. Group nullable categories into an explicit Uncategorized result without manufacturing a Category row. Sort category results by descending net spending then category name; sort balances by account name then UUID. |
||||
|
||||
- [ ] **Step 8: Add report indexes through a migration** |
||||
|
||||
Add named indexes on `Transaction(posting_date)` and `Transaction(account, posting_date)`. Do not alter financial constraints. |
||||
|
||||
- [ ] **Step 9: Run focused tests and migration drift check** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/finance/test_queries.py tests/finance/test_reports.py` |
||||
|
||||
Run: `docker compose run --rm manage makemigrations --check --dry-run` |
||||
|
||||
- [ ] **Step 10: Commit the query and reporting slice** |
||||
|
||||
```powershell |
||||
git add finance/queries.py finance/reports.py finance/migrations/0002_transaction_report_indexes.py tests/finance/test_queries.py tests/finance/test_reports.py |
||||
git diff --cached --name-only |
||||
git diff --cached |
||||
git commit -m "feat: add financial queries and monthly reports" |
||||
``` |
||||
|
||||
### Task 2: Local web shell and audited management workflows |
||||
|
||||
**Files:** |
||||
- Create: `core/forms.py` |
||||
- Create: `core/context.py` |
||||
- Create: `core/templates/core/base.html` |
||||
- Create: `core/templates/core/transactions/list.html` |
||||
- Create: `core/templates/core/transactions/detail.html` |
||||
- Create: `core/templates/core/transactions/form.html` |
||||
- Create: `core/templates/core/references/list.html` |
||||
- Create: `core/templates/core/references/form.html` |
||||
- Create: `core/static/core/app.css` |
||||
- Modify: `core/views.py` |
||||
- Modify: `config/urls.py` |
||||
- Modify: `config/settings.py` |
||||
- Test: `tests/test_web_workflows.py` |
||||
|
||||
**Interfaces:** |
||||
- Consumes: Task 1 query APIs and existing `finance.services` mutation APIs. |
||||
- Produces named URLs `home`, `transaction-list`, `transaction-detail`, `transaction-create`, `transaction-annotate`, `account-list`, `account-create`, `account-archive`, `category-list`, `category-create`, `category-archive`, `payee-list`, `payee-create`, and `payee-archive`. |
||||
- Produces forms that return validated scalar values and UUIDs but never save models. |
||||
|
||||
- [ ] **Step 1: Write failing request tests for navigation, list filters, detail, and CSRF** |
||||
|
||||
```python |
||||
def test_root_and_transaction_pages_are_discoverable(client): |
||||
assert client.get(reverse("home")).status_code == 200 |
||||
page = client.get(reverse("transaction-list"), {"currency": "BRL"}) |
||||
assert page.status_code == 200 |
||||
assert "Synthetic purchase" in page.content.decode() |
||||
|
||||
def test_mutation_requires_csrf(): |
||||
guarded = Client(enforce_csrf_checks=True) |
||||
response = guarded.post(reverse("category-create"), {"name": "Synthetic Food", "kind": "EXPENSE"}) |
||||
assert response.status_code == 403 |
||||
``` |
||||
|
||||
- [ ] **Step 2: Run web workflow tests and confirm missing routes fail** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/test_web_workflows.py` |
||||
|
||||
- [ ] **Step 3: Enable template, static, CSRF, session, and message infrastructure** |
||||
|
||||
Add Django sessions and messages apps/middleware, template backend with request and message context processors, `STATIC_URL`, and signed-cookie sessions. Run the sessions migration; no financial data is stored in the session. |
||||
|
||||
- [ ] **Step 4: Implement forms and request-to-domain error mapping** |
||||
|
||||
Build plain `forms.Form` classes for manual transactions, annotations, accounts, categories, and payees. Query active references for choices. Convert `DomainError.code` into a fixed safe message dictionary; unknown codes map to `Unable to save this change.` |
||||
|
||||
- [ ] **Step 5: Implement transaction list/detail/create/annotate views** |
||||
|
||||
Use `finance.queries` for reads, Django `Paginator` with 50 rows per page, and existing finance services for writes. Derive `LOCAL_ACTOR = Actor(kind="HUMAN", identifier="local-owner")` server-side. Redirect after successful POST. |
||||
|
||||
- [ ] **Step 6: Implement reference list/create/archive views** |
||||
|
||||
Use services only for mutations. Archive endpoints accept POST only and render archived values as historical/inactive on list pages. |
||||
|
||||
- [ ] **Step 7: Add accessible templates and minimal responsive CSS** |
||||
|
||||
Provide one `h1`, associated labels, error summaries, visible focus styles, tabular headings, empty states, and navigation on every page. Keep all core workflows functional without JavaScript. |
||||
|
||||
- [ ] **Step 8: Run request and existing domain tests** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/test_web_workflows.py tests/finance/test_services.py` |
||||
|
||||
- [ ] **Step 9: Commit the audited web workflow slice after staged privacy inspection** |
||||
|
||||
```powershell |
||||
git add config/settings.py config/urls.py core tests/test_web_workflows.py |
||||
git diff --cached --name-only |
||||
git diff --cached |
||||
git commit -m "feat: add local transaction workflows" |
||||
``` |
||||
|
||||
### Task 3: Dashboard and monthly report screens |
||||
|
||||
**Files:** |
||||
- Create: `core/templates/core/dashboard.html` |
||||
- Create: `core/templates/core/reports/monthly.html` |
||||
- Modify: `core/forms.py` |
||||
- Modify: `core/views.py` |
||||
- Modify: `config/urls.py` |
||||
- Test: `tests/test_report_views.py` |
||||
|
||||
**Interfaces:** |
||||
- Consumes: `monthly_summary`, `spending_by_category`, `account_balances`, and `available_currencies` from Task 1. |
||||
- Produces named URL `monthly-report`; `home` renders the current selected month and currency rather than transaction-management content. |
||||
|
||||
- [ ] **Step 1: Write failing report-view tests** |
||||
|
||||
```python |
||||
def test_monthly_report_renders_exact_totals_and_category_rows(client): |
||||
response = client.get(reverse("monthly-report"), {"month": "2026-01", "currency": "BRL"}) |
||||
assert response.status_code == 200 |
||||
body = response.content.decode() |
||||
assert "750.00" in body |
||||
assert "Synthetic Food" in body |
||||
|
||||
def test_invalid_month_is_visible_and_not_silently_replaced(client): |
||||
response = client.get(reverse("monthly-report"), {"month": "01/2026", "currency": "BRL"}) |
||||
assert response.status_code == 400 |
||||
assert "invalid_month" in response.content.decode() |
||||
``` |
||||
|
||||
- [ ] **Step 2: Run the focused tests and confirm the route/template behavior is absent** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/test_report_views.py` |
||||
|
||||
- [ ] **Step 3: Implement month/currency selection and report views** |
||||
|
||||
Parse `YYYY-MM` into the first day of the month. Default to the current local calendar month and the first available currency, falling back to `BRL` for an empty database. Invalid supplied values return status 400 with field errors. |
||||
|
||||
- [ ] **Step 4: Render dashboard and detailed report templates** |
||||
|
||||
Show inflow, expense, refund, net spending, net activity, categorized/uncategorized rows, and per-account balances. Label negative net spending and balances without changing their sign. |
||||
|
||||
- [ ] **Step 5: Run report service and view tests** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/finance/test_reports.py tests/test_report_views.py` |
||||
|
||||
- [ ] **Step 6: Commit the report UI slice after staged privacy inspection** |
||||
|
||||
```powershell |
||||
git add core/forms.py core/views.py core/templates/core/dashboard.html core/templates/core/reports/monthly.html config/urls.py tests/test_report_views.py |
||||
git diff --cached --name-only |
||||
git diff --cached |
||||
git commit -m "feat: add monthly finance dashboard" |
||||
``` |
||||
|
||||
### Task 4: Versioned portable exports |
||||
|
||||
**Files:** |
||||
- Create: `finance/exports.py` |
||||
- Create: `finance/management/__init__.py` |
||||
- Create: `finance/management/commands/__init__.py` |
||||
- Create: `finance/management/commands/export_finances.py` |
||||
- Create: `core/templates/core/exports.html` |
||||
- Modify: `core/views.py` |
||||
- Modify: `config/urls.py` |
||||
- Modify: `config/settings.py` |
||||
- Modify: `compose.yaml` |
||||
- Test: `tests/finance/test_exports.py` |
||||
- Test: `tests/test_export_command.py` |
||||
- Test: `tests/test_export_view.py` |
||||
|
||||
**Interfaces:** |
||||
- Consumes: finance models through read-only ordered querysets. |
||||
- Produces `build_export() -> ExportArtifact` for an in-memory snapshot and `write_export(name: str | None = None) -> ExportResult` under `settings.EXPORT_ROOT`. |
||||
- Produces `export_finances [--output NAME]` and named URLs `export-index`, `export-download`. |
||||
|
||||
- [ ] **Step 1: Write failing package-contract tests** |
||||
|
||||
```python |
||||
def test_export_has_versioned_manifest_and_stable_csv_contract(): |
||||
artifact = build_export() |
||||
with ZipFile(BytesIO(artifact.content)) as package: |
||||
manifest = json.loads(package.read("manifest.json")) |
||||
assert manifest["format"] == "gizmoduck-export" |
||||
assert manifest["version"] == 1 |
||||
assert package.read("transactions.csv").decode("utf-8").splitlines()[0] == EXPECTED_TRANSACTION_HEADER |
||||
assert manifest["files"]["transactions.csv"]["sha256"] == sha256(package.read("transactions.csv")).hexdigest() |
||||
``` |
||||
|
||||
- [ ] **Step 2: Run export tests and confirm the service is missing** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/finance/test_exports.py` |
||||
|
||||
- [ ] **Step 3: Implement the consistent export snapshot** |
||||
|
||||
Use `transaction.atomic()` and `SET TRANSACTION ISOLATION LEVEL REPEATABLE READ` before reads. Serialize all specified tables in UUID order, exact decimals, ISO dates/timestamps, lowercase booleans, and blank nullable fields. Write deterministic UTF-8 CSV content and checksums into a ZIP with `manifest.json` last. |
||||
|
||||
- [ ] **Step 4: Write failing safe-file and command tests** |
||||
|
||||
Cover generated names, traversal, absolute paths, symlinks, existing targets, safe stdout, temporary cleanup, and database failure before publication. |
||||
|
||||
- [ ] **Step 5: Implement exclusive atomic publication and command entry point** |
||||
|
||||
Resolve the real export root, accept plain `.zip` names only, create the destination exclusively, write to a same-directory private temporary file, flush and `os.fsync`, then `os.replace`. Reject an existing destination before building the financial snapshot and remove temporary files on every failure. |
||||
|
||||
- [ ] **Step 6: Write failing browser download tests and implement export pages** |
||||
|
||||
The GET index explains export contents. CSRF-protected POST builds the same in-memory artifact and returns it with `Content-Disposition: attachment`; the request cannot supply a server path. |
||||
|
||||
- [ ] **Step 7: Mount ignored export storage and add settings** |
||||
|
||||
Set `EXPORT_ROOT` from environment with `/app/exports` default. Mount `./exports:/app/exports` into `manage`; the browser download remains in memory and does not require a writable host mount. |
||||
|
||||
- [ ] **Step 8: Run all export tests** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/finance/test_exports.py tests/test_export_command.py tests/test_export_view.py` |
||||
|
||||
- [ ] **Step 9: Commit the export slice after checking that no generated ZIP is staged** |
||||
|
||||
```powershell |
||||
git add finance/exports.py finance/management core config/settings.py config/urls.py compose.yaml tests/finance/test_exports.py tests/test_export_command.py tests/test_export_view.py |
||||
git diff --cached --name-only |
||||
git diff --cached |
||||
git commit -m "feat: add versioned financial exports" |
||||
``` |
||||
|
||||
### Task 5: PostgreSQL backup creation and isolated restore verification |
||||
|
||||
**Files:** |
||||
- Create: `core/backup.py` |
||||
- Create: `core/management/__init__.py` |
||||
- Create: `core/management/commands/__init__.py` |
||||
- Create: `core/management/commands/backup_database.py` |
||||
- Create: `core/management/commands/verify_backup.py` |
||||
- Modify: `Dockerfile` |
||||
- Modify: `config/settings.py` |
||||
- Modify: `compose.yaml` |
||||
- Modify: `.env.example` |
||||
- Test: `tests/test_backup.py` |
||||
- Test: `tests/test_backup_commands.py` |
||||
- Test: `tests/test_backup_restore.py` |
||||
|
||||
**Interfaces:** |
||||
- Produces `create_backup(name: str | None = None) -> BackupResult` and `verify_backup(name: str, target: DatabaseTarget) -> VerificationResult`. |
||||
- Produces commands `backup_database [--output NAME]` and `verify_backup --input NAME --target-name NAME --target-host HOST --target-user USER`. |
||||
- Verification requires `BACKUP_VERIFY_ALLOWED=1` and rejects the live `(DB_HOST, DB_NAME)` pair. |
||||
|
||||
- [ ] **Step 1: Write failing unit tests for safe subprocess construction and manifests** |
||||
|
||||
```python |
||||
def test_pg_dump_receives_password_only_in_environment(runner, settings): |
||||
create_backup("synthetic.dump") |
||||
argv, env = runner.calls[0] |
||||
assert argv[:3] == ["pg_dump", "--format=custom", "--no-owner"] |
||||
assert settings.DATABASES["default"]["PASSWORD"] not in argv |
||||
assert env["PGPASSWORD"] == settings.DATABASES["default"]["PASSWORD"] |
||||
|
||||
def test_verification_rejects_live_database(settings): |
||||
with pytest.raises(BackupError, match="live_target"): |
||||
verify_backup("synthetic.dump", DatabaseTarget.from_default(settings.DATABASES["default"])) |
||||
``` |
||||
|
||||
- [ ] **Step 2: Run backup tests and confirm the module is absent** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/test_backup.py` |
||||
|
||||
- [ ] **Step 3: Install PostgreSQL 17 clients and implement safe backup creation** |
||||
|
||||
Add the PostgreSQL apt repository and pin/install `postgresql-client-17` in the application image. Build subprocess argument arrays without a shell. Use the default database settings and `PGPASSWORD`, custom format, `--no-owner`, `--no-acl`, private temporary files, digest manifest, `fsync`, exclusive destination reservation, and atomic rename. |
||||
|
||||
- [ ] **Step 4: Implement manifest validation and isolated restore verification** |
||||
|
||||
Require exact manifest format/version, matching size/digest, explicit allow flag, a target distinct from live host/name, and a target containing no non-system tables. Invoke `pg_restore --exit-on-error --single-transaction --no-owner --no-acl` without a shell. Validate `django_migrations` equality and manifest row counts through a separate psycopg connection to the target. |
||||
|
||||
- [ ] **Step 5: Write failing command tests and implement safe command adapters** |
||||
|
||||
Commands print artifact name, format version, and table counts only. Convert `BackupError.code` into `CommandError(code)` without paths, SQL, subprocess stderr, or credentials. |
||||
|
||||
- [ ] **Step 6: Add backup and verification Compose configuration** |
||||
|
||||
Set `BACKUP_ROOT=/app/backups`, mount `./backups:/app/backups` into `manage`, and add a profile-gated `restore-db` PostgreSQL 17 service on tmpfs. Add a `verify-backup` application service whose default database settings point to `restore-db`, with `BACKUP_VERIFY_ALLOWED=1`; keep live credentials out of that service. |
||||
|
||||
- [ ] **Step 7: Write and run the synthetic backup/restore integration test** |
||||
|
||||
Populate only synthetic finance, provenance, audit, and import-review rows; create a backup; restore it into the empty `restore-db`; compare migrations and per-table counts; then query the original database to prove it remains unchanged. |
||||
|
||||
Run: `docker compose build` |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/test_backup.py tests/test_backup_commands.py` |
||||
|
||||
Run: `docker compose --profile backup run --rm verify-backup python manage.py test_backup_restore` |
||||
|
||||
- [ ] **Step 8: Commit the backup slice after verifying no dumps, manifests, secrets, or generated reports are staged** |
||||
|
||||
```powershell |
||||
git add Dockerfile config/settings.py compose.yaml .env.example core/backup.py core/management tests/test_backup.py tests/test_backup_commands.py tests/test_backup_restore.py |
||||
git diff --cached --name-only |
||||
git diff --cached |
||||
git commit -m "feat: add verified PostgreSQL backups" |
||||
``` |
||||
|
||||
### Task 6: Owner documentation and complete synthetic acceptance workflow |
||||
|
||||
**Files:** |
||||
- Create: `docs/local-workflows.md` |
||||
- Create: `tests/test_mvp_acceptance.py` |
||||
- Modify: `README.md` |
||||
- Modify: `docs/development.md` |
||||
- Modify: `docs/roadmap.md` |
||||
- Modify: `.gitignore` |
||||
|
||||
**Interfaces:** |
||||
- Consumes: every public workflow created by Tasks 1–5 and the existing `import_review` command. |
||||
- Produces a copy-paste local workflow from synthetic import through categorization, reports, export, backup, and disposable restore verification. |
||||
|
||||
- [ ] **Step 1: Write the failing acceptance test** |
||||
|
||||
The test must create an account, preview/apply/commit `tests/fixtures/synthetic/ingestion/bank.csv`, POST a category assignment through the CSRF-enabled browser flow, assert the monthly report totals, inspect a generated export manifest, create a logical backup through the injectable runner boundary, and verify that the restore-verification boundary checks migrations and table counts. |
||||
|
||||
- [ ] **Step 2: Run the acceptance test and confirm any missing integration behavior** |
||||
|
||||
Run: `docker compose run --rm test pytest -q tests/test_mvp_acceptance.py` |
||||
|
||||
- [ ] **Step 3: Fix only integration gaps exposed by the failing acceptance test** |
||||
|
||||
Keep changes within existing Task 1–5 boundaries. Add no new product behavior. Re-run the focused tests for every amended module. |
||||
|
||||
- [ ] **Step 4: Document the owner workflow and recovery procedure** |
||||
|
||||
Document local URLs, supported edits, report semantics, command output safety, export contents, backup creation, disposable restore verification, and deliberate live recovery. The live recovery sequence must stop `web`, create a separate safety backup, name the target database explicitly, restore with PostgreSQL 17 tools, run migrations/checks, and restart only after validation. |
||||
|
||||
- [ ] **Step 5: Reconcile configuration and project status documentation** |
||||
|
||||
Document the loopback PostgreSQL port currently present in Compose, link the workflow guide from README, and mark roadmap step 5 complete only after full verification. Confirm `.gitignore` covers export and backup contents plus temporary suffixes while retaining synthetic fixtures. |
||||
|
||||
- [ ] **Step 6: Run the complete verification matrix** |
||||
|
||||
```powershell |
||||
docker compose config --quiet |
||||
docker compose build |
||||
docker compose run --rm manage migrate |
||||
docker compose run --rm manage check |
||||
docker compose run --rm manage makemigrations --check --dry-run |
||||
docker compose run --rm manage migrate --check |
||||
docker compose run --rm test |
||||
docker compose run --rm permissions |
||||
docker compose --profile backup run --rm verify-backup python manage.py test_backup_restore |
||||
Invoke-RestMethod http://127.0.0.1:8000/health/ |
||||
``` |
||||
|
||||
- [ ] **Step 7: Inspect all staged content for financial data and secrets, then commit** |
||||
|
||||
```powershell |
||||
git add README.md docs/development.md docs/local-workflows.md docs/roadmap.md .gitignore tests/test_mvp_acceptance.py |
||||
git diff --cached --name-only |
||||
git diff --cached |
||||
git commit -m "docs: complete local MVP workflow" |
||||
``` |
||||
|
||||
- [ ] **Step 8: Run a final whole-branch review against this plan and resolve all critical or important findings** |
||||
|
||||
Use the review package required by `superpowers:subagent-driven-development`, include all deferred-minor ledger entries, and re-run the complete verification matrix after any fix wave. |
||||
@ -0,0 +1,105 @@
|
||||
# Gizmoduck Agent Foundation Design |
||||
|
||||
**Date:** 2026-09-06 |
||||
**Status:** Approved for planning |
||||
|
||||
## Purpose |
||||
|
||||
Establish a small, durable set of documents that gives human and automated |
||||
contributors a shared understanding of Gizmoduck before application code is |
||||
written. The documents must make binding technical decisions, privacy |
||||
boundaries, project scope, and agent behavior easy to discover. |
||||
|
||||
## Document structure |
||||
|
||||
The agent foundation will use a layered structure: |
||||
|
||||
- `AGENTS.md` is the concise, authoritative entry point for every agent. |
||||
- `docs/product.md` defines the product purpose, users, MVP, and exclusions. |
||||
- `docs/architecture.md` records binding technology choices and component |
||||
boundaries. |
||||
- `docs/data-policy.md` governs access to and handling of real financial data. |
||||
- `docs/roadmap.md` orders the initial delivery milestones without becoming a |
||||
task tracker. |
||||
|
||||
The README remains the human-facing introduction and links to these documents |
||||
once they exist. Detailed feature designs will be written separately rather |
||||
than accumulated in `AGENTS.md`. |
||||
|
||||
## Binding project constraints |
||||
|
||||
Until the owner explicitly approves a change, agents must treat the following |
||||
as binding: |
||||
|
||||
- The project is named Gizmoduck and is the clean successor to Robopato. |
||||
- The backend uses Python and Django. |
||||
- Persistent storage uses PostgreSQL with migrations from the beginning. |
||||
- Local infrastructure uses Docker Compose. |
||||
- Automated tests use pytest. |
||||
- Bank-account transactions are imported from CSV. |
||||
- Credit-card transactions are imported from Banco do Brasil/Ourocard TXT. |
||||
- OFX is not supported. |
||||
- Parsing, normalization, validation, duplicate detection, preview, and |
||||
persistence remain separate stages. |
||||
|
||||
Agents may identify risks or propose reconsideration, but they may not silently |
||||
replace these choices while implementing unrelated work. |
||||
|
||||
## Robopato access and financial-data policy |
||||
|
||||
Gizmoduck agents are authorized to read every file in the Robopato repository. |
||||
They may copy real financial documents from Robopato for local development, |
||||
parser validation, migration, and compatibility testing. |
||||
|
||||
Real financial data must never be added to Git. This prohibition includes raw |
||||
documents, database dumps, copied excerpts, test snapshots, logs, screenshots, |
||||
generated reports, and documentation containing identifying transaction data. |
||||
Real data may exist only in ignored local directories and local databases. |
||||
|
||||
Committed tests must use synthetic data or data that has been irreversibly |
||||
sanitized. Before every commit, agents must inspect staged paths and staged |
||||
content for financial data and secrets. When uncertain whether data is safe, |
||||
agents must stop and ask the owner. |
||||
|
||||
## Agent operating policy |
||||
|
||||
Agents will: |
||||
|
||||
- read `AGENTS.md` and its linked documents before substantive work; |
||||
- preserve user changes and avoid unrelated edits; |
||||
- clarify changes that would alter binding constraints or product scope; |
||||
- prefer small, cohesive changes with tests appropriate to their risk; |
||||
- use isolated Git worktrees for concurrent implementation that could overlap; |
||||
- report assumptions, verification performed, and unresolved risks; and |
||||
- avoid destructive operations, pushes, merges, releases, and history rewrites |
||||
without explicit authorization. |
||||
|
||||
Agents may create focused local commits after relevant verification succeeds. |
||||
Agent-authored commits use a repository-local identity: |
||||
|
||||
```text |
||||
Codex <codex@localhost> |
||||
``` |
||||
|
||||
Human-authored commits retain the human contributor's own Git identity. |
||||
|
||||
## Verification |
||||
|
||||
The documentation task is complete when: |
||||
|
||||
- every planned document exists and links resolve; |
||||
- the README points readers to the agent and project documentation; |
||||
- constraints are consistent across all documents; |
||||
- no placeholders or unresolved policy choices remain; |
||||
- `.gitignore` protects the approved local-data locations and common secret and |
||||
dump formats; |
||||
- sample checks confirm real-data paths are ignored while synthetic fixtures |
||||
remain trackable; and |
||||
- Git reports `Codex <codex@localhost>` as the repository-local identity. |
||||
|
||||
## Deferred work |
||||
|
||||
This foundation does not define the detailed financial domain model, database |
||||
schema, UI, importer grammar, duplicate-detection algorithm, deployment model, |
||||
or backup implementation. Each substantial subsystem will receive its own |
||||
design and implementation plan before development. |
||||
@ -0,0 +1,103 @@
|
||||
# Runnable Application Skeleton Design |
||||
|
||||
**Date:** 2026-09-06 |
||||
**Status:** Approved |
||||
|
||||
## Goal and scope |
||||
|
||||
Deliver roadmap step 2: a clean checkout can start Django with PostgreSQL, |
||||
apply migrations, and run database-backed pytest checks using documented |
||||
commands. No financial domain models, importers, or product UI are included. |
||||
|
||||
## Approach |
||||
|
||||
Use Docker Compose for both application and database. This makes the primary |
||||
workflow consistent on Windows and avoids requiring host Python dependencies. |
||||
A host-run Django environment is a viable alternative but adds setup variation; |
||||
a dual supported workflow adds documentation and verification cost without |
||||
helping this milestone. Recommend the Compose-only primary workflow. |
||||
|
||||
Use Python 3.13, Django 5.2 LTS, PostgreSQL 17, psycopg 3, pytest, |
||||
pytest-django, and uv with a committed lockfile. Resolve supported patch versions |
||||
at implementation time. Freeze dependency installation from the lockfile. |
||||
Django documents Python 3.13 compatibility for its 5.2 series: |
||||
https://docs.djangoproject.com/en/5.2/releases/5.2/ |
||||
PostgreSQL 17 is a supported major version: |
||||
https://www.postgresql.org/support/versioning/ |
||||
|
||||
## Components |
||||
|
||||
- `config/`: Django settings, URLs, WSGI and ASGI entry points. |
||||
- `core/`: a database readiness endpoint; no financial models. |
||||
- `manage.py`: normal Django management entry point. |
||||
- `pyproject.toml` and `uv.lock`: dependency and pytest configuration. |
||||
- `Dockerfile`, `.dockerignore`, and `compose.yaml`: reproducible local services. |
||||
- `docker/postgres/`: provisioning scripts, using shell files to avoid the |
||||
repository's raw SQL exclusion rules. |
||||
- `tests/`: endpoint, database, and permission checks using synthetic data. |
||||
- `.env.example` and README setup instructions: configuration contract. |
||||
|
||||
Expose the development web server only on host loopback port 8000. PostgreSQL |
||||
is reachable through the Compose network without a published host port. Persist |
||||
its data in a project-scoped Docker volume. Never delete volumes automatically. |
||||
The container build context excludes real-data directories, secrets, Git, |
||||
virtual environments, and generated artifacts. |
||||
|
||||
## Configuration and database roles |
||||
|
||||
Require explicit local secrets in ignored `.env`; commit only non-secret example |
||||
values and instructions for generating replacements. Read database connection |
||||
settings and the Django secret from the environment. Missing required settings |
||||
fail clearly without printing credentials. Restrict allowed hosts to localhost |
||||
and 127.0.0.1 for the default local setup. |
||||
|
||||
Provision three roles: a PostgreSQL bootstrap administrator used only for |
||||
initialization, a development/migration role owning the application schema, |
||||
and an application runtime role with connect, schema usage, table DML, and |
||||
sequence privileges. Runtime has no superuser, role creation, database creation, |
||||
or schema creation permissions. Default privileges cover new objects created |
||||
by the migration role. The application service receives only runtime credentials. |
||||
A separate management service uses migration credentials for explicit migrate |
||||
commands. Application startup never silently runs migrations. |
||||
|
||||
Run tests against a separate PostgreSQL service under a Compose test profile, |
||||
with disposable test data and a dedicated test role allowed to create the test |
||||
database. Tests cannot target the development database through default settings. |
||||
Verify the production-equivalent runtime grants separately against the local |
||||
application schema using synthetic records and rollback/cleanup. |
||||
|
||||
## Behavior and failure handling |
||||
|
||||
`GET /health/` executes a small database query and returns a minimal JSON status. |
||||
Return HTTP 200 on connectivity success or 503 for database connectivity failure; |
||||
never return exception messages or connection details. This is a readiness check, |
||||
not proof that every migration is applied. Use migration checks independently. |
||||
|
||||
Compose waits for PostgreSQL readiness. Document the sequence: create local |
||||
configuration, build, start the database, run migrations, start web, run tests. |
||||
Initialization scripts apply only to new database volumes. Existing-volume role |
||||
or credential changes require explicit management; explain this in the README |
||||
without recommending destructive resets as the default fix. |
||||
|
||||
## Verification and acceptance |
||||
|
||||
1. Validate Compose configuration without echoing resolved secrets. |
||||
2. Build the application image using locked dependencies. |
||||
3. Initialize PostgreSQL and apply Django's built-in migrations using the |
||||
migration role; check that no migrations are pending or missing. |
||||
4. Run Django system checks and pytest against the isolated test database. |
||||
5. Verify health success and database-failure behavior without exposing errors. |
||||
6. Confirm the runtime role can use migrated tables but cannot create tables or |
||||
databases, and cannot assume the migration role. |
||||
7. Start the web service and request its health endpoint through localhost. |
||||
8. Inspect Git changes for secrets and financial data; check ignore rules for |
||||
configuration and verify that required source files remain trackable. |
||||
|
||||
Record actual results, including any Docker availability blocker. No SQLite |
||||
fallback is acceptable evidence for PostgreSQL acceptance criteria. |
||||
|
||||
## Deferred decisions |
||||
|
||||
Financial models, source parser contracts, authentication UX, product pages, |
||||
background tasks, hosted deployment, and backup implementation belong to later |
||||
milestones and do not block this skeleton. |
||||
@ -0,0 +1,199 @@
|
||||
# Financial Domain Design |
||||
|
||||
**Date:** 2026-09-06 |
||||
**Status:** Approved |
||||
**Milestone:** Roadmap step 3 |
||||
|
||||
## Goal |
||||
|
||||
Provide Django models, PostgreSQL migrations, and tested domain operations for |
||||
accounts, credit cards, transactions, payees, categories, transfers, refunds, |
||||
installments, and provenance. Import parsers, import preview, product pages, |
||||
reporting screens, and real-data migration remain later milestones. |
||||
|
||||
## Approach |
||||
|
||||
Use one transaction table for bank accounts and credit cards. Separate tables |
||||
would duplicate categorization and provenance rules. A full double-entry ledger |
||||
would add balancing accounts and posting machinery beyond the current MVP. |
||||
Explicit relationships around a common transaction model provide the required |
||||
semantics without that extra accounting subsystem. |
||||
|
||||
Add a `finance` Django application. Keep models and database constraints in |
||||
`finance/models/`, public write operations in `finance/services/`, and tests in |
||||
`tests/finance/`. Existing health behavior remains unchanged. |
||||
|
||||
## Money and dates |
||||
|
||||
Store signed Decimal amounts with two fractional digits and 18 total digits. |
||||
Reject floats, non-finite numbers, excessive precision, and zero-value activity. |
||||
Positive amounts increase the owner's position; negative amounts decrease it. |
||||
A bank deposit is positive, a purchase is negative, and a card debt balance is |
||||
negative. A card payment is a negative bank leg and a positive card leg. |
||||
|
||||
An account has an immutable three-letter currency code, default BRL. Codes must |
||||
be uppercase ASCII letters. No conversion or exchange-rate handling is included. |
||||
Paired transfers must use the same currency. Balances in different currencies |
||||
must never be added together. Monetary fields use the account currency. |
||||
|
||||
Use a required posting date and optional occurrence date. Do not collapse these |
||||
into timestamps: an invoice can post a purchase later than its purchase date. |
||||
Creation and audit timestamps are timezone-aware UTC. |
||||
|
||||
## Entities |
||||
|
||||
### Account |
||||
|
||||
UUID identifier, nonblank display name, kind (`BANK` or `CREDIT_CARD`), currency, |
||||
archived flag, creation and modification timestamps. Display names are unique |
||||
within the local dataset. Archive accounts instead of deleting their history. |
||||
Do not store a mutable cached balance: derive it from opening entries and posted |
||||
transactions. New transactions cannot target archived accounts. |
||||
|
||||
A credit card is an account with kind `CREDIT_CARD`; no card number, credential, |
||||
credit limit, or inferred billing schedule is required for this milestone. |
||||
|
||||
### Payee and Category |
||||
|
||||
UUID identifiers, nonblank unique names, archived flag, timestamps. Categories |
||||
have kind `INCOME` or `EXPENSE` and are flat for the MVP. Transactions may remain |
||||
uncategorized or lack a payee. Archived values remain readable in history and |
||||
cannot be assigned to new transactions. Category kind must match the activity; |
||||
refunds can use an expense category to reverse spending. |
||||
|
||||
### CardStatement |
||||
|
||||
UUID, credit-card account, period start/end, due date, optional signed statement |
||||
balance, and timestamps. Period end is not earlier than period start; due date |
||||
is not earlier than period end. One statement per card and period end. |
||||
|
||||
A transaction can reference a statement only for its own credit-card account. |
||||
Statements group posted rows; they do not create another expense. A quoted |
||||
statement balance is source metadata, not the sum of imported rows: a partial |
||||
import must not manufacture a reconciliation result. Bank transactions cannot |
||||
reference statements. Statement creation does not require a parser. |
||||
|
||||
### Transaction |
||||
|
||||
UUID, account, signed amount, posting date, optional occurrence date, description, |
||||
kind (`INCOME`, `EXPENSE`, `TRANSFER`, `REFUND`, `OPENING`), optional payee/category, |
||||
optional statement, optional original transaction for a refund, optional |
||||
installment plan and number, and timestamps. |
||||
|
||||
INCOME and REFUND are positive; EXPENSE is negative; TRANSFER may have either |
||||
sign. OPENING may have either sign and is excluded from income/spending. Permit |
||||
at most one opening entry per account. An opening entry has no category, payee, |
||||
statement, refund link, or installment relationship. |
||||
|
||||
Transfers have no income/expense category, refund link, or installment link. |
||||
Refund references are allowed only on REFUND rows. A linked refund must target |
||||
an EXPENSE in the same currency, cannot reference itself, and can use a different |
||||
account. Refunds may be unlinked when the original purchase is absent. Multiple |
||||
partial refunds are supported; no automatic cumulative cap is imposed because |
||||
real adjustments can exceed an original charge. Refunds are recorded separately |
||||
and never overwrite the purchase. No automatic payee/category inheritance. |
||||
|
||||
### Transfer |
||||
|
||||
UUID and two distinct, uniquely linked Transaction rows: outgoing and incoming. |
||||
The outgoing amount is negative; incoming is positive and equal in magnitude. |
||||
Accounts must differ and currencies must match. Both rows have TRANSFER kind. |
||||
Dates can differ to represent settlement delays. Each transaction belongs to at |
||||
most one transfer. A transfer may connect a bank account and a credit card. |
||||
|
||||
An unmatched TRANSFER transaction is allowed because imports can arrive one |
||||
account at a time. Creating a complete transfer writes both legs atomically; |
||||
matching two existing legs links them without creating more money movement. |
||||
Unmatched legs remain identifiable by the absence of a Transfer relationship. |
||||
|
||||
### InstallmentPlan |
||||
|
||||
UUID, account, description, positive total installment count, timestamps. |
||||
Transactions reference the plan and their one-based installment number. Require |
||||
EXPENSE kind, the same account, a number between 1 and total count, and unique |
||||
(plan, installment number). Plan membership and number are either both present |
||||
or both absent. Refunds refer to the individual installment expense. |
||||
|
||||
Only observed installments are posted. A 3/10 source row does not create the |
||||
other nine entries, and a plan's total purchase value is not inferred from a |
||||
single installment. Amounts may differ due to rounding. Freeze a plan's account |
||||
and total count once it has posted members. |
||||
|
||||
### SourceRecord and AuditEvent |
||||
|
||||
SourceRecord identifies a parsed source item independently from transaction |
||||
classification: UUID, document SHA-256, source format (`BANK_CSV` or |
||||
`OUROCARD_TXT`), parser version, nonblank item locator, and creation timestamp. |
||||
A unique (document hash, item locator) prevents attaching the same physical item |
||||
twice. Each source item links to at most one transaction. A transaction may have |
||||
multiple source records when later duplicate matching identifies another source |
||||
for the same activity. This is provenance identity, not the future semantic |
||||
duplicate-detection algorithm. Raw source text and file contents are not stored |
||||
in this model. Hashes and provenance still remain local financial data. |
||||
|
||||
AuditEvent stores actor kind (`HUMAN`, `APPLICATION`, `AGENT`), nonblank actor |
||||
identifier, operation, target UUID, timestamp, outcome, and changed field names. |
||||
Do not copy raw source rows, descriptions, credentials, or monetary payloads |
||||
into generic audit messages. Successful writes and their audit event commit in |
||||
one transaction. Domain validation failures raise a structured error code; |
||||
write an independent failure audit after rollback, without sensitive input. |
||||
Database outages may prevent audit persistence and must not be masked. |
||||
|
||||
## Mutation boundaries and validation |
||||
|
||||
Public service operations create accounts/reference data, archive reference |
||||
data, create statements/plans, record transactions, create/match transfers, |
||||
attach provenance, and update a transaction's description/payee/category. |
||||
Financial amounts, accounts, dates, kinds, refund links, and installment identity |
||||
are immutable after recording in this milestone. Financial correction/reversal |
||||
workflows receive a later design; no unrestricted generic update API is exposed. |
||||
|
||||
Every service takes an explicit actor. Use transaction.atomic and row locking |
||||
for cross-row invariants, including transfer matching and installment membership. |
||||
Lock accounts/reference rows consistently before validating archived state. |
||||
Protect referenced history against deletion. Do not register editable admin |
||||
models or expose endpoints that bypass the service layer in this milestone. |
||||
|
||||
PostgreSQL constraints enforce nonzero amounts, sign/kind combinations, valid |
||||
choice values, opening uniqueness, installment-pair nullability and uniqueness, |
||||
source identity, and distinct/unique transfer legs. Service validation enforces |
||||
cross-table account, currency, archive, category, statement, and refund rules. |
||||
Direct SQL or ORM bulk writes can bypass cross-table rules and are unsupported |
||||
application mutation paths; this limit must be explicit in developer guidance. |
||||
|
||||
Use a common lock order by UUID for transfer legs so competing match requests |
||||
serialize. Enforce transfer leg uniqueness across both incoming and outgoing |
||||
roles by the immutable signed amount: a negative leg can never be incoming, |
||||
and a positive leg can never be outgoing. Test concurrent attempts against real |
||||
PostgreSQL with separate connections, not mocked locks. |
||||
|
||||
## Acceptance examples (synthetic) |
||||
|
||||
- Bank starts at +1,000 BRL; a -120 expense leaves +880. |
||||
- A -300 card purchase is spending. Paying it creates -300 bank and +300 card |
||||
transfer legs; payment contributes zero additional spending. |
||||
- A +50 refund references a -120 expense and reduces net spending to 70 while |
||||
retaining both records and both source identities. |
||||
- Only installment 3/10 is imported: exactly one expense exists; another 3/10 |
||||
on the same plan is rejected, while a different plan's 3/10 is valid. |
||||
- A USD-to-BRL transfer is rejected without leaving either new leg or a success |
||||
audit. The validation failure is recorded without source payloads. |
||||
- Matching already-paired legs, including simultaneous attempts, cannot create |
||||
a second transfer or duplicate transactions. |
||||
- Source metadata from a repeated physical item is rejected, while a new source |
||||
item may be attached to the same transaction explicitly. |
||||
- Invalid foreign-account statements/refunds/installments and assigning archived |
||||
reference data are rejected without partial writes. |
||||
|
||||
## Delivery and verification |
||||
|
||||
Build models and migrations first, then ordinary posting/provenance operations, |
||||
then linked transfers/refunds/installments and audit failure handling. Each |
||||
unit gets PostgreSQL-backed tests. Add the finance app to Docker build copies |
||||
and the allowlisted build context. Verify migration application on both fresh |
||||
and existing skeleton databases, migration drift, runtime grants on new tables, |
||||
and all existing health/configuration tests. |
||||
|
||||
No real data is needed. Use synthetic records only. Keep existing unrelated |
||||
working-tree changes and the running skeleton intact. Implementation starts |
||||
after owner approval of this domain design. |
||||
@ -0,0 +1,210 @@
|
||||
# Reviewed Imports Design |
||||
|
||||
**Date:** 2026-09-06 |
||||
**Status:** Approved |
||||
**Milestone:** Roadmap step 4 |
||||
|
||||
## Goal and delivery surface |
||||
|
||||
Import supported bank-account CSV and Banco do Brasil/Ourocard TXT through |
||||
separate parsing, normalization, validation, duplicate detection, preview, and |
||||
atomic commit stages. Use synthetic fixtures and the existing financial services. |
||||
|
||||
Recommend a local management-command workflow for this milestone: persist a |
||||
preview in PostgreSQL, export its review document into ignored local storage, |
||||
then commit explicit row decisions. This delivers a usable review boundary |
||||
without introducing authentication, upload endpoints, and product screens before |
||||
roadmap step 5. A web wizard is a viable alternative but expands this milestone |
||||
into UI and session-security design. An automatic import command would not meet |
||||
the required review-before-commit behavior. |
||||
|
||||
## Source evidence and supported profiles |
||||
|
||||
Inspected code cells only in Robopato's account and credit-card ETL notebooks. |
||||
No notebook output or financial records are copied into this repository. Its |
||||
parsers establish these initial compatibility profiles: |
||||
|
||||
- Account CSV: Latin-1 text, comma delimiter, a header followed by records with |
||||
at least six columns. Column 0 is DD/MM/YYYY, column 2 is description, column 5 |
||||
is a signed decimal with a decimal point. CSV quoting must work correctly. |
||||
Recognized `S A L D O` balance rows are excluded with an explicit summary count. |
||||
- Full Ourocard TXT: DD.MM.YYYY followed by fixed-width description/location |
||||
fields (37 characters in the regular profile), two-character country field, |
||||
and two Brazilian-formatted monetary columns. Installment markers use |
||||
`PARC n/total`. The known `PGTO DEBITO CONTA` payment line has a separate layout. |
||||
- Partial Ourocard TXT: DD/MM, 43-character description/location field, |
||||
two-character country, and two Brazilian-formatted monetary columns. |
||||
- Owner/section headers delimit source sections. Store an ordinal section ID |
||||
for mapping; do not depend on owner-name database lookups. |
||||
|
||||
These are explicit versioned compatibility profiles, not a claim to support |
||||
all bank exports. Validate expected CSV header width and row shapes rather than |
||||
guessing column mappings. Exact accepted header aliases and TXT grammar will |
||||
be documented alongside synthetic fixtures during implementation. The parser |
||||
must fail closed on date-like or transaction-like lines it cannot classify. |
||||
Known section headings, blank lines, and recognized totals may be skipped with |
||||
counts; other unrecognized nonblank text is surfaced for acknowledgement. |
||||
|
||||
Decode UTF-8 (including BOM) first, then Latin-1. Record the selected encoding; |
||||
never use replacement-character decoding. Reject NUL bytes and oversized input |
||||
(over 10 MiB or 20,000 candidate rows). Preserve physical line/item locators. |
||||
No OFX, PDF parsing, network fetches, bank APIs, or pandas dependency. |
||||
|
||||
## Context, dates, and monetary interpretation |
||||
|
||||
Caller selects the parser profile and destination account; formats must match |
||||
account kind. Bank and Ourocard profiles in this milestone post BRL amounts |
||||
only. Reject other account currencies, because neither profile supplies a |
||||
reliable conversion contract. The domain still supports other currencies for |
||||
manual activity. |
||||
|
||||
Full dates are used as written. For yearless lines require an explicit inclusive |
||||
start/end context of at most 366 days; resolve each DD/MM only if exactly one |
||||
valid date lies in that interval. Missing/ambiguous dates block commitment. |
||||
Never use the current year. Source transaction dates become occurrence and |
||||
posting dates for these initial profiles; statement association does not replace |
||||
individual dates. Show this convention in preview. |
||||
|
||||
CSV amounts retain their signs. In regular TXT rows the first monetary column |
||||
is debit and the second credit: normalized amount = credit - debit using exact |
||||
Decimal operations independent of caller context. Negative debit represents a |
||||
credit adjustment. Nonzero values in both columns require review as an ambiguous |
||||
row rather than silently netting. Known payment lines map the payment amount to |
||||
a positive unmatched TRANSFER leg; no artificial bank leg is created. Regular |
||||
card credits are proposed REFUND, bank credits INCOME, and debits EXPENSE. |
||||
|
||||
Source labels are proposals: a reviewer may classify permitted rows as unmatched |
||||
TRANSFER or REFUND, subject to amount sign and domain validation. Do not infer a |
||||
transfer from a merchant-like keyword. Balance/previous-balance totals never |
||||
become expenses or opening entries automatically. |
||||
|
||||
## Pure parser and normalized candidate contracts |
||||
|
||||
Create an `ingestion` Django application; avoid naming a source package `imports` |
||||
because that name is reserved for ignored local financial files. |
||||
|
||||
Parsers take bytes plus explicitly supplied date context and return immutable |
||||
parsed items/issues. They do no database or filesystem access. Normalization |
||||
converts parsed items into candidates containing source locator, section ordinal, |
||||
posting/occurrence dates, original description, signed decimal string, proposed |
||||
kind, optional installment number/count, and parser version. Candidate IDs are |
||||
stable per document/locator. Parser issues have code, severity, and locator, with |
||||
no raw line in generic diagnostic messages. |
||||
|
||||
Validation resolves account/reference IDs against existing domain rules and |
||||
returns field-level issues. Duplicate detection consumes validated candidates |
||||
and database snapshots through a separate boundary. Preview persistence records |
||||
results without creating Transaction, Transfer, SourceRecord, or InstallmentPlan |
||||
rows. Preview audit events and staging records are allowed. |
||||
|
||||
## Persisted review lifecycle |
||||
|
||||
ImportBatch: UUID, document SHA-256, format/profile/version, context, selected |
||||
account/section mapping, actor identity, creation timestamp, state (PREVIEW, |
||||
COMMITTED, CANCELLED), monotonically increasing revision, and commit result. |
||||
ImportItem: batch, unique physical locator, normalized candidate payload, |
||||
validation issues, duplicate suggestions, and explicit decision metadata. |
||||
|
||||
Staged descriptions and review documents are financial data: keep them only in |
||||
local PostgreSQL and ignored local files. Do not store uploaded bytes in the |
||||
batch or logs. CLI reads source documents from ignored `imports/`; only the |
||||
management service mounts that directory, not the web container. Commit uses |
||||
the immutable stored candidates, not a re-read of a mutable source file. |
||||
|
||||
A review document is JSON with batch UUID/revision and decisions keyed by item |
||||
locator. It may contain sensitive descriptions for local human inspection. |
||||
Reject unknown keys, duplicate locators, foreign-batch items, and stale revisions. |
||||
Review decisions may specify kind, category, payee, statement, refund reference, |
||||
and installment plan mapping; they cannot rewrite amount, date, or source text. |
||||
An incorrect amount/date requires a corrected source/new preview or later manual |
||||
correction workflow, not an untraceable edit to imported evidence. |
||||
|
||||
Updating decisions increments revision and recomputes preview validation. |
||||
Every importable row requires an explicit decision. Cancelled batches cannot be |
||||
committed. Committing an already committed batch returns the recorded result, |
||||
without new transactions or provenance; retry arguments must refer to that batch. |
||||
|
||||
## Duplicate handling and review decisions |
||||
|
||||
Exact physical source identity remains (document hash, item locator). An already |
||||
attached source item is shown as ALREADY_IMPORTED with its existing transaction; |
||||
it cannot be created or attached again, even through a new batch or parser version. |
||||
If it belongs to another account, show a blocking account conflict. |
||||
|
||||
Possible duplicates use the same account, currency, posting date, exact amount, |
||||
normalized description (Unicode NFC, casefold, collapsed whitespace), kind, |
||||
and installment number/count when present. Compare both existing records and |
||||
items within the batch. Do not impose a semantic uniqueness constraint: two |
||||
legitimate purchases can share this signature. |
||||
|
||||
Allowed decisions: |
||||
|
||||
- CREATE: record a new transaction. Possible duplicates require an explicit |
||||
acknowledgement flag; default preview never silently deduplicates them. |
||||
- MATCH: attach this source item to a specifically selected existing transaction, |
||||
only when its immutable financial fields and duplicate signature still match. |
||||
No new transaction is created. Do not mutate the existing record's annotations. |
||||
- SKIP: explicitly omit a row. Require a reason code, not a free-form sensitive |
||||
diagnostic. A skipped source remains unclaimed and can be reviewed in a future |
||||
batch. Already-imported rows are skipped with an explicit existing-source reason. |
||||
|
||||
Malformed transaction rows block commit until explicitly skipped with a reason; |
||||
file-level corruption or unsupported profile cannot be overridden. Unrecognized |
||||
nontransaction lines require acknowledgement. Do not suppress skipped/error |
||||
counts from the review result. |
||||
|
||||
## Installments, refunds, and multiple sections |
||||
|
||||
Do not guess installment identity from merchant and amount. Rows with installment |
||||
metadata require an explicit existing plan or a named new-plan decision scoped |
||||
to this batch, with account and total count validated. Create new plans only in |
||||
the commit transaction, reuse the chosen plan across selected rows, and never |
||||
create unobserved installments. Do not use bank periodic-transfer text to infer |
||||
an expense installment automatically; show metadata as a review hint. |
||||
|
||||
Refunds may remain unlinked or explicitly reference a compatible expense. |
||||
Statements may be selected from existing records but are not auto-created from |
||||
incomplete header data. A source with multiple owner sections must map each |
||||
section explicitly to an account or mark it skipped. The same card account can |
||||
receive multiple sections. Never use a person's name as an automatic account key. |
||||
|
||||
## Commit integrity and concurrency |
||||
|
||||
Use one outermost transaction for all accepted rows, provenance, chosen new |
||||
plans, batch status and success audit. Reuse internal domain validation/posting |
||||
helpers, not nested public service transactions. A rejected commit rolls back |
||||
all financial writes and records a sanitized failure audit after rollback; |
||||
PREVIEW remains available for correction. |
||||
|
||||
Lock the batch, then all affected accounts in UUID order; lock reference rows |
||||
consistently before target transactions. Revalidate archive state, relationships, |
||||
source identity and duplicate suggestions under locks immediately before writes. |
||||
Lock existing candidate-match transactions before comparing annotations. If live |
||||
state differs materially from the reviewed snapshot, return STALE_PREVIEW and |
||||
require a refreshed preview/review rather than silently changing a decision. |
||||
Account locks serialize commits and ordinary postings on the same account. |
||||
Database source uniqueness resolves identical-file races across accounts. |
||||
|
||||
Ensure the multi-row commit uses one consistent lock order across the entire |
||||
batch, not each row independently. Plan creation, source attachment and auditing |
||||
must leave no partial records if a later row fails. Only transaction UUIDs, |
||||
counts, operation and error codes enter generic completion messages. |
||||
|
||||
## Commands and acceptance |
||||
|
||||
Provide management commands to create/export a preview, apply a decision file, |
||||
refresh a preview, commit a specific batch/revision, and cancel a batch. Commands |
||||
require explicit actor kind/identifier and account/section mapping. Document |
||||
PowerShell examples using the management container and ignored local import |
||||
files. No real financial records are included in the examples. |
||||
|
||||
Test with synthetic files representing quoting, encodings, invalid dates/money, |
||||
full/partial card layouts, year boundaries, payments, refunds, balance lines, |
||||
multiple sections, installments, and unknown lines. Verify preview creates no |
||||
financial rows, cancellation is inert, skipping is explicit, exact reimport is |
||||
idempotent, semantic duplicates remain reviewable, and MATCH preserves history. |
||||
|
||||
PostgreSQL tests must cover a failing later row rolling back the entire batch, |
||||
concurrent identical imports, stale review after other writes, archive changes, |
||||
repeated commit, source conflicts, and failure audits. Verify new table grants, |
||||
fresh/existing migrations, all financial-domain regressions, and app health. |
||||
@ -0,0 +1,212 @@
|
||||
# Local MVP Workflows Design |
||||
|
||||
## Purpose and scope |
||||
|
||||
Roadmap step 5 gives the local owner a server-rendered Django interface for |
||||
managing reference data and transactions, categorizing imported activity, and |
||||
inspecting monthly summaries. It also provides durable, local exports and |
||||
PostgreSQL backup and restore-verification commands. Together these complete a |
||||
documented synthetic import-to-report-to-backup workflow. |
||||
|
||||
The reviewed-import parser and decision workflow remains command based. The web |
||||
application begins with the committed financial records and links to the local |
||||
import instructions. Bank APIs, investments, forecasting, receipt recognition, |
||||
mobile clients, hosted multi-user operation, and automatic or AI categorization |
||||
remain outside the MVP. |
||||
|
||||
## Chosen approach |
||||
|
||||
Use Django templates, forms, and ordinary HTTP requests. Keep read behavior in |
||||
explicit query and report modules, presentation in `core`, and all mutations in |
||||
the existing audited `finance.services` boundary. This adds no JavaScript |
||||
framework and does not expose Django admin as a financial editor. |
||||
|
||||
Alternatives considered were a command-only workflow and a client-rendered API |
||||
application. Command-only operation would not provide the planned local screens. |
||||
A separate API and JavaScript client would add authentication, serialization, |
||||
state-management, and build-tooling work without improving this single-owner |
||||
milestone. |
||||
|
||||
## Application surface |
||||
|
||||
The root page is a monthly dashboard. The local navigation exposes accounts, |
||||
transactions, categories, payees, reports, and exports, plus a link to the |
||||
reviewed-import guide. Server-rendered pages use semantic HTML and remain usable |
||||
without client-side JavaScript. |
||||
|
||||
Transaction pages provide: |
||||
|
||||
- a paginated list ordered by posting date and stable identifier; |
||||
- filters for inclusive date range, account, kind, payee, category, currency, |
||||
and uncategorized status; |
||||
- a detail page with relationships and source provenance; |
||||
- manual creation through the supported financial posting service; and |
||||
- annotation of description, payee, and category through |
||||
`annotate_transaction`. |
||||
|
||||
The interface does not delete transactions or rewrite amount, posting date, |
||||
occurrence date, account, kind, refund, transfer, statement, installment, or |
||||
source identity. Financial corrections are represented by new entries, refunds, |
||||
or transfers through the domain services. Imported transaction evidence remains |
||||
immutable. |
||||
|
||||
Reference pages list active and archived accounts, categories, and payees. |
||||
Creation and archival use `create_account`, `create_category`, `create_payee`, |
||||
and `archive`. Archived references remain visible in historical records and |
||||
cannot be selected for new assignments. |
||||
|
||||
## Request security and accountability |
||||
|
||||
All mutation forms use Django CSRF protection and POST requests. Views never |
||||
call model `save()` directly for financial writes. The local server derives the |
||||
actor as `Actor(kind="HUMAN", identifier="local-owner")`; actor identity is not |
||||
accepted from request data. Domain errors become safe form messages keyed by |
||||
their stable error codes and never include transaction descriptions or source |
||||
rows. |
||||
|
||||
The application continues to bind only to loopback. Formal login and multi-user |
||||
authorization are outside this local single-owner release. Django session and |
||||
message support may be enabled only to carry local form feedback; no financial |
||||
values are stored in cookies. |
||||
|
||||
## Query boundaries |
||||
|
||||
`finance.queries` owns reusable read models for transaction and reference lists. |
||||
It validates filter values, applies deterministic ordering, selects related |
||||
objects deliberately, and returns querysets or immutable result values without |
||||
mutating records. Views remain responsible for HTTP parsing and pagination. |
||||
|
||||
`finance.reports` owns aggregate calculations and returns immutable report |
||||
values. It performs decimal aggregation in PostgreSQL and never combines |
||||
currencies. Empty periods return explicit zero totals and empty category rows. |
||||
|
||||
Only query shapes used by the screens and exports receive indexes. Initial |
||||
indexes cover transaction posting date and the common account-plus-posting-date |
||||
path. Their migrations are verified against PostgreSQL. |
||||
|
||||
## Reporting semantics |
||||
|
||||
All monthly reports use `posting_date` and an inclusive calendar month selected |
||||
by the owner. Every report selects exactly one currency. Archived accounts and |
||||
references remain included when they have activity in the selected period. |
||||
|
||||
The monthly cash-flow summary uses economic activity across bank and credit-card |
||||
accounts: |
||||
|
||||
- `INCOME` contributes its positive amount to inflow; |
||||
- `EXPENSE` contributes the absolute value of its negative amount to spending; |
||||
- `REFUND` reduces spending in the month in which the refund posts; |
||||
- `TRANSFER` is excluded, including credit-card payments; and |
||||
- `OPENING` is excluded. |
||||
|
||||
Net activity is inflow minus net spending. Net spending may be negative when a |
||||
month contains refunds greater than expenses. A card purchase is counted as |
||||
spending when it posts; its later card-payment transfer is excluded, so it is |
||||
not counted twice. |
||||
|
||||
The spending summary groups `EXPENSE` and `REFUND` amounts by expense category. |
||||
An explicit Uncategorized row includes transactions without a category. |
||||
Refunds use their own assigned category and are not automatically inherited from |
||||
the original purchase. Rows show expense, refund, and net-spending totals and |
||||
sort by net spending descending with a stable category-name tie-breaker. |
||||
|
||||
Account balances are separate from cash flow. A balance is the signed sum of all |
||||
entries through the selected month end, including openings and transfer legs. |
||||
Balances are reported per account and currency and are never summed across |
||||
currencies. |
||||
|
||||
## Export format and safety |
||||
|
||||
An export service and management command create a versioned ZIP package under |
||||
the ignored repository-root `exports/` directory. The browser can request the |
||||
same package and receives it as a download; it does not choose a server |
||||
filesystem path. |
||||
|
||||
Each package contains a JSON manifest and UTF-8 CSV files for accounts, payees, |
||||
categories, statements, installment plans, transactions, transfers, and source |
||||
records. Stable UUIDs, ISO-8601 dates and timestamps, exact decimal strings, |
||||
archival state, relationship UUIDs, and provenance metadata are retained. |
||||
Audit events and import-review working state belong to backups rather than the |
||||
portable financial export. |
||||
|
||||
The manifest records the export schema version, creation time, application |
||||
version, file names, row counts, and SHA-256 checksums. CSV headers and file |
||||
ordering are fixed by export schema version. Database reads occur inside one |
||||
consistent transaction so the package represents one snapshot. |
||||
|
||||
Command output contains only the package name, version, and row counts. Export |
||||
names are generated or validated as plain file names. Paths, traversal, |
||||
symlinks, existing destinations, and files outside `EXPORT_ROOT` are rejected. |
||||
The package is written to a private temporary file, flushed, and atomically |
||||
renamed. Failures remove the temporary file without replacing an existing |
||||
export. |
||||
|
||||
## PostgreSQL backups and restore verification |
||||
|
||||
Backup management commands invoke the PostgreSQL 17 `pg_dump` and `pg_restore` |
||||
clients from the application image. They use the management service's migration |
||||
database credentials and never accept passwords as command arguments. Backups |
||||
use PostgreSQL custom format with `--no-owner` and `--no-acl` and are stored |
||||
under the ignored repository-root `backups/` directory. |
||||
|
||||
Each backup has a JSON manifest containing format version, creation time, |
||||
application version, PostgreSQL client version, database name, applied Django |
||||
migrations, byte size, and SHA-256 checksum. Creation uses a private temporary |
||||
file, `fsync`, and atomic rename; existing targets are never overwritten. |
||||
Terminal output excludes financial data and credentials. |
||||
|
||||
Restore verification requires a database connection explicitly marked as a |
||||
verification target and rejects the configured live database host/name pair. |
||||
The target database must be empty. Verification checks the manifest and digest, |
||||
restores with `pg_restore --exit-on-error --single-transaction`, then runs Django |
||||
schema checks and deterministic row-count/integrity checks. The acceptance test |
||||
uses a disposable PostgreSQL service populated only with synthetic data. |
||||
|
||||
Restoring over the live database is not part of the web UI or automated |
||||
verification. The owner-facing recovery guide documents a deliberate command |
||||
sequence and requires stopping the web service, creating a separate safety |
||||
backup, and naming the target database explicitly. |
||||
|
||||
## Configuration and storage |
||||
|
||||
`EXPORT_ROOT` and `BACKUP_ROOT` default to `/app/exports` and `/app/backups` in |
||||
containers. Docker Compose mounts the ignored repository directories into the |
||||
management and web services with only the access each workflow needs. The |
||||
PostgreSQL client major version matches the PostgreSQL 17 server. |
||||
|
||||
Generated exports, backups, temporary files, database dumps, reports, and logs |
||||
remain ignored. Automated tests use synthetic records and temporary directories. |
||||
No fixture or assertion contains copied financial data. |
||||
|
||||
## Error handling |
||||
|
||||
Form validation errors stay on the submitted page without mutation. Domain |
||||
conflicts are mapped to stable, human-readable local messages. Missing objects |
||||
return 404. Invalid report and list filters return field-level errors rather |
||||
than silently changing the requested range. |
||||
|
||||
Export and backup commands fail closed for invalid paths, existing files, |
||||
checksum mismatch, incompatible format versions, unavailable PostgreSQL tools, |
||||
database errors, or nonempty verification targets. They emit stable safe error |
||||
codes and leave no published partial artifact. |
||||
|
||||
## Verification and completion |
||||
|
||||
Unit and PostgreSQL-backed tests cover query filters, pagination stability, |
||||
report calculations, currency separation, refunds, transfers, openings, |
||||
uncategorized spending, archived references, and account balances. Request tests |
||||
cover page rendering, CSRF-protected POST workflows, audited service writes, |
||||
immutable financial fields, and safe errors. |
||||
|
||||
Export tests inspect package schemas, ordering, exact decimals, manifests, |
||||
checksums, snapshot behavior, path rejection, non-overwrite behavior, and failed |
||||
write cleanup. Backup integration tests create a synthetic database state, |
||||
produce a dump, restore it into a separate empty database, verify migrations and |
||||
row integrity, and confirm the original database is unchanged. |
||||
|
||||
The documented acceptance workflow creates synthetic reference data, previews |
||||
and commits a synthetic import, categorizes a transaction in the browser, |
||||
inspects monthly cash flow and spending, downloads an export, creates a backup, |
||||
and verifies restoration into the disposable database. Roadmap step 5 is |
||||
complete when that workflow and the full automated suite pass from the Docker |
||||
Compose environment. |
||||
@ -0,0 +1 @@
|
||||
"""Financial domain persistence and audited write operations.""" |
||||
@ -0,0 +1,268 @@
|
||||
import csv |
||||
import json |
||||
import os |
||||
import re |
||||
import tempfile |
||||
from dataclasses import dataclass |
||||
from datetime import date, datetime, timezone as datetime_timezone |
||||
from decimal import Decimal |
||||
from hashlib import sha256 |
||||
from io import BytesIO, StringIO |
||||
from pathlib import Path |
||||
from uuid import UUID |
||||
from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo |
||||
|
||||
from django.conf import settings |
||||
from django.db import connection, transaction |
||||
from django.utils import timezone |
||||
|
||||
from finance.models import ( |
||||
Account, |
||||
CardStatement, |
||||
Category, |
||||
InstallmentPlan, |
||||
Payee, |
||||
SourceRecord, |
||||
Transaction, |
||||
Transfer, |
||||
) |
||||
|
||||
|
||||
EXPORT_FORMAT = "gizmoduck-export" |
||||
EXPORT_VERSION = 1 |
||||
APPLICATION_VERSION = "0.1.0" |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class ExportArtifact: |
||||
filename: str |
||||
content: bytes |
||||
version: int |
||||
row_counts: dict[str, int] |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class ExportResult: |
||||
name: str |
||||
path: Path |
||||
version: int |
||||
row_counts: dict[str, int] |
||||
|
||||
|
||||
class ExportError(Exception): |
||||
def __init__(self, code): |
||||
self.code = code |
||||
super().__init__(code) |
||||
|
||||
|
||||
_PLAIN_ZIP_NAME = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9._-]{0,198}\.zip\Z") |
||||
|
||||
|
||||
SCHEMAS = ( |
||||
( |
||||
"accounts.csv", |
||||
Account, |
||||
("id", "name", "kind", "currency", "archived", "created_at", "updated_at"), |
||||
), |
||||
( |
||||
"payees.csv", |
||||
Payee, |
||||
("id", "name", "archived", "created_at", "updated_at"), |
||||
), |
||||
( |
||||
"categories.csv", |
||||
Category, |
||||
("id", "name", "kind", "archived", "created_at", "updated_at"), |
||||
), |
||||
( |
||||
"statements.csv", |
||||
CardStatement, |
||||
( |
||||
"id", |
||||
"account_id", |
||||
"period_start", |
||||
"period_end", |
||||
"due_date", |
||||
"statement_balance", |
||||
"created_at", |
||||
"updated_at", |
||||
), |
||||
), |
||||
( |
||||
"installment_plans.csv", |
||||
InstallmentPlan, |
||||
("id", "account_id", "description", "total_installments", "created_at", "updated_at"), |
||||
), |
||||
( |
||||
"transactions.csv", |
||||
Transaction, |
||||
( |
||||
"id", |
||||
"account_id", |
||||
"amount", |
||||
"posting_date", |
||||
"occurrence_date", |
||||
"description", |
||||
"kind", |
||||
"payee_id", |
||||
"category_id", |
||||
"statement_id", |
||||
"refund_of_id", |
||||
"installment_plan_id", |
||||
"installment_number", |
||||
"created_at", |
||||
"updated_at", |
||||
), |
||||
), |
||||
( |
||||
"transfers.csv", |
||||
Transfer, |
||||
("id", "outgoing_id", "incoming_id", "created_at", "updated_at"), |
||||
), |
||||
( |
||||
"source_records.csv", |
||||
SourceRecord, |
||||
( |
||||
"id", |
||||
"transaction_id", |
||||
"document_hash", |
||||
"source_format", |
||||
"parser_version", |
||||
"item_locator", |
||||
"created_at", |
||||
), |
||||
), |
||||
) |
||||
|
||||
|
||||
def build_export() -> ExportArtifact: |
||||
with transaction.atomic(): |
||||
with connection.cursor() as cursor: |
||||
cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") |
||||
created_at = timezone.now() |
||||
csv_files = { |
||||
filename: _csv_content(model, fields) |
||||
for filename, model, fields in SCHEMAS |
||||
} |
||||
|
||||
file_manifest = { |
||||
filename: { |
||||
"rows": row_count, |
||||
"sha256": sha256(content).hexdigest(), |
||||
} |
||||
for (filename, _model, _fields), (content, row_count) in zip( |
||||
SCHEMAS, csv_files.values(), strict=True |
||||
) |
||||
} |
||||
manifest = { |
||||
"format": EXPORT_FORMAT, |
||||
"version": EXPORT_VERSION, |
||||
"created_at": _value(created_at), |
||||
"application_version": APPLICATION_VERSION, |
||||
"files": file_manifest, |
||||
} |
||||
package = BytesIO() |
||||
with ZipFile(package, "w") as archive: |
||||
for filename, (content, _row_count) in csv_files.items(): |
||||
_write_member(archive, filename, content) |
||||
manifest_content = ( |
||||
json.dumps(manifest, ensure_ascii=False, separators=(",", ":")) + "\n" |
||||
).encode("utf-8") |
||||
_write_member(archive, "manifest.json", manifest_content) |
||||
|
||||
return ExportArtifact( |
||||
filename=f"gizmoduck-export-{created_at.astimezone(datetime_timezone.utc):%Y%m%dT%H%M%SZ}.zip", |
||||
content=package.getvalue(), |
||||
version=EXPORT_VERSION, |
||||
row_counts={name: details["rows"] for name, details in file_manifest.items()}, |
||||
) |
||||
|
||||
|
||||
def write_export(name: str | None = None) -> ExportResult: |
||||
if name is None: |
||||
name = f"gizmoduck-export-{timezone.now().astimezone(datetime_timezone.utc):%Y%m%dT%H%M%SZ}.zip" |
||||
if not isinstance(name, str) or not _PLAIN_ZIP_NAME.fullmatch(name): |
||||
raise ExportError("invalid_name") |
||||
|
||||
try: |
||||
root = Path(settings.EXPORT_ROOT).resolve(strict=True) |
||||
except (OSError, RuntimeError, TypeError): |
||||
raise ExportError("invalid_root") from None |
||||
if not root.is_dir(): |
||||
raise ExportError("invalid_root") |
||||
|
||||
destination = root / name |
||||
if destination.is_symlink(): |
||||
raise ExportError("invalid_name") |
||||
if destination.exists(): |
||||
raise ExportError("existing_destination") |
||||
|
||||
temporary_path = None |
||||
phase = "snapshot" |
||||
try: |
||||
artifact = build_export() |
||||
phase = "publication" |
||||
temporary_fd, temporary_name = tempfile.mkstemp( |
||||
prefix=".gizmoduck-export-", suffix=".tmp", dir=root |
||||
) |
||||
temporary_path = Path(temporary_name) |
||||
with os.fdopen(temporary_fd, "wb") as stream: |
||||
stream.write(artifact.content) |
||||
stream.flush() |
||||
os.fsync(stream.fileno()) |
||||
try: |
||||
os.link(temporary_path, destination) |
||||
except FileExistsError: |
||||
raise ExportError("existing_destination") from None |
||||
temporary_path.unlink() |
||||
temporary_path = None |
||||
return ExportResult( |
||||
name=name, |
||||
path=destination, |
||||
version=artifact.version, |
||||
row_counts=artifact.row_counts, |
||||
) |
||||
except ExportError: |
||||
raise |
||||
except Exception: |
||||
code = "export_failed" if phase == "snapshot" else "publication_failed" |
||||
raise ExportError(code) from None |
||||
finally: |
||||
if temporary_path is not None: |
||||
try: |
||||
temporary_path.unlink(missing_ok=True) |
||||
except OSError: |
||||
pass |
||||
|
||||
|
||||
def _csv_content(model, fields): |
||||
stream = StringIO(newline="") |
||||
writer = csv.writer(stream, lineterminator="\n") |
||||
writer.writerow(fields) |
||||
row_count = 0 |
||||
for row in model.objects.order_by("id").values_list(*fields): |
||||
writer.writerow(_value(value) for value in row) |
||||
row_count += 1 |
||||
return stream.getvalue().encode("utf-8"), row_count |
||||
|
||||
|
||||
def _value(value): |
||||
if value is None: |
||||
return "" |
||||
if isinstance(value, bool): |
||||
return str(value).lower() |
||||
if isinstance(value, Decimal): |
||||
return format(value, "f") |
||||
if isinstance(value, UUID): |
||||
return str(value) |
||||
if isinstance(value, (date, datetime)): |
||||
return value.isoformat() |
||||
return str(value) |
||||
|
||||
|
||||
def _write_member(archive, filename, content): |
||||
info = ZipInfo(filename, date_time=(1980, 1, 1, 0, 0, 0)) |
||||
info.compress_type = ZIP_DEFLATED |
||||
info.create_system = 3 |
||||
info.external_attr = 0o600 << 16 |
||||
archive.writestr(info, content, compress_type=ZIP_DEFLATED, compresslevel=9) |
||||
@ -0,0 +1,23 @@
|
||||
import json |
||||
|
||||
from django.core.management.base import BaseCommand, CommandError |
||||
|
||||
from finance.exports import ExportError, write_export |
||||
|
||||
|
||||
class Command(BaseCommand): |
||||
help = "Create a versioned portable financial export." |
||||
|
||||
def add_arguments(self, parser): |
||||
parser.add_argument("--output", dest="name") |
||||
|
||||
def handle(self, *args, **options): |
||||
try: |
||||
result = write_export(options["name"]) |
||||
except ExportError as exc: |
||||
raise CommandError(exc.code) from None |
||||
self.stdout.write(json.dumps({ |
||||
"name": result.name, |
||||
"version": result.version, |
||||
"rows": result.row_counts, |
||||
}, separators=(",", ":"))) |
||||
@ -0,0 +1,230 @@
|
||||
# Generated by Django 5.2.17 on 2026-09-06 19:02 |
||||
|
||||
import django.core.validators |
||||
import django.db.models.deletion |
||||
import uuid |
||||
from django.db import migrations, models |
||||
|
||||
|
||||
class Migration(migrations.Migration): |
||||
|
||||
initial = True |
||||
|
||||
dependencies = [ |
||||
] |
||||
|
||||
operations = [ |
||||
migrations.CreateModel( |
||||
name='Account', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('name', models.CharField(max_length=255, unique=True)), |
||||
('kind', models.CharField(choices=[('BANK', 'Bank'), ('CREDIT_CARD', 'Credit card')], max_length=16)), |
||||
('currency', models.CharField(default='BRL', max_length=3, validators=[django.core.validators.RegexValidator('\\A[A-Z]{3}\\Z', 'Use three uppercase ASCII letters.')])), |
||||
('archived', models.BooleanField(default=False)), |
||||
], |
||||
options={ |
||||
'constraints': [models.CheckConstraint(condition=models.Q(('kind__in', ['BANK', 'CREDIT_CARD'])), name='account_valid_kind'), models.CheckConstraint(condition=models.Q(('currency__regex', '^[A-Z]{3}$')), name='account_valid_currency'), models.CheckConstraint(condition=models.Q(('name__regex', '\\S')), name='account_nonblank_name')], |
||||
}, |
||||
), |
||||
migrations.CreateModel( |
||||
name='AuditEvent', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('actor_kind', models.CharField(choices=[('HUMAN', 'Human'), ('APPLICATION', 'Application'), ('AGENT', 'Agent')], max_length=16)), |
||||
('actor_identifier', models.CharField(max_length=255)), |
||||
('operation', models.CharField(max_length=128)), |
||||
('target_id', models.UUIDField(blank=True, null=True)), |
||||
('outcome', models.CharField(choices=[('SUCCESS', 'Success'), ('FAILURE', 'Failure')], max_length=16)), |
||||
('changed_fields', models.JSONField(blank=True, default=list)), |
||||
('error_code', models.CharField(blank=True, max_length=128)), |
||||
], |
||||
options={ |
||||
'constraints': [models.CheckConstraint(condition=models.Q(('actor_kind__in', ['HUMAN', 'APPLICATION', 'AGENT'])), name='audit_valid_actor'), models.CheckConstraint(condition=models.Q(('outcome__in', ['SUCCESS', 'FAILURE'])), name='audit_valid_outcome'), models.CheckConstraint(condition=models.Q(('actor_identifier__regex', '\\S')), name='audit_nonblank_actor'), models.CheckConstraint(condition=models.Q(('operation__regex', '\\S')), name='audit_nonblank_operation')], |
||||
}, |
||||
), |
||||
migrations.CreateModel( |
||||
name='CardStatement', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('period_start', models.DateField()), |
||||
('period_end', models.DateField()), |
||||
('due_date', models.DateField()), |
||||
('statement_balance', models.DecimalField(blank=True, decimal_places=2, max_digits=18, null=True)), |
||||
('account', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='statements', to='finance.account')), |
||||
], |
||||
), |
||||
migrations.CreateModel( |
||||
name='Category', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('name', models.CharField(max_length=255, unique=True)), |
||||
('kind', models.CharField(choices=[('INCOME', 'Income'), ('EXPENSE', 'Expense')], max_length=16)), |
||||
('archived', models.BooleanField(default=False)), |
||||
], |
||||
options={ |
||||
'constraints': [models.CheckConstraint(condition=models.Q(('kind__in', ['INCOME', 'EXPENSE'])), name='category_valid_kind'), models.CheckConstraint(condition=models.Q(('name__regex', '\\S')), name='category_nonblank_name')], |
||||
}, |
||||
), |
||||
migrations.CreateModel( |
||||
name='InstallmentPlan', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('description', models.TextField(blank=True)), |
||||
('total_installments', models.PositiveIntegerField(validators=[django.core.validators.MinValueValidator(1)])), |
||||
('account', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='installment_plans', to='finance.account')), |
||||
], |
||||
), |
||||
migrations.CreateModel( |
||||
name='Payee', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('name', models.CharField(max_length=255, unique=True)), |
||||
('archived', models.BooleanField(default=False)), |
||||
], |
||||
options={ |
||||
'constraints': [models.CheckConstraint(condition=models.Q(('name__regex', '\\S')), name='payee_nonblank_name')], |
||||
}, |
||||
), |
||||
migrations.CreateModel( |
||||
name='Transaction', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('amount', models.DecimalField(decimal_places=2, max_digits=18)), |
||||
('posting_date', models.DateField()), |
||||
('occurrence_date', models.DateField(blank=True, null=True)), |
||||
('description', models.TextField(blank=True)), |
||||
('kind', models.CharField(choices=[('INCOME', 'Income'), ('EXPENSE', 'Expense'), ('TRANSFER', 'Transfer'), ('REFUND', 'Refund'), ('OPENING', 'Opening')], max_length=16)), |
||||
('installment_number', models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(1)])), |
||||
('account', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='transactions', to='finance.account')), |
||||
('category', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='transactions', to='finance.category')), |
||||
('installment_plan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='transactions', to='finance.installmentplan')), |
||||
('payee', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='transactions', to='finance.payee')), |
||||
('refund_of', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='refunds', to='finance.transaction')), |
||||
('statement', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='transactions', to='finance.cardstatement')), |
||||
], |
||||
), |
||||
migrations.CreateModel( |
||||
name='SourceRecord', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('document_hash', models.CharField(max_length=64, validators=[django.core.validators.RegexValidator('\\A[0-9a-f]{64}\\Z', 'Use a lowercase SHA-256 hexadecimal digest.')])), |
||||
('source_format', models.CharField(choices=[('BANK_CSV', 'Bank CSV'), ('OUROCARD_TXT', 'Ourocard TXT')], max_length=16)), |
||||
('parser_version', models.CharField(max_length=128)), |
||||
('item_locator', models.CharField(max_length=255)), |
||||
('transaction', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='source_records', to='finance.transaction')), |
||||
], |
||||
), |
||||
migrations.CreateModel( |
||||
name='Transfer', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('incoming', models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='incoming_transfer', to='finance.transaction')), |
||||
('outgoing', models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='outgoing_transfer', to='finance.transaction')), |
||||
], |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='cardstatement', |
||||
constraint=models.CheckConstraint(condition=models.Q(('period_end__gte', models.F('period_start'))), name='statement_period_order'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='cardstatement', |
||||
constraint=models.CheckConstraint(condition=models.Q(('due_date__gte', models.F('period_end'))), name='statement_due_order'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='cardstatement', |
||||
constraint=models.UniqueConstraint(fields=('account', 'period_end'), name='statement_account_period_unique'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='installmentplan', |
||||
constraint=models.CheckConstraint(condition=models.Q(('total_installments__gt', 0)), name='plan_positive_total'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(('kind__in', ['INCOME', 'EXPENSE', 'TRANSFER', 'REFUND', 'OPENING'])), name='transaction_valid_kind'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(('amount', 0), _negated=True), name='transaction_nonzero_amount'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(models.Q(('amount__gt', 0), ('kind__in', ['INCOME', 'REFUND'])), models.Q(('amount__lt', 0), ('kind', 'EXPENSE')), ('kind__in', ['TRANSFER', 'OPENING']), _connector='OR'), name='transaction_kind_sign'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.UniqueConstraint(condition=models.Q(('kind', 'OPENING')), fields=('account',), name='transaction_one_opening'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(models.Q(('kind', 'OPENING'), _negated=True), models.Q(('category__isnull', True), ('installment_number__isnull', True), ('installment_plan__isnull', True), ('payee__isnull', True), ('refund_of__isnull', True), ('statement__isnull', True)), _connector='OR'), name='transaction_opening_unassociated'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(models.Q(('kind', 'TRANSFER'), _negated=True), models.Q(('category__isnull', True), ('installment_number__isnull', True), ('installment_plan__isnull', True), ('refund_of__isnull', True)), _connector='OR'), name='transaction_transfer_unclassified'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(('refund_of__isnull', True), ('kind', 'REFUND'), _connector='OR'), name='transaction_refund_link_kind'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(('refund_of__isnull', True), models.Q(('refund_of', models.F('id')), _negated=True), _connector='OR'), name='transaction_refund_not_self'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(models.Q(('installment_number__isnull', True), ('installment_plan__isnull', True)), models.Q(('installment_number__isnull', False), ('installment_plan__isnull', False)), _connector='OR'), name='transaction_installment_pair'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(('installment_number__isnull', True), ('installment_number__gt', 0), _connector='OR'), name='transaction_installment_positive'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.CheckConstraint(condition=models.Q(('installment_plan__isnull', True), ('kind', 'EXPENSE'), _connector='OR'), name='transaction_installment_expense'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transaction', |
||||
constraint=models.UniqueConstraint(condition=models.Q(('installment_plan__isnull', False)), fields=('installment_plan', 'installment_number'), name='transaction_installment_unique'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='sourcerecord', |
||||
constraint=models.CheckConstraint(condition=models.Q(('source_format__in', ['BANK_CSV', 'OUROCARD_TXT'])), name='source_valid_format'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='sourcerecord', |
||||
constraint=models.CheckConstraint(condition=models.Q(('document_hash__regex', '^[0-9a-f]{64}$')), name='source_valid_hash'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='sourcerecord', |
||||
constraint=models.CheckConstraint(condition=models.Q(('parser_version__regex', '\\S')), name='source_nonblank_parser'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='sourcerecord', |
||||
constraint=models.CheckConstraint(condition=models.Q(('item_locator__regex', '\\S')), name='source_nonblank_locator'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='sourcerecord', |
||||
constraint=models.UniqueConstraint(fields=('document_hash', 'item_locator'), name='source_document_item_unique'), |
||||
), |
||||
migrations.AddConstraint( |
||||
model_name='transfer', |
||||
constraint=models.CheckConstraint(condition=models.Q(('outgoing', models.F('incoming')), _negated=True), name='transfer_distinct_legs'), |
||||
), |
||||
] |
||||
@ -0,0 +1,24 @@
|
||||
from django.db import migrations |
||||
|
||||
|
||||
class Migration(migrations.Migration): |
||||
dependencies = [ |
||||
("finance", "0001_initial"), |
||||
] |
||||
|
||||
operations = [ |
||||
migrations.RunSQL( |
||||
sql=( |
||||
"CREATE INDEX finance_tx_posting_date_idx " |
||||
"ON finance_transaction (posting_date)" |
||||
), |
||||
reverse_sql="DROP INDEX finance_tx_posting_date_idx", |
||||
), |
||||
migrations.RunSQL( |
||||
sql=( |
||||
"CREATE INDEX finance_tx_account_posting_idx " |
||||
"ON finance_transaction (account_id, posting_date)" |
||||
), |
||||
reverse_sql="DROP INDEX finance_tx_account_posting_idx", |
||||
), |
||||
] |
||||
@ -0,0 +1,8 @@
|
||||
from .ledger import CardStatement, InstallmentPlan, Transaction, Transfer |
||||
from .provenance import AuditEvent, SourceRecord |
||||
from .reference import Account, Category, Payee |
||||
|
||||
__all__ = [ |
||||
"Account", "AuditEvent", "CardStatement", "Category", "InstallmentPlan", |
||||
"Payee", "SourceRecord", "Transaction", "Transfer", |
||||
] |
||||
@ -0,0 +1,18 @@
|
||||
import uuid |
||||
|
||||
from django.db import models |
||||
|
||||
|
||||
class IdentifiedModel(models.Model): |
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) |
||||
created_at = models.DateTimeField(auto_now_add=True) |
||||
|
||||
class Meta: |
||||
abstract = True |
||||
|
||||
|
||||
class TimestampedModel(IdentifiedModel): |
||||
updated_at = models.DateTimeField(auto_now=True) |
||||
|
||||
class Meta: |
||||
abstract = True |
||||
@ -0,0 +1,75 @@
|
||||
from django.core.validators import MinValueValidator |
||||
from django.db import models |
||||
from django.db.models import F, Q |
||||
|
||||
from .base import TimestampedModel |
||||
|
||||
|
||||
class CardStatement(TimestampedModel): |
||||
account = models.ForeignKey("finance.Account", on_delete=models.PROTECT, related_name="statements") |
||||
period_start = models.DateField() |
||||
period_end = models.DateField() |
||||
due_date = models.DateField() |
||||
statement_balance = models.DecimalField(max_digits=18, decimal_places=2, null=True, blank=True) |
||||
|
||||
class Meta: |
||||
constraints = [ |
||||
models.CheckConstraint(condition=Q(period_end__gte=F("period_start")), name="statement_period_order"), |
||||
models.CheckConstraint(condition=Q(due_date__gte=F("period_end")), name="statement_due_order"), |
||||
models.UniqueConstraint(fields=["account", "period_end"], name="statement_account_period_unique"), |
||||
] |
||||
|
||||
|
||||
class InstallmentPlan(TimestampedModel): |
||||
account = models.ForeignKey("finance.Account", on_delete=models.PROTECT, related_name="installment_plans") |
||||
description = models.TextField(blank=True) |
||||
total_installments = models.PositiveIntegerField(validators=[MinValueValidator(1)]) |
||||
|
||||
class Meta: |
||||
constraints = [models.CheckConstraint(condition=Q(total_installments__gt=0), name="plan_positive_total")] |
||||
|
||||
|
||||
class Transaction(TimestampedModel): |
||||
class Kind(models.TextChoices): |
||||
INCOME = "INCOME", "Income" |
||||
EXPENSE = "EXPENSE", "Expense" |
||||
TRANSFER = "TRANSFER", "Transfer" |
||||
REFUND = "REFUND", "Refund" |
||||
OPENING = "OPENING", "Opening" |
||||
|
||||
account = models.ForeignKey("finance.Account", on_delete=models.PROTECT, related_name="transactions") |
||||
amount = models.DecimalField(max_digits=18, decimal_places=2) |
||||
posting_date = models.DateField() |
||||
occurrence_date = models.DateField(null=True, blank=True) |
||||
description = models.TextField(blank=True) |
||||
kind = models.CharField(max_length=16, choices=Kind.choices) |
||||
payee = models.ForeignKey("finance.Payee", on_delete=models.PROTECT, related_name="transactions", null=True, blank=True) |
||||
category = models.ForeignKey("finance.Category", on_delete=models.PROTECT, related_name="transactions", null=True, blank=True) |
||||
statement = models.ForeignKey(CardStatement, on_delete=models.PROTECT, related_name="transactions", null=True, blank=True) |
||||
refund_of = models.ForeignKey("self", on_delete=models.PROTECT, related_name="refunds", null=True, blank=True) |
||||
installment_plan = models.ForeignKey(InstallmentPlan, on_delete=models.PROTECT, related_name="transactions", null=True, blank=True) |
||||
installment_number = models.PositiveIntegerField(null=True, blank=True, validators=[MinValueValidator(1)]) |
||||
|
||||
class Meta: |
||||
constraints = [ |
||||
models.CheckConstraint(condition=Q(kind__in=["INCOME", "EXPENSE", "TRANSFER", "REFUND", "OPENING"]), name="transaction_valid_kind"), |
||||
models.CheckConstraint(condition=~Q(amount=0), name="transaction_nonzero_amount"), |
||||
models.CheckConstraint(condition=(Q(kind__in=["INCOME", "REFUND"], amount__gt=0) | Q(kind="EXPENSE", amount__lt=0) | Q(kind__in=["TRANSFER", "OPENING"])), name="transaction_kind_sign"), |
||||
models.UniqueConstraint(fields=["account"], condition=Q(kind="OPENING"), name="transaction_one_opening"), |
||||
models.CheckConstraint(condition=(~Q(kind="OPENING") | Q(category__isnull=True, payee__isnull=True, statement__isnull=True, refund_of__isnull=True, installment_plan__isnull=True, installment_number__isnull=True)), name="transaction_opening_unassociated"), |
||||
models.CheckConstraint(condition=(~Q(kind="TRANSFER") | Q(category__isnull=True, refund_of__isnull=True, installment_plan__isnull=True, installment_number__isnull=True)), name="transaction_transfer_unclassified"), |
||||
models.CheckConstraint(condition=Q(refund_of__isnull=True) | Q(kind="REFUND"), name="transaction_refund_link_kind"), |
||||
models.CheckConstraint(condition=Q(refund_of__isnull=True) | ~Q(refund_of=F("id")), name="transaction_refund_not_self"), |
||||
models.CheckConstraint(condition=(Q(installment_plan__isnull=True, installment_number__isnull=True) | Q(installment_plan__isnull=False, installment_number__isnull=False)), name="transaction_installment_pair"), |
||||
models.CheckConstraint(condition=Q(installment_number__isnull=True) | Q(installment_number__gt=0), name="transaction_installment_positive"), |
||||
models.CheckConstraint(condition=Q(installment_plan__isnull=True) | Q(kind="EXPENSE"), name="transaction_installment_expense"), |
||||
models.UniqueConstraint(fields=["installment_plan", "installment_number"], condition=Q(installment_plan__isnull=False), name="transaction_installment_unique"), |
||||
] |
||||
|
||||
|
||||
class Transfer(TimestampedModel): |
||||
outgoing = models.OneToOneField(Transaction, on_delete=models.PROTECT, related_name="outgoing_transfer") |
||||
incoming = models.OneToOneField(Transaction, on_delete=models.PROTECT, related_name="incoming_transfer") |
||||
|
||||
class Meta: |
||||
constraints = [models.CheckConstraint(condition=~Q(outgoing=F("incoming")), name="transfer_distinct_legs")] |
||||
@ -0,0 +1,54 @@
|
||||
from django.core.validators import RegexValidator |
||||
from django.db import models |
||||
from django.db.models import Q |
||||
|
||||
from .base import IdentifiedModel |
||||
|
||||
|
||||
class SourceRecord(IdentifiedModel): |
||||
class Format(models.TextChoices): |
||||
BANK_CSV = "BANK_CSV", "Bank CSV" |
||||
OUROCARD_TXT = "OUROCARD_TXT", "Ourocard TXT" |
||||
|
||||
transaction = models.ForeignKey("finance.Transaction", on_delete=models.PROTECT, related_name="source_records") |
||||
document_hash = models.CharField(max_length=64, validators=[ |
||||
RegexValidator(r"\A[0-9a-f]{64}\Z", "Use a lowercase SHA-256 hexadecimal digest.")]) |
||||
source_format = models.CharField(max_length=16, choices=Format.choices) |
||||
parser_version = models.CharField(max_length=128) |
||||
item_locator = models.CharField(max_length=255) |
||||
|
||||
class Meta: |
||||
constraints = [ |
||||
models.CheckConstraint(condition=Q(source_format__in=["BANK_CSV", "OUROCARD_TXT"]), name="source_valid_format"), |
||||
models.CheckConstraint(condition=Q(document_hash__regex=r"^[0-9a-f]{64}$"), name="source_valid_hash"), |
||||
models.CheckConstraint(condition=Q(parser_version__regex=r"\S"), name="source_nonblank_parser"), |
||||
models.CheckConstraint(condition=Q(item_locator__regex=r"\S"), name="source_nonblank_locator"), |
||||
models.UniqueConstraint(fields=["document_hash", "item_locator"], name="source_document_item_unique"), |
||||
] |
||||
|
||||
|
||||
class AuditEvent(IdentifiedModel): |
||||
class ActorKind(models.TextChoices): |
||||
HUMAN = "HUMAN", "Human" |
||||
APPLICATION = "APPLICATION", "Application" |
||||
AGENT = "AGENT", "Agent" |
||||
|
||||
class Outcome(models.TextChoices): |
||||
SUCCESS = "SUCCESS", "Success" |
||||
FAILURE = "FAILURE", "Failure" |
||||
|
||||
actor_kind = models.CharField(max_length=16, choices=ActorKind.choices) |
||||
actor_identifier = models.CharField(max_length=255) |
||||
operation = models.CharField(max_length=128) |
||||
target_id = models.UUIDField(null=True, blank=True) |
||||
outcome = models.CharField(max_length=16, choices=Outcome.choices) |
||||
changed_fields = models.JSONField(default=list, blank=True) |
||||
error_code = models.CharField(max_length=128, blank=True) |
||||
|
||||
class Meta: |
||||
constraints = [ |
||||
models.CheckConstraint(condition=Q(actor_kind__in=["HUMAN", "APPLICATION", "AGENT"]), name="audit_valid_actor"), |
||||
models.CheckConstraint(condition=Q(outcome__in=["SUCCESS", "FAILURE"]), name="audit_valid_outcome"), |
||||
models.CheckConstraint(condition=Q(actor_identifier__regex=r"\S"), name="audit_nonblank_actor"), |
||||
models.CheckConstraint(condition=Q(operation__regex=r"\S"), name="audit_nonblank_operation"), |
||||
] |
||||
@ -0,0 +1,48 @@
|
||||
from django.core.validators import RegexValidator |
||||
from django.db import models |
||||
from django.db.models import Q |
||||
|
||||
from .base import TimestampedModel |
||||
|
||||
|
||||
class Account(TimestampedModel): |
||||
class Kind(models.TextChoices): |
||||
BANK = "BANK", "Bank" |
||||
CREDIT_CARD = "CREDIT_CARD", "Credit card" |
||||
|
||||
name = models.CharField(max_length=255, unique=True) |
||||
kind = models.CharField(max_length=16, choices=Kind.choices) |
||||
currency = models.CharField(max_length=3, default="BRL", validators=[ |
||||
RegexValidator(r"\A[A-Z]{3}\Z", "Use three uppercase ASCII letters.")]) |
||||
archived = models.BooleanField(default=False) |
||||
|
||||
class Meta: |
||||
constraints = [ |
||||
models.CheckConstraint(condition=Q(kind__in=["BANK", "CREDIT_CARD"]), name="account_valid_kind"), |
||||
models.CheckConstraint(condition=Q(currency__regex=r"^[A-Z]{3}$"), name="account_valid_currency"), |
||||
models.CheckConstraint(condition=Q(name__regex=r"\S"), name="account_nonblank_name"), |
||||
] |
||||
|
||||
|
||||
class Payee(TimestampedModel): |
||||
name = models.CharField(max_length=255, unique=True) |
||||
archived = models.BooleanField(default=False) |
||||
|
||||
class Meta: |
||||
constraints = [models.CheckConstraint(condition=Q(name__regex=r"\S"), name="payee_nonblank_name")] |
||||
|
||||
|
||||
class Category(TimestampedModel): |
||||
class Kind(models.TextChoices): |
||||
INCOME = "INCOME", "Income" |
||||
EXPENSE = "EXPENSE", "Expense" |
||||
|
||||
name = models.CharField(max_length=255, unique=True) |
||||
kind = models.CharField(max_length=16, choices=Kind.choices) |
||||
archived = models.BooleanField(default=False) |
||||
|
||||
class Meta: |
||||
constraints = [ |
||||
models.CheckConstraint(condition=Q(kind__in=["INCOME", "EXPENSE"]), name="category_valid_kind"), |
||||
models.CheckConstraint(condition=Q(name__regex=r"\S"), name="category_nonblank_name"), |
||||
] |
||||
@ -0,0 +1,148 @@
|
||||
from dataclasses import dataclass |
||||
from datetime import date |
||||
import re |
||||
from typing import Mapping |
||||
from uuid import UUID |
||||
|
||||
from django.db.models import QuerySet |
||||
|
||||
from finance.models import Account, Category, Payee, Transaction |
||||
|
||||
|
||||
_CURRENCY_RE = re.compile(r"[A-Z]{3}", re.ASCII) |
||||
|
||||
|
||||
class QueryInputError(ValueError): |
||||
def __init__(self, errors: dict[str, str]): |
||||
super().__init__("Invalid transaction filters") |
||||
self.errors = errors |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class TransactionFilters: |
||||
date_from: date | None = None |
||||
date_to: date | None = None |
||||
account_id: UUID | None = None |
||||
kind: str | None = None |
||||
payee_id: UUID | None = None |
||||
category_id: UUID | None = None |
||||
currency: str | None = None |
||||
uncategorized: bool | None = None |
||||
|
||||
@classmethod |
||||
def from_mapping(cls, mapping: Mapping[str, object]) -> "TransactionFilters": |
||||
errors: dict[str, str] = {} |
||||
date_from = _parse_date(mapping.get("date_from"), "date_from", errors) |
||||
date_to = _parse_date(mapping.get("date_to"), "date_to", errors) |
||||
account_id = _parse_uuid(mapping.get("account"), "account", errors) |
||||
payee_id = _parse_uuid(mapping.get("payee"), "payee", errors) |
||||
category_id = _parse_uuid(mapping.get("category"), "category", errors) |
||||
|
||||
kind = _optional_string(mapping.get("kind")) |
||||
if kind is not None and kind not in Transaction.Kind.values: |
||||
errors["kind"] = "invalid_choice" |
||||
|
||||
currency = _optional_string(mapping.get("currency")) |
||||
if currency is not None and _CURRENCY_RE.fullmatch(currency) is None: |
||||
errors["currency"] = "invalid_currency" |
||||
|
||||
uncategorized_value = _optional_string(mapping.get("uncategorized")) |
||||
uncategorized = None |
||||
if uncategorized_value == "yes": |
||||
uncategorized = True |
||||
elif uncategorized_value == "no": |
||||
uncategorized = False |
||||
elif uncategorized_value is not None: |
||||
errors["uncategorized"] = "invalid_choice" |
||||
|
||||
if date_from is not None and date_to is not None and date_to < date_from: |
||||
errors["date_to"] = "before_date_from" |
||||
|
||||
if errors: |
||||
raise QueryInputError(errors) |
||||
return cls( |
||||
date_from=date_from, |
||||
date_to=date_to, |
||||
account_id=account_id, |
||||
kind=kind, |
||||
payee_id=payee_id, |
||||
category_id=category_id, |
||||
currency=currency, |
||||
uncategorized=uncategorized, |
||||
) |
||||
|
||||
|
||||
def filtered_transactions(filters: TransactionFilters) -> QuerySet[Transaction]: |
||||
rows = Transaction.objects.select_related( |
||||
"account", "payee", "category", "statement", "refund_of", "installment_plan" |
||||
) |
||||
if filters.date_from is not None: |
||||
rows = rows.filter(posting_date__gte=filters.date_from) |
||||
if filters.date_to is not None: |
||||
rows = rows.filter(posting_date__lte=filters.date_to) |
||||
if filters.account_id is not None: |
||||
rows = rows.filter(account_id=filters.account_id) |
||||
if filters.kind is not None: |
||||
rows = rows.filter(kind=filters.kind) |
||||
if filters.payee_id is not None: |
||||
rows = rows.filter(payee_id=filters.payee_id) |
||||
if filters.category_id is not None: |
||||
rows = rows.filter(category_id=filters.category_id) |
||||
if filters.currency is not None: |
||||
rows = rows.filter(account__currency=filters.currency) |
||||
if filters.uncategorized is not None: |
||||
rows = rows.filter(category__isnull=filters.uncategorized) |
||||
return rows.order_by("-posting_date", "-id") |
||||
|
||||
|
||||
def listed_accounts() -> QuerySet[Account]: |
||||
return Account.objects.order_by("archived", "name", "id") |
||||
|
||||
|
||||
def listed_categories() -> QuerySet[Category]: |
||||
return Category.objects.order_by("archived", "name", "id") |
||||
|
||||
|
||||
def listed_payees() -> QuerySet[Payee]: |
||||
return Payee.objects.order_by("archived", "name", "id") |
||||
|
||||
|
||||
def active_accounts() -> QuerySet[Account]: |
||||
return Account.objects.filter(archived=False).order_by("name", "id") |
||||
|
||||
|
||||
def active_categories() -> QuerySet[Category]: |
||||
return Category.objects.filter(archived=False).order_by("name", "id") |
||||
|
||||
|
||||
def active_payees() -> QuerySet[Payee]: |
||||
return Payee.objects.filter(archived=False).order_by("name", "id") |
||||
|
||||
|
||||
def _optional_string(value: object) -> str | None: |
||||
if value is None: |
||||
return None |
||||
text = str(value).strip() |
||||
return text or None |
||||
|
||||
|
||||
def _parse_date(value: object, field: str, errors: dict[str, str]) -> date | None: |
||||
text = _optional_string(value) |
||||
if text is None: |
||||
return None |
||||
try: |
||||
return date.fromisoformat(text) |
||||
except ValueError: |
||||
errors[field] = "invalid_date" |
||||
return None |
||||
|
||||
|
||||
def _parse_uuid(value: object, field: str, errors: dict[str, str]) -> UUID | None: |
||||
text = _optional_string(value) |
||||
if text is None: |
||||
return None |
||||
try: |
||||
return UUID(text) |
||||
except (ValueError, AttributeError): |
||||
errors[field] = "invalid_uuid" |
||||
return None |
||||
@ -0,0 +1,161 @@
|
||||
from dataclasses import dataclass |
||||
from datetime import date |
||||
from decimal import Decimal |
||||
import re |
||||
from uuid import UUID |
||||
|
||||
from django.db.models import DecimalField, F, Q, Sum, Value |
||||
from django.db.models.functions import Coalesce |
||||
|
||||
from finance.models import Account, Transaction |
||||
|
||||
|
||||
_CURRENCY_RE = re.compile(r"[A-Z]{3}", re.ASCII) |
||||
_MONEY_FIELD = DecimalField(max_digits=24, decimal_places=2) |
||||
_ZERO = Decimal("0.00") |
||||
|
||||
|
||||
class ReportInputError(ValueError): |
||||
def __init__(self, errors: dict[str, str]): |
||||
super().__init__("Invalid report inputs") |
||||
self.errors = errors |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class MonthlySummary: |
||||
month: date |
||||
currency: str |
||||
inflow: Decimal |
||||
expense: Decimal |
||||
refund: Decimal |
||||
net_spending: Decimal |
||||
net_activity: Decimal |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class SpendingCategory: |
||||
category_id: UUID | None |
||||
category_name: str |
||||
expense: Decimal |
||||
refund: Decimal |
||||
net_spending: Decimal |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class AccountBalance: |
||||
account_id: UUID |
||||
account_name: str |
||||
balance: Decimal |
||||
|
||||
|
||||
def available_currencies() -> tuple[str, ...]: |
||||
return tuple( |
||||
Account.objects.order_by("currency") |
||||
.values_list("currency", flat=True) |
||||
.distinct() |
||||
) |
||||
|
||||
|
||||
def monthly_summary(month: date, currency: str) -> MonthlySummary: |
||||
month, currency, month_after = _validated_period(month, currency) |
||||
totals = _monthly_rows(month, month_after, currency).aggregate( |
||||
inflow=_sum("amount", Q(kind=Transaction.Kind.INCOME)), |
||||
expense=_sum(-F("amount"), Q(kind=Transaction.Kind.EXPENSE)), |
||||
refund=_sum("amount", Q(kind=Transaction.Kind.REFUND)), |
||||
) |
||||
inflow = totals["inflow"] |
||||
expense = totals["expense"] |
||||
refund = totals["refund"] |
||||
net_spending = expense - refund |
||||
return MonthlySummary( |
||||
month=month, |
||||
currency=currency, |
||||
inflow=inflow, |
||||
expense=expense, |
||||
refund=refund, |
||||
net_spending=net_spending, |
||||
net_activity=inflow - net_spending, |
||||
) |
||||
|
||||
|
||||
def spending_by_category(month: date, currency: str) -> tuple[SpendingCategory, ...]: |
||||
month, currency, month_after = _validated_period(month, currency) |
||||
grouped = ( |
||||
_monthly_rows(month, month_after, currency) |
||||
.filter(kind__in=[Transaction.Kind.EXPENSE, Transaction.Kind.REFUND]) |
||||
.values("category_id", "category__name") |
||||
.annotate( |
||||
expense=_sum(-F("amount"), Q(kind=Transaction.Kind.EXPENSE)), |
||||
refund=_sum("amount", Q(kind=Transaction.Kind.REFUND)), |
||||
) |
||||
) |
||||
rows = tuple( |
||||
SpendingCategory( |
||||
category_id=row["category_id"], |
||||
category_name=row["category__name"] or "Uncategorized", |
||||
expense=row["expense"], |
||||
refund=row["refund"], |
||||
net_spending=row["expense"] - row["refund"], |
||||
) |
||||
for row in grouped |
||||
) |
||||
return tuple(sorted(rows, key=lambda row: (-row.net_spending, row.category_name))) |
||||
|
||||
|
||||
def account_balances(month: date, currency: str) -> tuple[AccountBalance, ...]: |
||||
month, currency, month_after = _validated_period(month, currency) |
||||
del month |
||||
rows = ( |
||||
Account.objects.filter(currency=currency) |
||||
.annotate( |
||||
report_balance=Coalesce( |
||||
Sum("transactions__amount", filter=Q(transactions__posting_date__lt=month_after)), |
||||
Value(_ZERO), |
||||
output_field=_MONEY_FIELD, |
||||
) |
||||
) |
||||
.order_by("name", "id") |
||||
) |
||||
return tuple( |
||||
AccountBalance( |
||||
account_id=row.pk, |
||||
account_name=row.name, |
||||
balance=row.report_balance, |
||||
) |
||||
for row in rows |
||||
) |
||||
|
||||
|
||||
def _monthly_rows(month: date, month_after: date, currency: str): |
||||
return Transaction.objects.filter( |
||||
posting_date__gte=month, |
||||
posting_date__lt=month_after, |
||||
account__currency=currency, |
||||
) |
||||
|
||||
|
||||
def _sum(expression, condition: Q): |
||||
return Coalesce( |
||||
Sum(expression, filter=condition), |
||||
Value(_ZERO), |
||||
output_field=_MONEY_FIELD, |
||||
) |
||||
|
||||
|
||||
def _validated_period(month: date, currency: str) -> tuple[date, str, date]: |
||||
errors: dict[str, str] = {} |
||||
if not isinstance(month, date): |
||||
errors["month"] = "invalid_date" |
||||
elif month.day != 1: |
||||
errors["month"] = "must_be_first_day" |
||||
elif month.year == date.max.year and month.month == 12: |
||||
errors["month"] = "invalid_month" |
||||
if not isinstance(currency, str) or _CURRENCY_RE.fullmatch(currency) is None: |
||||
errors["currency"] = "invalid_currency" |
||||
if errors: |
||||
raise ReportInputError(errors) |
||||
if month.month == 12: |
||||
month_after = date(month.year + 1, 1, 1) |
||||
else: |
||||
month_after = date(month.year, month.month + 1, 1) |
||||
return month, currency, month_after |
||||
@ -0,0 +1,7 @@
|
||||
"""Supported financial writes. Direct ORM mutation bypasses cross-row invariants.""" |
||||
from .base import Actor, DomainError |
||||
from .reference import create_account, create_payee, create_category, archive, create_statement, create_plan |
||||
from .posting import record_transaction, annotate_transaction, attach_source |
||||
from .transfers import create_transfer, match_transfer |
||||
|
||||
__all__ = ["Actor", "DomainError", "create_account", "create_payee", "create_category", "archive", "create_statement", "create_plan", "record_transaction", "annotate_transaction", "attach_source", "create_transfer", "match_transfer"] |
||||
@ -0,0 +1,89 @@
|
||||
"""Public writes own a transaction; internal helpers never open public operations.""" |
||||
from dataclasses import dataclass |
||||
from decimal import Decimal, InvalidOperation |
||||
from functools import wraps |
||||
from uuid import UUID |
||||
|
||||
from django.core.exceptions import ValidationError |
||||
from django.db import IntegrityError, connection, transaction |
||||
|
||||
from finance.models import AuditEvent |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class Actor: |
||||
kind: str |
||||
identifier: str |
||||
|
||||
|
||||
class DomainError(ValueError): |
||||
def __init__(self, code): |
||||
self.code = code |
||||
super().__init__(code) |
||||
|
||||
|
||||
def money(value, *, allow_zero=False): |
||||
if not isinstance(value, (Decimal, str, int)) or isinstance(value, bool): |
||||
raise DomainError("invalid_amount") |
||||
try: |
||||
result = Decimal(value) |
||||
except (InvalidOperation, ValueError): |
||||
raise DomainError("invalid_amount") from None |
||||
if not result.is_finite() or result.as_tuple().exponent < -2 or result.adjusted() >= 16: |
||||
raise DomainError("invalid_amount") |
||||
if not allow_zero and result == 0: |
||||
raise DomainError("zero_amount") |
||||
return result |
||||
|
||||
|
||||
def identity(value): |
||||
try: |
||||
return UUID(str(value)) |
||||
except (ValueError, TypeError, AttributeError): |
||||
raise DomainError("invalid_identifier") from None |
||||
|
||||
|
||||
def lock(model, identifier, *, active=False): |
||||
try: |
||||
obj = model.objects.select_for_update().get(pk=identity(identifier)) |
||||
except model.DoesNotExist: |
||||
raise DomainError("not_found") from None |
||||
if active and obj.archived: |
||||
raise DomainError("archived_reference") |
||||
return obj |
||||
|
||||
|
||||
def save(obj): |
||||
obj.full_clean() |
||||
obj.save() |
||||
return obj |
||||
|
||||
|
||||
def audit(actor, operation, outcome, target_id=None, fields=(), error_code=""): |
||||
AuditEvent.objects.create(actor_kind=actor.kind, actor_identifier=actor.identifier, |
||||
operation=operation, target_id=target_id, outcome=outcome, |
||||
changed_fields=list(fields), error_code=error_code) |
||||
|
||||
|
||||
def write(operation): |
||||
def decorate(function): |
||||
@wraps(function) |
||||
def wrapped(*args, actor, **kwargs): |
||||
if not isinstance(actor, Actor) or actor.kind not in {"HUMAN", "APPLICATION", "AGENT"} or not isinstance(actor.identifier, str) or not actor.identifier.strip() or len(actor.identifier) > 200: |
||||
raise DomainError("invalid_actor") |
||||
if connection.in_atomic_block: |
||||
raise DomainError("transaction_boundary") |
||||
try: |
||||
with transaction.atomic(durable=True): |
||||
result, fields = function(*args, **kwargs) |
||||
audit(actor, operation, "SUCCESS", result.pk, fields) |
||||
return result |
||||
except (DomainError, ValidationError, IntegrityError) as exc: |
||||
code = exc.code if isinstance(exc, DomainError) else ("conflict" if isinstance(exc, IntegrityError) else "validation_error") |
||||
# The failed mutation transaction has exited before this audit. |
||||
# Audit persistence errors propagate: do not conceal a database outage. |
||||
with transaction.atomic(durable=True): |
||||
audit(actor, operation, "FAILURE", error_code=code) |
||||
raise DomainError(code) from None |
||||
return wrapped |
||||
return decorate |
||||
@ -0,0 +1,75 @@
|
||||
from finance.models import Account, Payee, Category, CardStatement, InstallmentPlan, Transaction, SourceRecord |
||||
from .base import DomainError, identity, lock, money, save, write |
||||
|
||||
|
||||
def validate_category(kind, category): |
||||
if category is None: |
||||
return |
||||
expected = {"INCOME": "INCOME", "EXPENSE": "EXPENSE", "REFUND": "EXPENSE"}.get(kind) |
||||
if expected != category.kind: |
||||
raise DomainError("category_kind_mismatch") |
||||
|
||||
|
||||
def record(*, account_id, amount, posting_date, kind, description="", occurrence_date=None, |
||||
payee_id=None, category_id=None, statement_id=None, refund_of_id=None, |
||||
installment_plan_id=None, installment_number=None): |
||||
amount = money(amount) |
||||
account = lock(Account, account_id, active=True) |
||||
payee = lock(Payee, payee_id, active=True) if payee_id is not None else None |
||||
category = lock(Category, category_id, active=True) if category_id is not None else None |
||||
validate_category(kind, category) |
||||
statement = lock(CardStatement, statement_id) if statement_id is not None else None |
||||
if statement and (account.kind != "CREDIT_CARD" or statement.account_id != account.pk): |
||||
raise DomainError("statement_account_mismatch") |
||||
plan = lock(InstallmentPlan, installment_plan_id) if installment_plan_id is not None else None |
||||
if plan: |
||||
if kind != "EXPENSE" or plan.account_id != account.pk: |
||||
raise DomainError("installment_account_or_kind_mismatch") |
||||
if not isinstance(installment_number, int) or isinstance(installment_number, bool) or not 1 <= installment_number <= plan.total_installments: |
||||
raise DomainError("invalid_installment_number") |
||||
elif installment_number is not None: |
||||
raise DomainError("missing_installment_plan") |
||||
original = lock(Transaction, refund_of_id) if refund_of_id is not None else None |
||||
if original: |
||||
if kind != "REFUND" or original.kind != "EXPENSE" or original.account.currency != account.currency: |
||||
raise DomainError("invalid_refund_reference") |
||||
obj = Transaction(account=account, amount=amount, posting_date=posting_date, kind=kind, |
||||
description=description, occurrence_date=occurrence_date, payee=payee, |
||||
category=category, statement=statement, refund_of=original, |
||||
installment_plan=plan, installment_number=installment_number) |
||||
return save(obj) |
||||
|
||||
|
||||
@write("record_transaction") |
||||
def record_transaction(**kwargs): |
||||
return record(**kwargs), sorted(key.removesuffix("_id") for key in kwargs) |
||||
|
||||
|
||||
@write("annotate_transaction") |
||||
def annotate_transaction(*, transaction_id, **changes): |
||||
if not changes or set(changes) - {"description", "payee_id", "category_id"}: |
||||
raise DomainError("immutable_transaction") |
||||
# Reference rows precede transaction rows in every write path. This avoids |
||||
# a cycle with refund posting (category -> original transaction). |
||||
references = {} |
||||
for field, model in (("payee_id", Payee), ("category_id", Category)): |
||||
if field in changes: |
||||
value = changes[field] |
||||
references[field] = lock(model, value, active=True) if value is not None else None |
||||
obj = lock(Transaction, transaction_id) |
||||
if "description" in changes: |
||||
obj.description = changes["description"] |
||||
if "payee_id" in changes: |
||||
obj.payee = references["payee_id"] |
||||
if "category_id" in changes: |
||||
obj.category = references["category_id"] |
||||
validate_category(obj.kind, obj.category) |
||||
return save(obj), sorted(key.removesuffix("_id") for key in changes) |
||||
|
||||
|
||||
@write("attach_source") |
||||
def attach_source(*, transaction_id, document_hash, source_format, parser_version, item_locator): |
||||
obj = lock(Transaction, transaction_id) |
||||
source = save(SourceRecord(transaction=obj, document_hash=document_hash, source_format=source_format, |
||||
parser_version=parser_version, item_locator=item_locator)) |
||||
return source, ["transaction", "document_hash", "source_format", "parser_version", "item_locator"] |
||||
@ -0,0 +1,53 @@
|
||||
from finance.models import Account, Payee, Category, CardStatement, InstallmentPlan |
||||
from .base import DomainError, lock, money, save, write |
||||
|
||||
|
||||
def normalized_name(value): |
||||
if not isinstance(value, str) or not value.strip(): |
||||
raise DomainError("invalid_name") |
||||
return value.strip() |
||||
|
||||
|
||||
@write("create_account") |
||||
def create_account(*, name, kind, currency="BRL"): |
||||
obj = save(Account(name=normalized_name(name), kind=kind, currency=currency)) |
||||
return obj, ["name", "kind", "currency"] |
||||
|
||||
|
||||
@write("create_payee") |
||||
def create_payee(*, name): |
||||
return save(Payee(name=normalized_name(name))), ["name"] |
||||
|
||||
|
||||
@write("create_category") |
||||
def create_category(*, name, kind): |
||||
return save(Category(name=normalized_name(name), kind=kind)), ["name", "kind"] |
||||
|
||||
|
||||
@write("archive") |
||||
def archive(entity): |
||||
if type(entity) not in (Account, Payee, Category): |
||||
raise DomainError("invalid_reference") |
||||
obj = lock(type(entity), entity.pk) |
||||
obj.archived = True |
||||
return save(obj), ["archived"] |
||||
|
||||
|
||||
@write("create_statement") |
||||
def create_statement(*, account_id, period_start, period_end, due_date, statement_balance=None): |
||||
account = lock(Account, account_id, active=True) |
||||
if account.kind != "CREDIT_CARD": |
||||
raise DomainError("statement_requires_card") |
||||
balance = None if statement_balance is None else money(statement_balance, allow_zero=True) |
||||
obj = save(CardStatement(account=account, period_start=period_start, period_end=period_end, |
||||
due_date=due_date, statement_balance=balance)) |
||||
return obj, ["account", "period_start", "period_end", "due_date", "statement_balance"] |
||||
|
||||
|
||||
@write("create_plan") |
||||
def create_plan(*, account_id, description, total_installments): |
||||
account = lock(Account, account_id, active=True) |
||||
if not isinstance(total_installments, int) or isinstance(total_installments, bool) or total_installments < 1: |
||||
raise DomainError("invalid_installment_count") |
||||
obj = save(InstallmentPlan(account=account, description=description, total_installments=total_installments)) |
||||
return obj, ["account", "description", "total_installments"] |
||||
@ -0,0 +1,45 @@
|
||||
from finance.models import Account, Transaction, Transfer |
||||
from .base import DomainError, identity, lock, money, save, write |
||||
from .posting import record |
||||
|
||||
|
||||
def match(outgoing_id, incoming_id): |
||||
ids = [identity(outgoing_id), identity(incoming_id)] |
||||
if ids[0] == ids[1]: |
||||
raise DomainError("invalid_transfer_legs") |
||||
rows = {obj.pk: obj for obj in Transaction.objects.select_for_update().filter(pk__in=ids).order_by("pk")} |
||||
if len(rows) != 2: |
||||
raise DomainError("not_found") |
||||
outgoing, incoming = rows[ids[0]], rows[ids[1]] |
||||
if outgoing.kind != "TRANSFER" or incoming.kind != "TRANSFER" or outgoing.amount >= 0 or incoming.amount <= 0 or outgoing.amount != incoming.amount.copy_negate(): |
||||
raise DomainError("invalid_transfer_legs") |
||||
if outgoing.account_id == incoming.account_id or outgoing.account.currency != incoming.account.currency: |
||||
raise DomainError("transfer_account_or_currency_mismatch") |
||||
if Transfer.objects.filter(outgoing=outgoing).exists() or Transfer.objects.filter(incoming=incoming).exists(): |
||||
raise DomainError("already_paired") |
||||
return save(Transfer(outgoing=outgoing, incoming=incoming)) |
||||
|
||||
|
||||
@write("match_transfer") |
||||
def match_transfer(*, outgoing_id, incoming_id): |
||||
return match(outgoing_id, incoming_id), ["outgoing", "incoming"] |
||||
|
||||
|
||||
@write("create_transfer") |
||||
def create_transfer(*, outgoing_account_id, incoming_account_id, amount, outgoing_date, incoming_date): |
||||
amount = money(amount) |
||||
if amount <= 0: |
||||
raise DomainError("transfer_amount_must_be_positive") |
||||
ids = [identity(outgoing_account_id), identity(incoming_account_id)] |
||||
if ids[0] == ids[1]: |
||||
raise DomainError("transfer_account_or_currency_mismatch") |
||||
accounts = {obj.pk: obj for obj in Account.objects.select_for_update().filter(pk__in=ids).order_by("pk")} |
||||
if len(accounts) != 2: |
||||
raise DomainError("not_found") |
||||
if any(obj.archived for obj in accounts.values()): |
||||
raise DomainError("archived_reference") |
||||
if accounts[ids[0]].currency != accounts[ids[1]].currency: |
||||
raise DomainError("transfer_account_or_currency_mismatch") |
||||
outgoing = record(account_id=ids[0], amount=amount.copy_negate(), posting_date=outgoing_date, kind="TRANSFER") |
||||
incoming = record(account_id=ids[1], amount=amount, posting_date=incoming_date, kind="TRANSFER") |
||||
return match(outgoing.pk, incoming.pk), ["outgoing", "incoming"] |
||||
@ -0,0 +1,25 @@
|
||||
"""Compare snapshots without writing or discarding candidate movements.""" |
||||
from decimal import Decimal |
||||
import unicodedata |
||||
|
||||
|
||||
def signature(*, account_id, currency, posting_date, amount, description, kind, |
||||
installment_number=None, installment_total=None): |
||||
return (str(account_id), currency, str(posting_date), Decimal(amount), |
||||
" ".join(unicodedata.normalize("NFC", description).casefold().split()), |
||||
kind, installment_number, installment_total) |
||||
|
||||
|
||||
def candidate_signature(candidate, account, decision): |
||||
return signature(account_id=account.pk, currency=account.currency, |
||||
posting_date=candidate["posting_date"], amount=candidate["amount"], |
||||
description=candidate["description"], kind=decision.get("kind", candidate["kind"]), |
||||
installment_number=candidate.get("installment_number"), |
||||
installment_total=candidate.get("installment_total")) |
||||
|
||||
|
||||
def transaction_signature(row): |
||||
return signature(account_id=row.account_id, currency=row.account.currency, |
||||
posting_date=row.posting_date, amount=row.amount, description=row.description, |
||||
kind=row.kind, installment_number=row.installment_number, |
||||
installment_total=row.installment_plan.total_installments if row.installment_plan_id else None) |
||||
@ -0,0 +1,208 @@
|
||||
"""Local, explicit review workflow. Financial details never go to stdout.""" |
||||
import json |
||||
import re |
||||
from datetime import date |
||||
from pathlib import Path |
||||
from uuid import UUID |
||||
|
||||
from django.conf import settings |
||||
from django.core.management.base import BaseCommand, CommandError |
||||
|
||||
from finance.services import Actor, DomainError |
||||
from ingestion import services |
||||
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024 |
||||
PROFILES = ("BANK_CSV_V1", "OUROCARD_FULL_V1", "OUROCARD_PARTIAL_V1") |
||||
|
||||
|
||||
def local_path(value): |
||||
root = Path(getattr(settings, "IMPORT_ROOT", "/app/imports")).resolve() |
||||
requested = Path(value) |
||||
if ".." in requested.parts: |
||||
raise CommandError("INVALID_PATH") |
||||
resolved = (root / requested).resolve() |
||||
if not resolved.is_relative_to(root) or resolved == root: |
||||
raise CommandError("INVALID_PATH") |
||||
return resolved |
||||
|
||||
|
||||
def read_bytes(value): |
||||
path = local_path(value) |
||||
if not path.is_file(): |
||||
raise CommandError("INVALID_INPUT") |
||||
with path.open("rb") as stream: |
||||
data = stream.read(MAX_BYTES + 1) |
||||
if len(data) > MAX_BYTES: |
||||
raise CommandError("INPUT_TOO_LARGE") |
||||
return data |
||||
|
||||
|
||||
def unique_object(pairs): |
||||
result = {} |
||||
for key, value in pairs: |
||||
if key in result: |
||||
raise ValueError("duplicate key") |
||||
result[key] = value |
||||
return result |
||||
|
||||
|
||||
def reject_constant(value): |
||||
raise ValueError("nonfinite JSON number") |
||||
|
||||
|
||||
def read_json(value): |
||||
data = read_bytes(value) |
||||
try: |
||||
return json.loads(data.decode("utf-8-sig"), object_pairs_hook=unique_object, |
||||
parse_constant=reject_constant) |
||||
except (ValueError, RecursionError): |
||||
raise CommandError("INVALID_JSON") from None |
||||
|
||||
|
||||
def identifier(value): |
||||
try: |
||||
return str(UUID(str(value))) |
||||
except (ValueError, AttributeError, TypeError): |
||||
raise CommandError("INVALID_IDENTIFIER") from None |
||||
|
||||
|
||||
def revision_number(value): |
||||
if type(value) is not int or value < 1: |
||||
raise CommandError("INVALID_REVISION") |
||||
return value |
||||
|
||||
|
||||
def checked_decisions(payload, batch_id, actor): |
||||
minimum = {"batch_id", "revision", "decisions"} |
||||
if not isinstance(payload, dict) or not minimum.issubset(payload): |
||||
raise CommandError("INVALID_REVIEW") |
||||
if identifier(payload["batch_id"]) != batch_id: |
||||
raise CommandError("INVALID_REVIEW") |
||||
revision_number(payload["revision"]) |
||||
if not isinstance(payload["decisions"], list): |
||||
raise CommandError("INVALID_REVIEW") |
||||
locators = set() |
||||
for decision in payload["decisions"]: |
||||
if not isinstance(decision, dict) or not isinstance(decision.get("locator"), str): |
||||
raise CommandError("INVALID_REVIEW") |
||||
locator = decision["locator"] |
||||
if locator in locators: |
||||
raise CommandError("INVALID_REVIEW") |
||||
locators.add(locator) |
||||
if set(payload) != minimum: |
||||
current = services.export_review(batch_id=batch_id) |
||||
if set(payload) != set(current): |
||||
raise CommandError("INVALID_REVIEW") |
||||
# All exported fields except the dedicated decisions array are evidence. |
||||
# JSON encoding also distinguishes true/1 and numeric/string edits. |
||||
for key in current: |
||||
if key != "decisions" and json.dumps(payload[key], sort_keys=True) != json.dumps(current[key], sort_keys=True): |
||||
raise CommandError("READ_ONLY_REVIEW_CHANGED") |
||||
return payload["revision"], payload["decisions"] |
||||
|
||||
|
||||
class Command(BaseCommand): |
||||
help = "Preview, export, apply, refresh, commit or cancel a local reviewed import." |
||||
requires_system_checks = [] |
||||
|
||||
def add_arguments(self, parser): |
||||
commands = parser.add_subparsers(dest="operation", required=True) |
||||
for operation in ("preview", "export", "apply", "refresh", "commit", "cancel"): |
||||
sub = commands.add_parser(operation) |
||||
sub.add_argument("--actor-kind", required=True, choices=("HUMAN", "APPLICATION", "AGENT")) |
||||
sub.add_argument("--actor-identifier", required=True) |
||||
if operation == "preview": |
||||
sub.add_argument("--input", required=True) |
||||
sub.add_argument("--mapping", required=True) |
||||
sub.add_argument("--profile", required=True, choices=PROFILES) |
||||
sub.add_argument("--context-start") |
||||
sub.add_argument("--context-end") |
||||
else: |
||||
sub.add_argument("--batch", required=True) |
||||
if operation == "export": |
||||
sub.add_argument("--output", required=True) |
||||
if operation == "apply": |
||||
sub.add_argument("--input", required=True) |
||||
if operation in {"refresh", "commit", "cancel"}: |
||||
sub.add_argument("--revision", required=True, type=int) |
||||
|
||||
def handle(self, *args, **options): |
||||
try: |
||||
self.perform(options) |
||||
except CommandError: |
||||
raise |
||||
except DomainError as exc: |
||||
code = exc.code |
||||
if not isinstance(code, str) or re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,79}", code) is None: |
||||
code = "IMPORT_FAILED" |
||||
raise CommandError(code) from None |
||||
except Exception: |
||||
# Source paths, source text and database errors are local financial data. |
||||
raise CommandError("IMPORT_FAILED") from None |
||||
|
||||
def perform(self, options): |
||||
actor = Actor(options["actor_kind"], options["actor_identifier"]) |
||||
if actor.kind not in {"HUMAN", "APPLICATION", "AGENT"} or not actor.identifier.strip() or len(actor.identifier) > 200: |
||||
raise CommandError("INVALID_ACTOR") |
||||
operation = options["operation"] |
||||
if operation == "preview": |
||||
mapping = read_json(options["mapping"]) |
||||
if not isinstance(mapping, dict) or not mapping: |
||||
raise CommandError("INVALID_MAPPING") |
||||
for section, account in mapping.items(): |
||||
if re.fullmatch(r"[1-9][0-9]*", section) is None: |
||||
raise CommandError("INVALID_MAPPING") |
||||
if account is not None: |
||||
if not isinstance(account, str): |
||||
raise CommandError("INVALID_MAPPING") |
||||
identifier(account) |
||||
start, end = options.get("context_start"), options.get("context_end") |
||||
context = None |
||||
if start or end: |
||||
try: |
||||
first, last = date.fromisoformat(start), date.fromisoformat(end) |
||||
if not 0 <= (last - first).days <= 365: |
||||
raise ValueError |
||||
except (TypeError, ValueError): |
||||
raise CommandError("INVALID_CONTEXT") from None |
||||
context = {"start_date": first.isoformat(), "end_date": last.isoformat()} |
||||
batch = services.create_preview( |
||||
data=read_bytes(options["input"]), |
||||
profile=options["profile"], |
||||
section_mapping=mapping, |
||||
context=context, |
||||
actor=actor, |
||||
) |
||||
else: |
||||
batch_id = identifier(options["batch"]) |
||||
if operation == "export": |
||||
path = local_path(options["output"]) |
||||
review = services.export_review(batch_id=batch_id) |
||||
encoded = json.dumps(review, ensure_ascii=False, indent=2, allow_nan=False) + "\n" |
||||
# Exclusive creation protects source documents and previous reviews. |
||||
with path.open("x", encoding="utf-8") as stream: |
||||
stream.write(encoded) |
||||
self.stdout.write(json.dumps({"batch_id": batch_id, "state": review["state"], |
||||
"revision": review["revision"], "item_count": len(review["items"])})) |
||||
return |
||||
if operation == "apply": |
||||
revision, decisions = checked_decisions(read_json(options["input"]), batch_id, actor) |
||||
batch = services.apply_decisions( |
||||
batch_id=batch_id, |
||||
revision=revision, |
||||
decisions=decisions, |
||||
actor=actor, |
||||
) |
||||
else: |
||||
revision = revision_number(options["revision"]) |
||||
action = {"refresh": services.refresh_preview, "commit": services.commit_preview, |
||||
"cancel": services.cancel_preview}[operation] |
||||
batch = action(batch_id=batch_id, revision=revision, actor=actor) |
||||
counts = getattr(batch, "result", {}) if batch.state == "COMMITTED" else getattr(batch, "summary", {}) |
||||
safe_counts = { |
||||
key: value for key, value in counts.items() |
||||
if key in {"rows", "error_rows", "decided_rows", "created", "matched", "skipped"} |
||||
} |
||||
self.stdout.write(json.dumps({"batch_id": str(batch.pk), "state": batch.state, |
||||
"revision": batch.revision, "counts": safe_counts})) |
||||
@ -0,0 +1,58 @@
|
||||
# Generated by Django 5.2.17 on 2026-09-07 20:58 |
||||
|
||||
import django.db.models.deletion |
||||
import uuid |
||||
from django.db import migrations, models |
||||
|
||||
|
||||
class Migration(migrations.Migration): |
||||
|
||||
initial = True |
||||
|
||||
dependencies = [ |
||||
] |
||||
|
||||
operations = [ |
||||
migrations.CreateModel( |
||||
name='ImportBatch', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('document_hash', models.CharField(max_length=64)), |
||||
('source_format', models.CharField(max_length=16)), |
||||
('profile', models.CharField(max_length=32)), |
||||
('parser_version', models.CharField(default='1', max_length=32)), |
||||
('encoding', models.CharField(max_length=32)), |
||||
('context', models.JSONField(default=dict)), |
||||
('section_mapping', models.JSONField(default=dict)), |
||||
('summary', models.JSONField(default=dict)), |
||||
('snapshot', models.JSONField(default=dict)), |
||||
('actor_kind', models.CharField(max_length=16)), |
||||
('actor_identifier', models.CharField(max_length=200)), |
||||
('state', models.CharField(default='PREVIEW', max_length=16)), |
||||
('revision', models.PositiveIntegerField(default=1)), |
||||
('result', models.JSONField(default=dict)), |
||||
], |
||||
options={ |
||||
'constraints': [models.CheckConstraint(condition=models.Q(('state__in', ['PREVIEW', 'COMMITTED', 'CANCELLED'])), name='import_batch_state'), models.CheckConstraint(condition=models.Q(('revision__gte', 1)), name='import_batch_revision')], |
||||
}, |
||||
), |
||||
migrations.CreateModel( |
||||
name='ImportItem', |
||||
fields=[ |
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), |
||||
('created_at', models.DateTimeField(auto_now_add=True)), |
||||
('updated_at', models.DateTimeField(auto_now=True)), |
||||
('locator', models.CharField(max_length=255)), |
||||
('candidate', models.JSONField()), |
||||
('issues', models.JSONField(default=list)), |
||||
('duplicates', models.JSONField(default=dict)), |
||||
('decision', models.JSONField(default=dict)), |
||||
('batch', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='items', to='ingestion.importbatch')), |
||||
], |
||||
options={ |
||||
'constraints': [models.UniqueConstraint(fields=('batch', 'locator'), name='import_item_locator_unique')], |
||||
}, |
||||
), |
||||
] |
||||
@ -0,0 +1,38 @@
|
||||
from django.db import models |
||||
from django.db.models import Q |
||||
from finance.models.base import TimestampedModel |
||||
|
||||
|
||||
class ImportBatch(TimestampedModel): |
||||
document_hash = models.CharField(max_length=64) |
||||
source_format = models.CharField(max_length=16) |
||||
profile = models.CharField(max_length=32) |
||||
parser_version = models.CharField(max_length=32, default="1") |
||||
encoding = models.CharField(max_length=32) |
||||
context = models.JSONField(default=dict) |
||||
section_mapping = models.JSONField(default=dict) |
||||
summary = models.JSONField(default=dict) |
||||
snapshot = models.JSONField(default=dict) |
||||
actor_kind = models.CharField(max_length=16) |
||||
actor_identifier = models.CharField(max_length=200) |
||||
state = models.CharField(max_length=16, default="PREVIEW") |
||||
revision = models.PositiveIntegerField(default=1) |
||||
result = models.JSONField(default=dict) |
||||
|
||||
class Meta: |
||||
constraints = [ |
||||
models.CheckConstraint(condition=Q(state__in=["PREVIEW", "COMMITTED", "CANCELLED"]), name="import_batch_state"), |
||||
models.CheckConstraint(condition=Q(revision__gte=1), name="import_batch_revision"), |
||||
] |
||||
|
||||
|
||||
class ImportItem(TimestampedModel): |
||||
batch = models.ForeignKey(ImportBatch, on_delete=models.PROTECT, related_name="items") |
||||
locator = models.CharField(max_length=255) |
||||
candidate = models.JSONField() |
||||
issues = models.JSONField(default=list) |
||||
duplicates = models.JSONField(default=dict) |
||||
decision = models.JSONField(default=dict) |
||||
|
||||
class Meta: |
||||
constraints = [models.UniqueConstraint(fields=["batch", "locator"], name="import_item_locator_unique")] |
||||
@ -0,0 +1,117 @@
|
||||
"""Convert parser evidence into immutable, serializable review candidates.""" |
||||
|
||||
import re |
||||
from dataclasses import dataclass, asdict |
||||
from datetime import date, datetime |
||||
from decimal import Decimal |
||||
|
||||
from .parsers import ParseResult |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class Candidate: |
||||
locator: str |
||||
section: str |
||||
posting_date: str | None |
||||
occurrence_date: str | None |
||||
description: str |
||||
amount: str | None |
||||
kind: str |
||||
installment_number: int | None |
||||
installment_total: int | None |
||||
errors: tuple[str, ...] |
||||
|
||||
def to_dict(self): |
||||
result = asdict(self) |
||||
result['errors'] = list(self.errors) |
||||
return result |
||||
|
||||
|
||||
def _date(text, profile, context): |
||||
if profile != 'OUROCARD_PARTIAL_V1': |
||||
pattern = r'\d{2}/\d{2}/\d{4}' if profile == 'BANK_CSV_V1' else r'\d{2}\.\d{2}\.\d{4}' |
||||
if not re.fullmatch(pattern, text): |
||||
raise ValueError('INVALID_DATE') |
||||
try: |
||||
return datetime.strptime(text, '%d/%m/%Y' if profile == 'BANK_CSV_V1' else '%d.%m.%Y').date().isoformat() |
||||
except ValueError: |
||||
raise ValueError('INVALID_DATE') from None |
||||
if not context.get('start_date') or not context.get('end_date'): |
||||
raise ValueError('MISSING_DATE_CONTEXT') |
||||
try: |
||||
start, end = date.fromisoformat(context['start_date']), date.fromisoformat(context['end_date']) |
||||
if not 0 <= (end - start).days <= 365: |
||||
raise ValueError |
||||
except (ValueError, TypeError): |
||||
raise ValueError('INVALID_DATE_CONTEXT') from None |
||||
if not re.fullmatch(r'\d{2}/\d{2}', text): |
||||
raise ValueError('INVALID_DATE') |
||||
day, month = map(int, text.split('/')) |
||||
possible = [] |
||||
for year in range(start.year, end.year + 1): |
||||
try: |
||||
candidate = date(year, month, day) |
||||
except ValueError: |
||||
continue |
||||
if start <= candidate <= end: |
||||
possible.append(candidate) |
||||
if len(possible) != 1: |
||||
raise ValueError('AMBIGUOUS_DATE' if possible else 'INVALID_DATE') |
||||
return possible[0].isoformat() |
||||
|
||||
|
||||
def _cents(text, localized=False): |
||||
pattern = r'[+-]?(?:\d+|\d{1,3}(?:\.\d{3})+),\d{2}' if localized else r'[+-]?\d+(?:\.\d{1,2})?' |
||||
if len(text) > 40 or not re.fullmatch(pattern, text): |
||||
raise ValueError('INVALID_AMOUNT') |
||||
clean = text.replace('.', '').replace(',', '.') if localized else text |
||||
# Integer minor units keep all arithmetic independent of Decimal context. |
||||
negative = clean.startswith('-') |
||||
clean = clean.lstrip('+-') |
||||
whole, _, fraction = clean.partition('.') |
||||
return (-1 if negative else 1) * (int(whole) * 100 + int(fraction.ljust(2, '0'))) |
||||
|
||||
|
||||
def _format_cents(value): |
||||
return ('-' if value < 0 else '') + f'{abs(value) // 100}.{abs(value) % 100:02d}' |
||||
|
||||
|
||||
def normalize(result: ParseResult) -> tuple[Candidate, ...]: |
||||
candidates = [] |
||||
for item in result.items: |
||||
errors = list(item.errors) |
||||
posting_date = amount = installment_number = installment_total = None |
||||
kind = 'EXPENSE' |
||||
if not item.errors: |
||||
try: |
||||
posting_date = _date(item.date_text, result.profile, result.context) |
||||
except ValueError as exc: |
||||
errors.append(str(exc)) |
||||
try: |
||||
if result.profile == 'BANK_CSV_V1': |
||||
cents = _cents(item.debit) |
||||
kind = 'INCOME' if cents > 0 else 'EXPENSE' |
||||
else: |
||||
debit, credit = _cents(item.debit, True), _cents(item.credit, True) |
||||
if item.payment: |
||||
if debit and credit: |
||||
errors.append('AMBIGUOUS_AMOUNT') |
||||
# copy_abs is exact even when caller Decimal context traps rounding. |
||||
cents = int(Decimal(debit).copy_abs()) |
||||
kind = 'TRANSFER' |
||||
else: |
||||
if debit and credit: |
||||
errors.append('AMBIGUOUS_AMOUNT') |
||||
cents = credit - debit |
||||
kind = 'REFUND' if cents > 0 else 'EXPENSE' |
||||
amount = _format_cents(cents) |
||||
except ValueError as exc: |
||||
errors.append(str(exc)) |
||||
if result.profile != 'BANK_CSV_V1' and re.search(r'\bPARC\b', item.description): |
||||
match = re.search(r'\bPARC\s+(\d{1,4})/(\d{1,4})\b', item.description) |
||||
if match and 1 <= int(match[1]) <= int(match[2]): |
||||
installment_number, installment_total = map(int, match.groups()) |
||||
else: |
||||
errors.append('INVALID_INSTALLMENT') |
||||
candidates.append(Candidate(item.locator, item.section, posting_date, posting_date, item.description, amount, kind, installment_number, installment_total, tuple(errors))) |
||||
return tuple(candidates) |
||||
@ -0,0 +1,136 @@
|
||||
"""Pure, versioned source grammars. No filesystem or database access.""" |
||||
|
||||
import csv |
||||
import io |
||||
import re |
||||
import unicodedata |
||||
from collections import Counter |
||||
from dataclasses import dataclass |
||||
from types import MappingProxyType |
||||
from typing import Mapping |
||||
|
||||
|
||||
PARSER_VERSION = '1' |
||||
PROFILES = frozenset({'BANK_CSV_V1', 'OUROCARD_FULL_V1', 'OUROCARD_PARTIAL_V1'}) |
||||
MAX_BYTES = 10 * 1024 * 1024 |
||||
MAX_ITEMS = 20000 |
||||
|
||||
|
||||
class ParseError(ValueError): |
||||
def __init__(self, code): |
||||
self.code = code |
||||
super().__init__(code) |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class ParsedItem: |
||||
locator: str |
||||
section: str |
||||
date_text: str = '' |
||||
description: str = '' |
||||
debit: str = '' |
||||
credit: str = '' |
||||
payment: bool = False |
||||
errors: tuple[str, ...] = () |
||||
|
||||
|
||||
@dataclass(frozen=True) |
||||
class ParseResult: |
||||
encoding: str |
||||
profile: str |
||||
items: tuple[ParsedItem, ...] |
||||
skipped: Mapping[str, int] |
||||
context: Mapping[str, str] |
||||
|
||||
|
||||
def _header_key(value): |
||||
return ''.join(c for c in unicodedata.normalize('NFD', value.strip().casefold()) if not unicodedata.combining(c)) |
||||
|
||||
|
||||
def _balance(value): |
||||
return bool(re.fullmatch(r'(?:S\s*A\s*L\s*D\s*O|SALDO ANTERIOR)(?:\s+[-\d.,]+)?', value.strip(), re.I)) |
||||
|
||||
|
||||
def _csv_items(text, skipped): |
||||
reader = csv.reader(io.StringIO(text, newline=''), strict=True) |
||||
try: |
||||
header = next(reader) |
||||
aliases = ({'data', 'data movimento', 'data do movimento', 'data lancamento'}, {'historico', 'descricao', 'descricao do lancamento'}, {'valor', 'valor lancamento', 'valor do lancamento'}) |
||||
if len(header) < 6 or any(_header_key(header[i]) not in allowed for i, allowed in zip((0, 2, 5), aliases)): |
||||
raise ParseError('INVALID_HEADER') |
||||
while True: |
||||
line = reader.line_num + 1 |
||||
try: |
||||
row = next(reader) |
||||
except StopIteration: |
||||
return |
||||
if not row or all(not cell.strip() for cell in row): |
||||
skipped['blank'] += 1 |
||||
continue |
||||
if len(row) != len(header): |
||||
description = row[2] if len(row) > 2 else '' |
||||
yield ParsedItem(f'line:{line}', '1', description=description, |
||||
errors=('INVALID_ROW',)) |
||||
elif _balance(row[2]): |
||||
skipped['balance'] += 1 |
||||
else: |
||||
yield ParsedItem(f'line:{line}', '1', row[0].strip(), row[2], row[5].strip()) |
||||
except StopIteration: |
||||
raise ParseError('INVALID_HEADER') from None |
||||
except csv.Error: |
||||
raise ParseError('INVALID_CSV') from None |
||||
|
||||
|
||||
def _txt_items(text, profile, skipped): |
||||
partial = profile == 'OUROCARD_PARTIAL_V1' |
||||
date_size, body_size = (5, 43) if partial else (10, 37) |
||||
date_re = r'\d{2}/\d{2}' if partial else r'\d{2}\.\d{2}\.\d{4}' |
||||
regular = re.compile(rf'^({date_re})(.{{{body_size}}})([A-Z ]{{2}})\s+(\S+)\s+(\S+)\s*$') |
||||
payment = re.compile(rf'^({date_re})\s*(PGTO DEBITO CONTA.*?)200211\s+(\S+)\s+(\S+)\s*$') |
||||
section, section_count = '1', 0 |
||||
for number, line in enumerate(text.splitlines(), 1): |
||||
locator = f'line:{number}' |
||||
if not line.strip(): |
||||
skipped['blank'] += 1 |
||||
elif re.match(r'^\s*\d\s*-\s*[A-Z]', line): |
||||
section_count += 1 |
||||
section = str(section_count) |
||||
skipped['section'] += 1 |
||||
elif _balance(line): |
||||
skipped['balance'] += 1 |
||||
elif match := payment.fullmatch(line): |
||||
yield ParsedItem(locator, section, match[1], match[2].strip(), match[3], match[4], True) |
||||
elif match := regular.fullmatch(line): |
||||
if _balance(match[2]): |
||||
skipped['balance'] += 1 |
||||
else: |
||||
yield ParsedItem(locator, section, match[1], match[2].rstrip(), match[4], match[5]) |
||||
else: |
||||
code = 'INVALID_ROW' if re.match(r'^\s*\d{1,2}[./]', line) else 'UNRECOGNIZED_LINE' |
||||
yield ParsedItem(locator, section, description=line, errors=(code,)) |
||||
|
||||
|
||||
def parse_document(data: bytes, profile: str, context: dict | None = None) -> ParseResult: |
||||
if profile not in PROFILES: |
||||
raise ParseError('UNSUPPORTED_PROFILE') |
||||
if not isinstance(data, bytes): |
||||
raise ParseError('INVALID_INPUT') |
||||
if len(data) > MAX_BYTES: |
||||
raise ParseError('INPUT_TOO_LARGE') |
||||
if b'\x00' in data: |
||||
raise ParseError('NUL_BYTE') |
||||
if context is not None and (not isinstance(context, dict) or any(not isinstance(v, str) for v in context.values())): |
||||
raise ParseError('INVALID_DATE_CONTEXT') |
||||
try: |
||||
encoding = 'utf-8-sig' if data.startswith(b'\xef\xbb\xbf') else 'utf-8' |
||||
text = data.decode(encoding) |
||||
except UnicodeDecodeError: |
||||
encoding, text = 'latin-1', data.decode('latin-1') |
||||
skipped = Counter() |
||||
source = _csv_items(text, skipped) if profile == 'BANK_CSV_V1' else _txt_items(text, profile, skipped) |
||||
items = [] |
||||
for item in source: |
||||
items.append(item) |
||||
if len(items) > MAX_ITEMS: |
||||
raise ParseError('TOO_MANY_ITEMS') |
||||
return ParseResult(encoding, profile, tuple(items), MappingProxyType(dict(skipped)), MappingProxyType(dict(context or {}))) |
||||
@ -0,0 +1,335 @@
|
||||
"""Audited review lifecycle. No financial rows exist until an explicit commit.""" |
||||
from datetime import date |
||||
from functools import wraps |
||||
import hashlib |
||||
import re |
||||
from uuid import UUID, uuid5, NAMESPACE_URL |
||||
|
||||
from django.core.exceptions import ValidationError |
||||
from django.db import IntegrityError, connection, transaction |
||||
from django.db.models import Q |
||||
|
||||
from finance.models import Account, Payee, Category, CardStatement, InstallmentPlan, Transaction, SourceRecord |
||||
from finance.services import Actor, DomainError |
||||
from finance.services.base import audit, identity, save |
||||
from finance.services.posting import record |
||||
from ingestion.models import ImportBatch, ImportItem |
||||
from ingestion.parsers import parse_document, ParseError |
||||
from ingestion.normalization import normalize |
||||
from ingestion.duplicates import candidate_signature, transaction_signature |
||||
from ingestion.validation import validate |
||||
|
||||
|
||||
REASONS = {"USER_EXCLUDED", "MALFORMED", "UNRECOGNIZED", "ALREADY_IMPORTED", "SECTION_EXCLUDED"} |
||||
REFERENCE_FIELDS = {"payee_id": ("payees", Payee), "category_id": ("categories", Category), |
||||
"statement_id": ("statements", CardStatement), "installment_plan_id": ("plans", InstallmentPlan)} |
||||
|
||||
|
||||
def operation(name): |
||||
def decorate(function): |
||||
@wraps(function) |
||||
def wrapped(*args, actor, **kwargs): |
||||
if not isinstance(actor, Actor) or actor.kind not in {"HUMAN", "APPLICATION", "AGENT"} or not isinstance(actor.identifier, str) or not actor.identifier.strip() or len(actor.identifier) > 200: |
||||
raise DomainError("invalid_actor") |
||||
if connection.in_atomic_block: |
||||
raise DomainError("transaction_boundary") |
||||
try: |
||||
with transaction.atomic(durable=True): |
||||
batch, fields, emit = function(*args, actor=actor, **kwargs) |
||||
if emit: |
||||
audit(actor, name, "SUCCESS", batch.pk, fields) |
||||
return batch |
||||
except (DomainError, ParseError, ValidationError, IntegrityError) as error: |
||||
code = getattr(error, "code", None) |
||||
if not isinstance(code, str): |
||||
code = "CONFLICT" if isinstance(error, IntegrityError) else "VALIDATION_ERROR" |
||||
with transaction.atomic(durable=True): |
||||
audit(actor, name, "FAILURE", error_code=code) |
||||
raise DomainError(code) from None |
||||
return wrapped |
||||
return decorate |
||||
|
||||
|
||||
def _batch(batch_id, revision=None, *, preview=True): |
||||
try: |
||||
batch = ImportBatch.objects.select_for_update().get(pk=identity(batch_id)) |
||||
except ImportBatch.DoesNotExist: |
||||
raise DomainError("BATCH_NOT_FOUND") from None |
||||
if revision is not None and (type(revision) is not int or revision != batch.revision): |
||||
raise DomainError("STALE_REVISION") |
||||
if preview and batch.state != "PREVIEW": |
||||
raise DomainError("BATCH_NOT_PREVIEW") |
||||
return batch |
||||
|
||||
|
||||
def _rows(query, locking): |
||||
if locking: |
||||
query = query.select_for_update(of=("self",)) |
||||
return {str(obj.pk): obj for obj in query.order_by("pk")} |
||||
|
||||
|
||||
def _load(batch, items, locking): |
||||
account_ids = [value for value in batch.section_mapping.values() if value] |
||||
accounts = _rows(Account.objects.filter(pk__in=account_ids), locking) |
||||
refs = {} |
||||
for field, (key, model) in REFERENCE_FIELDS.items(): |
||||
ids = [item.decision[field] for item in items if item.decision.get(field)] |
||||
refs[key] = _rows(model.objects.filter(pk__in=ids), locking) |
||||
dates = set() |
||||
for item in items: |
||||
try: |
||||
dates.add(date.fromisoformat(item.candidate.get("posting_date"))) |
||||
except (TypeError, ValueError): |
||||
pass |
||||
target_ids = [item.decision[key] for item in items for key in ("transaction_id", "refund_of_id") if item.decision.get(key)] |
||||
query = Transaction.objects.filter(Q(account_id__in=account_ids, posting_date__in=dates) | Q(pk__in=target_ids)) |
||||
refs["transactions"] = _rows(query.select_related("account", "installment_plan"), locking) |
||||
sources = {row.item_locator: row for row in SourceRecord.objects.filter(document_hash=batch.document_hash).select_related("transaction")} |
||||
return accounts, refs, sources |
||||
|
||||
|
||||
def _fingerprint(obj): |
||||
# IDs and modification timestamps detect reference annotations/archive changes. |
||||
return [str(obj.pk), obj.updated_at.isoformat()] |
||||
|
||||
|
||||
def _evaluate(batch, items, *, locking=False): |
||||
accounts, refs, sources = _load(batch, items, locking) |
||||
keys = {} |
||||
existing = {} |
||||
for row in refs["transactions"].values(): |
||||
existing.setdefault(transaction_signature(row), []).append(str(row.pk)) |
||||
for item in items: |
||||
account = accounts.get(batch.section_mapping.get(item.candidate["section"])) |
||||
if account and not item.candidate.get("errors"): |
||||
try: |
||||
keys[item.locator] = candidate_signature(item.candidate, account, item.decision) |
||||
except (KeyError, TypeError, ValueError): |
||||
pass |
||||
results = {} |
||||
plan_definitions = {} |
||||
plan_memberships = set() |
||||
for item in items: |
||||
candidate, decision = item.candidate, item.decision |
||||
mapped = batch.section_mapping.get(candidate["section"]) |
||||
account = accounts.get(mapped) |
||||
issues = validate(candidate, decision, account, refs, batch.profile) if mapped else ["SECTION_EXCLUDED"] |
||||
key = keys.get(item.locator) |
||||
source = sources.get(item.locator) |
||||
matches = sorted(existing.get(key, [])) if key is not None else [] |
||||
within = sorted(locator for locator, other in keys.items() if locator != item.locator and other == key) if key is not None else [] |
||||
duplicate = {"existing": matches, "within": within, |
||||
"source": str(source.transaction_id) if source else None, |
||||
"source_account": str(source.transaction.account_id) if source else None, |
||||
"status": "ALREADY_IMPORTED" if source else None} |
||||
if source: |
||||
issues.append("ALREADY_IMPORTED" if str(source.transaction.account_id) == mapped else "SOURCE_ACCOUNT_CONFLICT") |
||||
if decision.get("new_plan") and decision.get("action") == "CREATE": |
||||
name = decision["new_plan"] |
||||
definition = (mapped, candidate.get("installment_total")) |
||||
if name in plan_definitions and plan_definitions[name] != definition: |
||||
issues.append("NEW_PLAN_CONFLICT") |
||||
plan_definitions[name] = definition |
||||
if decision.get("action") == "CREATE" and candidate.get("installment_number"): |
||||
plan_key = decision.get("installment_plan_id") or decision.get("new_plan") |
||||
membership = (plan_key, candidate["installment_number"]) |
||||
if plan_key and membership in plan_memberships: |
||||
issues.append("DUPLICATE_INSTALLMENT") |
||||
plan_memberships.add(membership) |
||||
results[item.locator] = {"issues": sorted(set(issues)), "duplicates": duplicate} |
||||
reference_snapshot = {name: [_fingerprint(obj) for obj in table.values()] for name, table in refs.items() if name != "transactions"} |
||||
relevant_ids = {tid for result in results.values() for tid in result["duplicates"]["existing"]} |
||||
relevant_ids.update(item.decision[key] for item in items for key in ("transaction_id", "refund_of_id") if item.decision.get(key)) |
||||
reference_snapshot["transactions"] = [_fingerprint(obj) for key, obj in refs["transactions"].items() if key in relevant_ids] |
||||
return {"accounts": [_fingerprint(obj) for obj in accounts.values()], "references": reference_snapshot, "items": results} |
||||
|
||||
|
||||
def _persist(batch, items, snapshot): |
||||
for item in items: |
||||
current = snapshot["items"][item.locator] |
||||
item.issues, item.duplicates = current["issues"], current["duplicates"] |
||||
ImportItem.objects.bulk_update(items, ["issues", "duplicates", "decision"]) |
||||
batch.snapshot = snapshot |
||||
batch.summary = {**batch.summary, "rows": len(items), "error_rows": sum(bool(item.issues) for item in items), |
||||
"decided_rows": sum(bool(item.decision) for item in items)} |
||||
batch.save() |
||||
|
||||
|
||||
def _current(batch, items): |
||||
snapshot = _evaluate(batch, items, locking=True) |
||||
if snapshot != batch.snapshot: |
||||
raise DomainError("STALE_PREVIEW") |
||||
return snapshot |
||||
|
||||
|
||||
@operation("import_preview") |
||||
def create_preview(*, data, profile, section_mapping, context=None, actor): |
||||
parsed = parse_document(data, profile, context or {}) |
||||
candidates = [candidate.to_dict() for candidate in normalize(parsed)] |
||||
if not isinstance(section_mapping, dict) or any(not isinstance(key, str) for key in section_mapping): |
||||
raise DomainError("INVALID_SECTION_MAPPING") |
||||
sections = {item["section"] for item in candidates} |
||||
if set(section_mapping) != sections: |
||||
raise DomainError("SECTION_MAPPING_REQUIRED") |
||||
mapping = {key: str(identity(value)) if value is not None else None for key, value in section_mapping.items()} |
||||
batch = ImportBatch.objects.create(document_hash=hashlib.sha256(data).hexdigest(), |
||||
source_format="BANK_CSV" if profile == "BANK_CSV_V1" else "OUROCARD_TXT", |
||||
profile=profile, encoding=parsed.encoding, context=context or {}, section_mapping=mapping, |
||||
summary={"parser_skipped": dict(parsed.skipped)}, actor_kind=actor.kind, actor_identifier=actor.identifier) |
||||
items = [ImportItem(batch=batch, locator=candidate["locator"], candidate=candidate) for candidate in candidates] |
||||
ImportItem.objects.bulk_create(items) |
||||
_persist(batch, items, _evaluate(batch, items, locking=True)) |
||||
return batch, ["preview"], True |
||||
|
||||
|
||||
def export_review(*, batch_id): |
||||
# Keep exported revision and item decisions consistent with concurrent editing. |
||||
with transaction.atomic(): |
||||
batch = _batch(batch_id, preview=False) |
||||
items = list(batch.items.order_by("created_at", "locator")) |
||||
return {"batch_id": str(batch.pk), "revision": batch.revision, "state": batch.state, |
||||
"profile": batch.profile, "encoding": batch.encoding, "summary": batch.summary, |
||||
"items": [{"locator": row.locator, "candidate_id": str(uuid5(NAMESPACE_URL, batch.document_hash + ":" + row.locator)), |
||||
"candidate": row.candidate, "issues": row.issues, "duplicates": row.duplicates, |
||||
"decision": row.decision} for row in items], |
||||
"decisions": [{"locator": row.locator, **row.decision} for row in items]} |
||||
|
||||
|
||||
def _decisions(value, locators): |
||||
if not isinstance(value, list): |
||||
raise DomainError("INVALID_DECISIONS") |
||||
result = {} |
||||
for supplied in value: |
||||
if not isinstance(supplied, dict): |
||||
raise DomainError("INVALID_DECISION") |
||||
row = dict(supplied) |
||||
locator = row.pop("locator", None) |
||||
if not isinstance(locator, str) or locator not in locators or locator in result: |
||||
raise DomainError("INVALID_LOCATOR") |
||||
action = row.get("action") |
||||
allowed = {"action"} |
||||
if action == "CREATE": |
||||
allowed |= {"acknowledge_duplicate", "kind", "payee_id", "category_id", "statement_id", "refund_of_id", "installment_plan_id", "new_plan"} |
||||
elif action == "MATCH": |
||||
allowed.add("transaction_id") |
||||
elif action == "SKIP": |
||||
allowed.add("reason") |
||||
else: |
||||
raise DomainError("INVALID_ACTION") |
||||
if set(row) - allowed: |
||||
raise DomainError("UNKNOWN_DECISION_FIELD") |
||||
if "acknowledge_duplicate" in row and type(row["acknowledge_duplicate"]) is not bool: |
||||
raise DomainError("INVALID_ACKNOWLEDGEMENT") |
||||
if action == "SKIP" and row.get("reason") not in REASONS: |
||||
raise DomainError("SKIP_REASON_REQUIRED") |
||||
if action == "MATCH" and not row.get("transaction_id"): |
||||
raise DomainError("MATCH_TARGET_REQUIRED") |
||||
for field in set(row) & {"transaction_id", "payee_id", "category_id", "statement_id", "refund_of_id", "installment_plan_id"}: |
||||
if row[field] is not None: |
||||
row[field] = str(identity(row[field])) |
||||
if "new_plan" in row and (not isinstance(row["new_plan"], str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", row["new_plan"])): |
||||
raise DomainError("INVALID_NEW_PLAN") |
||||
if "kind" in row and row["kind"] not in {"INCOME", "EXPENSE", "REFUND", "TRANSFER"}: |
||||
raise DomainError("INVALID_KIND") |
||||
result[locator] = row |
||||
return result |
||||
|
||||
|
||||
@operation("import_decisions") |
||||
def apply_decisions(*, batch_id, revision, decisions, actor): |
||||
batch = _batch(batch_id, revision) |
||||
items = list(batch.items.order_by("locator")) |
||||
_current(batch, items) |
||||
values = _decisions(decisions, {item.locator for item in items}) |
||||
for item in items: |
||||
item.decision = values.get(item.locator, {}) |
||||
batch.revision += 1 |
||||
_persist(batch, items, _evaluate(batch, items, locking=True)) |
||||
return batch, ["decisions", "revision"], True |
||||
|
||||
|
||||
@operation("import_refresh") |
||||
def refresh_preview(*, batch_id, revision, actor): |
||||
batch = _batch(batch_id, revision) |
||||
items = list(batch.items.order_by("locator")) |
||||
for item in items: |
||||
item.decision = {} |
||||
batch.revision += 1 |
||||
_persist(batch, items, _evaluate(batch, items, locking=True)) |
||||
return batch, ["preview", "decisions", "revision"], True |
||||
|
||||
|
||||
@operation("import_cancel") |
||||
def cancel_preview(*, batch_id, revision, actor): |
||||
batch = _batch(batch_id, revision) |
||||
batch.state = "CANCELLED" |
||||
batch.save() |
||||
return batch, ["state"], True |
||||
|
||||
|
||||
def _check_decision(item): |
||||
decision = item.decision |
||||
if not decision: |
||||
raise DomainError("DECISION_REQUIRED") |
||||
if "SOURCE_ACCOUNT_CONFLICT" in item.issues: |
||||
raise DomainError("SOURCE_ACCOUNT_CONFLICT") |
||||
if decision["action"] == "SKIP": |
||||
if item.duplicates.get("source") and decision["reason"] != "ALREADY_IMPORTED": |
||||
raise DomainError("ALREADY_IMPORTED") |
||||
if "SECTION_EXCLUDED" in item.issues and decision["reason"] != "SECTION_EXCLUDED": |
||||
raise DomainError("SECTION_EXCLUDED") |
||||
return |
||||
if item.duplicates.get("source"): |
||||
raise DomainError("ALREADY_IMPORTED") |
||||
# MATCH validates the signature rather than requiring a new plan assignment. |
||||
issues = [issue for issue in item.issues if not (decision["action"] == "MATCH" and issue == "INSTALLMENT_PLAN_REQUIRED")] |
||||
if issues: |
||||
raise DomainError("UNRESOLVED_ISSUES") |
||||
if decision["action"] == "CREATE" and (item.duplicates["existing"] or item.duplicates["within"]) and not decision.get("acknowledge_duplicate"): |
||||
raise DomainError("DUPLICATE_ACKNOWLEDGEMENT_REQUIRED") |
||||
if decision["action"] == "MATCH" and decision["transaction_id"] not in item.duplicates["existing"]: |
||||
raise DomainError("MATCH_SIGNATURE_MISMATCH") |
||||
|
||||
|
||||
@operation("import_commit") |
||||
def commit_preview(*, batch_id, revision, actor): |
||||
batch = _batch(batch_id, revision, preview=False) |
||||
if batch.state == "COMMITTED": |
||||
return batch, [], False |
||||
if batch.state != "PREVIEW": |
||||
raise DomainError("BATCH_NOT_PREVIEW") |
||||
items = list(batch.items.order_by("locator")) |
||||
_current(batch, items) |
||||
for item in items: |
||||
_check_decision(item) |
||||
plans = {} |
||||
result = {"created": 0, "matched": 0, "skipped": 0, "transaction_ids": [], "parser_skipped": batch.summary.get("parser_skipped", {}), "error_rows": batch.summary.get("error_rows", 0)} |
||||
for item in items: |
||||
candidate, decision = item.candidate, item.decision |
||||
if decision["action"] == "SKIP": |
||||
result["skipped"] += 1 |
||||
continue |
||||
if decision["action"] == "MATCH": |
||||
row = Transaction.objects.get(pk=decision["transaction_id"]) |
||||
result["matched"] += 1 |
||||
else: |
||||
account_id = batch.section_mapping[candidate["section"]] |
||||
plan_id = decision.get("installment_plan_id") |
||||
if decision.get("new_plan"): |
||||
name = decision["new_plan"] |
||||
if name not in plans: |
||||
plans[name] = save(InstallmentPlan(account_id=account_id, description=name, total_installments=candidate["installment_total"])) |
||||
plan_id = plans[name].pk |
||||
fields = {key: decision.get(key) for key in ("payee_id", "category_id", "statement_id", "refund_of_id")} |
||||
row = record(account_id=account_id, amount=candidate["amount"], posting_date=candidate["posting_date"], |
||||
occurrence_date=candidate["occurrence_date"], kind=decision.get("kind", candidate["kind"]), |
||||
description=candidate["description"], installment_plan_id=plan_id, |
||||
installment_number=candidate.get("installment_number"), **fields) |
||||
result["created"] += 1 |
||||
save(SourceRecord(transaction=row, document_hash=batch.document_hash, source_format=batch.source_format, |
||||
parser_version=batch.parser_version, item_locator=item.locator)) |
||||
result["transaction_ids"].append(str(row.pk)) |
||||
batch.state = "COMMITTED" |
||||
batch.result = result |
||||
batch.save() |
||||
return batch, ["state", "transactions", "source_records", "plans"], True |
||||
@ -0,0 +1,57 @@
|
||||
"""Read-only domain validation for review; commit also uses finance posting rules.""" |
||||
from datetime import date |
||||
from finance.services.base import DomainError, money |
||||
from finance.models import Transaction |
||||
|
||||
|
||||
def validate(candidate, decision, account, refs, profile): |
||||
issues = list(candidate.get("errors", [])) |
||||
if account is None: |
||||
return issues + ["ACCOUNT_NOT_FOUND"] |
||||
if account.archived: |
||||
issues.append("ACCOUNT_ARCHIVED") |
||||
if account.currency != "BRL": |
||||
issues.append("UNSUPPORTED_CURRENCY") |
||||
expected = "BANK" if profile == "BANK_CSV_V1" else "CREDIT_CARD" |
||||
if account.kind != expected: |
||||
issues.append("ACCOUNT_KIND_MISMATCH") |
||||
if issues: |
||||
return issues |
||||
try: |
||||
amount = money(candidate["amount"]) |
||||
date.fromisoformat(candidate["posting_date"]) |
||||
except (DomainError, ValueError, TypeError, KeyError): |
||||
return ["INVALID_MOVEMENT"] |
||||
kind = decision.get("kind", candidate["kind"]) |
||||
if kind not in {"INCOME", "EXPENSE", "REFUND", "TRANSFER"} or (kind in {"INCOME", "REFUND"} and amount <= 0) or (kind == "EXPENSE" and amount >= 0): |
||||
issues.append("INVALID_KIND_OR_SIGN") |
||||
for field, table in (("payee_id", "payees"), ("category_id", "categories")): |
||||
if decision.get(field): |
||||
ref = refs[table].get(decision[field]) |
||||
if ref is None or ref.archived: |
||||
issues.append("INVALID_" + field.upper()) |
||||
category = refs["categories"].get(decision.get("category_id")) |
||||
if category and category.kind != {"INCOME": "INCOME", "EXPENSE": "EXPENSE", "REFUND": "EXPENSE"}.get(kind): |
||||
issues.append("CATEGORY_KIND_MISMATCH") |
||||
statement = refs["statements"].get(decision.get("statement_id")) |
||||
if decision.get("statement_id") and (statement is None or statement.account_id != account.pk or account.kind != "CREDIT_CARD"): |
||||
issues.append("STATEMENT_ACCOUNT_MISMATCH") |
||||
original = refs["transactions"].get(decision.get("refund_of_id")) |
||||
if decision.get("refund_of_id") and (original is None or kind != "REFUND" or original.kind != "EXPENSE" or original.account.currency != account.currency): |
||||
issues.append("INVALID_REFUND_REFERENCE") |
||||
number, total = candidate.get("installment_number"), candidate.get("installment_total") |
||||
if number is not None or total is not None: |
||||
if not isinstance(number, int) or not isinstance(total, int) or not 1 <= number <= total or kind != "EXPENSE": |
||||
issues.append("INVALID_INSTALLMENT") |
||||
plan_id, new_plan = decision.get("installment_plan_id"), decision.get("new_plan") |
||||
if bool(plan_id) == bool(new_plan): |
||||
issues.append("INSTALLMENT_PLAN_REQUIRED") |
||||
if plan_id: |
||||
plan = refs["plans"].get(plan_id) |
||||
if plan is None or plan.account_id != account.pk or plan.total_installments != total: |
||||
issues.append("INSTALLMENT_PLAN_MISMATCH") |
||||
elif decision.get("action") == "CREATE" and Transaction.objects.filter(installment_plan=plan, installment_number=number).exists(): |
||||
issues.append("INSTALLMENT_ALREADY_POSTED") |
||||
elif decision.get("installment_plan_id") or decision.get("new_plan"): |
||||
issues.append("UNEXPECTED_INSTALLMENT_PLAN") |
||||
return issues |
||||
@ -0,0 +1,7 @@
|
||||
import os |
||||
import sys |
||||
from django.core.management import execute_from_command_line |
||||
|
||||
if __name__ == "__main__": |
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") |
||||
execute_from_command_line(sys.argv) |
||||
@ -0,0 +1,14 @@
|
||||
[project] |
||||
name = "gizmoduck" |
||||
version = "0.1.0" |
||||
requires-python = ">=3.13,<3.14" |
||||
dependencies = ["Django>=5.2,<5.3", "psycopg[binary]>=3.2,<4"] |
||||
|
||||
[dependency-groups] |
||||
dev = ["pytest>=8,<10", "pytest-django>=4.11,<5"] |
||||
|
||||
[tool.pytest.ini_options] |
||||
DJANGO_SETTINGS_MODULE = "config.settings" |
||||
pythonpath = ["."] |
||||
testpaths = ["tests"] |
||||
addopts = "-ra" |
||||
@ -0,0 +1,76 @@
|
||||
"""Run against the migrated development database; all DML is rolled back.""" |
||||
import os |
||||
import uuid |
||||
|
||||
import psycopg |
||||
from psycopg.errors import InsufficientPrivilege |
||||
|
||||
|
||||
def check_permissions(): |
||||
with psycopg.connect(host=os.environ["DB_HOST"], dbname=os.environ["DB_NAME"], |
||||
user=os.environ["DB_USER"], password=os.environ["DB_PASSWORD"], |
||||
autocommit=True) as conn: |
||||
row = conn.execute("SELECT rolsuper, rolcreatedb, rolcreaterole FROM pg_roles WHERE rolname = current_user").fetchone() |
||||
assert row == (False, False, False), "Runtime role has elevated privileges" |
||||
with conn.transaction(force_rollback=True): |
||||
name = "synthetic_permission_" + uuid.uuid4().hex |
||||
pk = conn.execute("INSERT INTO auth_group (name) VALUES (%s) RETURNING id", (name,)).fetchone()[0] |
||||
assert conn.execute("SELECT name FROM auth_group WHERE id = %s", (pk,)).fetchone() == (name,) |
||||
conn.execute("UPDATE auth_group SET name = %s WHERE id = %s", (name + "_updated", pk)) |
||||
assert conn.execute("SELECT name FROM auth_group WHERE id = %s", (pk,)).fetchone() == (name + "_updated",) |
||||
conn.execute("DELETE FROM auth_group WHERE id = %s", (pk,)) |
||||
assert conn.execute("SELECT id FROM auth_group WHERE id = %s", (pk,)).fetchone() is None |
||||
for statement in ["CREATE TABLE public.permission_probe (id integer)", |
||||
"CREATE SCHEMA permission_probe", |
||||
"SET ROLE gizmoduck_migrate"]: |
||||
try: |
||||
with conn.transaction(force_rollback=True): |
||||
conn.execute(statement) |
||||
except InsufficientPrivilege: |
||||
pass |
||||
else: |
||||
raise AssertionError("Runtime unexpectedly permitted: " + statement) |
||||
application_tables = conn.execute( |
||||
"SELECT tablename FROM pg_tables " |
||||
"WHERE schemaname = 'public' " |
||||
"AND (tablename LIKE 'finance_%' OR tablename LIKE 'ingestion_%')" |
||||
).fetchall() |
||||
assert len(application_tables) >= 11, "Application migrations are missing" |
||||
for (table,) in application_tables: |
||||
for privilege in ("SELECT", "INSERT", "UPDATE", "DELETE"): |
||||
assert conn.execute("SELECT has_table_privilege(current_user, %s, %s)", ("public." + table, privilege)).fetchone()[0], (table, privilege) |
||||
with conn.transaction(force_rollback=True): |
||||
account_id = uuid.uuid4() |
||||
conn.execute("INSERT INTO finance_account (id, name, kind, currency, archived, created_at, updated_at) VALUES (%s, %s, 'BANK', 'BRL', false, now(), now())", (account_id, "synthetic_" + account_id.hex)) |
||||
assert conn.execute("SELECT currency FROM finance_account WHERE id = %s", (account_id,)).fetchone() == ("BRL",) |
||||
conn.execute("UPDATE finance_account SET archived = true WHERE id = %s", (account_id,)) |
||||
assert conn.execute("SELECT archived FROM finance_account WHERE id = %s", (account_id,)).fetchone() == (True,) |
||||
conn.execute("DELETE FROM finance_account WHERE id = %s", (account_id,)) |
||||
with conn.transaction(force_rollback=True): |
||||
batch_id = uuid.uuid4() |
||||
conn.execute( |
||||
"INSERT INTO ingestion_importbatch " |
||||
"(id, created_at, updated_at, document_hash, source_format, " |
||||
"profile, parser_version, encoding, context, section_mapping, " |
||||
"summary, snapshot, actor_kind, actor_identifier, state, revision, result) " |
||||
"VALUES (%s, now(), now(), %s, 'BANK_CSV', 'BANK_CSV_V1', " |
||||
"'1', 'utf-8', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb, " |
||||
"'{}'::jsonb, 'AGENT', 'synthetic-permission-check', " |
||||
"'PREVIEW', 1, '{}'::jsonb)", |
||||
(batch_id, "0" * 64), |
||||
) |
||||
assert conn.execute( |
||||
"SELECT state FROM ingestion_importbatch WHERE id = %s", (batch_id,) |
||||
).fetchone() == ("PREVIEW",) |
||||
conn.execute( |
||||
"UPDATE ingestion_importbatch SET state = 'CANCELLED' WHERE id = %s", |
||||
(batch_id,), |
||||
) |
||||
conn.execute("DELETE FROM ingestion_importbatch WHERE id = %s", (batch_id,)) |
||||
# PostgreSQL CREATE DATABASE cannot run in a rollback transaction. The |
||||
# server's role attributes above verify that capability without creating one. |
||||
print("PASS: finance/ingestion DML, sequences, denied DDL and role escalation") |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
check_permissions() |
||||
@ -0,0 +1,56 @@
|
||||
"""Real PostgreSQL competing writers, with independent thread connections.""" |
||||
|
||||
from concurrent.futures import ThreadPoolExecutor |
||||
from datetime import date |
||||
from decimal import Decimal |
||||
from threading import Barrier |
||||
|
||||
import pytest |
||||
from django.db import close_old_connections, connection, connections |
||||
|
||||
from finance.models import AuditEvent, Transaction, Transfer |
||||
from finance.services import Actor, DomainError, create_account, match_transfer, record_transaction |
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True) |
||||
|
||||
|
||||
def test_competing_matches_cannot_reuse_one_outgoing_leg(): |
||||
assert connection.vendor == "postgresql" |
||||
actor = Actor(kind="AGENT", identifier="synthetic-concurrency-test") |
||||
bank = create_account(name="Synthetic Outgoing", kind="BANK", actor=actor) |
||||
other = create_account(name="Synthetic Incoming", kind="BANK", actor=actor) |
||||
common = dict(kind="TRANSFER", posting_date=date(2026, 1, 15), actor=actor) |
||||
outgoing = record_transaction(account_id=bank.pk, amount=Decimal("-25"), **common) |
||||
incoming = [record_transaction(account_id=other.pk, amount=Decimal("25"), **common) for _ in range(2)] |
||||
barrier = Barrier(2) |
||||
audit_ids = set(AuditEvent.objects.values_list("pk", flat=True)) |
||||
|
||||
def attempt(incoming_id): |
||||
close_old_connections() |
||||
try: |
||||
# Opening connections before the barrier ensures two real sessions. |
||||
with connection.cursor() as cursor: |
||||
cursor.execute("SELECT pg_backend_pid()") |
||||
backend_pid = cursor.fetchone()[0] |
||||
barrier.wait(timeout=15) |
||||
try: |
||||
result = match_transfer(outgoing_id=outgoing.pk, incoming_id=incoming_id, actor=actor) |
||||
return backend_pid, "success", result.pk |
||||
except DomainError as error: |
||||
return backend_pid, "failure", error.code |
||||
finally: |
||||
connections.close_all() |
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool: |
||||
results = list(pool.map(attempt, [row.pk for row in incoming])) |
||||
|
||||
assert len({result[0] for result in results}) == 2 |
||||
assert sorted(result[1] for result in results) == ["failure", "success"] |
||||
assert all(result[2] for result in results) |
||||
assert Transfer.objects.count() == 1 |
||||
paired = Transfer.objects.get() |
||||
assert paired.outgoing_id == outgoing.pk |
||||
assert paired.incoming_id in {row.pk for row in incoming} |
||||
assert Transaction.objects.count() == 3 |
||||
outcomes = list(AuditEvent.objects.exclude(pk__in=audit_ids).values_list("outcome", flat=True)) |
||||
assert sorted(outcomes) == ["FAILURE", "SUCCESS"] |
||||
@ -0,0 +1,139 @@
|
||||
"""Synthetic direct-ORM checks for the PostgreSQL persistence boundary.""" |
||||
|
||||
from datetime import date |
||||
from decimal import Decimal |
||||
|
||||
import pytest |
||||
from django.db import IntegrityError |
||||
from django.db.models.deletion import ProtectedError |
||||
|
||||
from finance.models import ( |
||||
Account, AuditEvent, CardStatement, Category, InstallmentPlan, Payee, |
||||
SourceRecord, Transaction, Transfer, |
||||
) |
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True) |
||||
|
||||
|
||||
@pytest.fixture |
||||
def account(): |
||||
return Account.objects.create(name="Synthetic bank", kind="BANK") |
||||
|
||||
|
||||
def post(account, **changes): |
||||
values = dict(account=account, amount=Decimal("-10.00"), |
||||
posting_date=date(2026, 9, 1), kind="EXPENSE") |
||||
values.update(changes) |
||||
return Transaction.objects.create(**values) |
||||
|
||||
|
||||
@pytest.mark.parametrize("kind,amount", [ |
||||
("EXPENSE", "0"), ("EXPENSE", "1"), ("INCOME", "-1"), |
||||
("REFUND", "-1"), ("TRANSFER", "0"), ("OPENING", "0"), |
||||
("INVALID", "-1"), |
||||
]) |
||||
def test_database_rejects_invalid_transaction_signs(account, kind, amount): |
||||
with pytest.raises(IntegrityError): |
||||
post(account, kind=kind, amount=Decimal(amount)) |
||||
|
||||
|
||||
@pytest.mark.parametrize("currency", ["brl", "B1L", "BR", "ÁBC"]) |
||||
def test_database_rejects_invalid_currency(currency): |
||||
with pytest.raises(IntegrityError): |
||||
Account.objects.create(name="Synthetic account", kind="BANK", currency=currency) |
||||
|
||||
|
||||
def test_database_rejects_invalid_reference_choices(): |
||||
with pytest.raises(IntegrityError): |
||||
Account.objects.create(name="Synthetic account", kind="INVALID") |
||||
with pytest.raises(IntegrityError): |
||||
Category.objects.create(name="Synthetic category", kind="INVALID") |
||||
|
||||
|
||||
def test_opening_is_unique_and_cannot_have_associations(account): |
||||
post(account, kind="OPENING") |
||||
with pytest.raises(IntegrityError): |
||||
post(account, kind="OPENING") |
||||
payee = Payee.objects.create(name="Synthetic shop") |
||||
other = Account.objects.create(name="Other synthetic bank", kind="BANK") |
||||
with pytest.raises(IntegrityError): |
||||
post(other, kind="OPENING", payee=payee) |
||||
|
||||
|
||||
def test_refund_reference_only_allowed_on_refund(account): |
||||
expense = post(account) |
||||
with pytest.raises(IntegrityError): |
||||
post(account, refund_of=expense) |
||||
with pytest.raises(IntegrityError): |
||||
Transaction.objects.filter(pk=expense.pk).update( |
||||
kind="REFUND", amount=Decimal("10"), refund_of=expense) |
||||
|
||||
|
||||
def test_transfer_cannot_have_category(account): |
||||
category = Category.objects.create(name="Synthetic expense", kind="EXPENSE") |
||||
with pytest.raises(IntegrityError): |
||||
post(account, kind="TRANSFER", category=category) |
||||
|
||||
|
||||
def test_installment_pair_number_kind_and_uniqueness(account): |
||||
plan = InstallmentPlan.objects.create(account=account, total_installments=10) |
||||
for changes in [dict(installment_plan=plan), dict(installment_number=3), |
||||
dict(installment_plan=plan, installment_number=0), |
||||
dict(installment_plan=plan, installment_number=3, |
||||
kind="REFUND", amount=Decimal("10"))]: |
||||
with pytest.raises(IntegrityError): |
||||
post(account, **changes) |
||||
post(account, installment_plan=plan, installment_number=3) |
||||
with pytest.raises(IntegrityError): |
||||
post(account, installment_plan=plan, installment_number=3) |
||||
with pytest.raises(IntegrityError): |
||||
InstallmentPlan.objects.create(account=account, total_installments=0) |
||||
|
||||
|
||||
def test_statement_date_order_and_identity(account): |
||||
values = dict(account=account, period_start=date(2026, 8, 1), |
||||
period_end=date(2026, 8, 31), due_date=date(2026, 9, 10)) |
||||
with pytest.raises(IntegrityError): |
||||
CardStatement.objects.create(**{**values, "period_end": date(2026, 7, 31)}) |
||||
with pytest.raises(IntegrityError): |
||||
CardStatement.objects.create(**{**values, "due_date": date(2026, 8, 30)}) |
||||
CardStatement.objects.create(**values) |
||||
with pytest.raises(IntegrityError): |
||||
CardStatement.objects.create(**values) |
||||
|
||||
|
||||
def test_source_identity_format_and_protected_history(account): |
||||
activity = post(account) |
||||
values = dict(transaction=activity, document_hash="a" * 64, |
||||
source_format="BANK_CSV", parser_version="synthetic-v1", |
||||
item_locator="row:1") |
||||
for changes in [dict(document_hash="bad"), dict(source_format="OFX")]: |
||||
with pytest.raises(IntegrityError): |
||||
SourceRecord.objects.create(**{**values, **changes}) |
||||
SourceRecord.objects.create(**values) |
||||
with pytest.raises(IntegrityError): |
||||
SourceRecord.objects.create(**values) |
||||
with pytest.raises(ProtectedError): |
||||
activity.delete() |
||||
with pytest.raises(ProtectedError): |
||||
account.delete() |
||||
|
||||
|
||||
def test_transfer_distinct_and_unique_roles(account): |
||||
outgoing = post(account, kind="TRANSFER") |
||||
incoming = post(account, kind="TRANSFER", amount=Decimal("10")) |
||||
with pytest.raises(IntegrityError): |
||||
Transfer.objects.create(outgoing=outgoing, incoming=outgoing) |
||||
Transfer.objects.create(outgoing=outgoing, incoming=incoming) |
||||
with pytest.raises(IntegrityError): |
||||
Transfer.objects.create(outgoing=outgoing, incoming=post(account, amount=Decimal("10"), kind="TRANSFER")) |
||||
with pytest.raises(IntegrityError): |
||||
Transfer.objects.create(outgoing=post(account, kind="TRANSFER"), incoming=incoming) |
||||
|
||||
|
||||
def test_audit_actor_and_outcome_choices(): |
||||
values = dict(actor_kind="HUMAN", actor_identifier="synthetic-owner", |
||||
operation="create_account", outcome="SUCCESS") |
||||
for changes in [dict(actor_kind="INVALID"), dict(outcome="INVALID")]: |
||||
with pytest.raises(IntegrityError): |
||||
AuditEvent.objects.create(**{**values, **changes}) |
||||
@ -0,0 +1,312 @@
|
||||
import csv |
||||
import json |
||||
import threading |
||||
from datetime import date, datetime, timezone as datetime_timezone |
||||
from decimal import Decimal |
||||
from hashlib import sha256 |
||||
from io import BytesIO, StringIO |
||||
from uuid import UUID |
||||
from zipfile import ZipFile |
||||
|
||||
import pytest |
||||
from django.db import connections, transaction |
||||
|
||||
import finance.exports as exports |
||||
from finance.exports import build_export |
||||
from finance.models import ( |
||||
Account, |
||||
AuditEvent, |
||||
CardStatement, |
||||
Category, |
||||
InstallmentPlan, |
||||
Payee, |
||||
SourceRecord, |
||||
Transaction, |
||||
Transfer, |
||||
) |
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True) |
||||
|
||||
EXPECTED_FILES = [ |
||||
"accounts.csv", |
||||
"payees.csv", |
||||
"categories.csv", |
||||
"statements.csv", |
||||
"installment_plans.csv", |
||||
"transactions.csv", |
||||
"transfers.csv", |
||||
"source_records.csv", |
||||
] |
||||
EXPECTED_HEADERS = { |
||||
"accounts.csv": "id,name,kind,currency,archived,created_at,updated_at", |
||||
"payees.csv": "id,name,archived,created_at,updated_at", |
||||
"categories.csv": "id,name,kind,archived,created_at,updated_at", |
||||
"statements.csv": ( |
||||
"id,account_id,period_start,period_end,due_date,statement_balance,created_at,updated_at" |
||||
), |
||||
"installment_plans.csv": ( |
||||
"id,account_id,description,total_installments,created_at,updated_at" |
||||
), |
||||
"transactions.csv": ( |
||||
"id,account_id,amount,posting_date,occurrence_date,description,kind,payee_id," |
||||
"category_id,statement_id,refund_of_id,installment_plan_id,installment_number," |
||||
"created_at,updated_at" |
||||
), |
||||
"transfers.csv": "id,outgoing_id,incoming_id,created_at,updated_at", |
||||
"source_records.csv": ( |
||||
"id,transaction_id,document_hash,source_format,parser_version,item_locator,created_at" |
||||
), |
||||
} |
||||
|
||||
|
||||
def rows(package, name): |
||||
content = package.read(name).decode("utf-8") |
||||
return list(csv.DictReader(StringIO(content))) |
||||
|
||||
|
||||
def seed_complete_export(): |
||||
account = Account.objects.create( |
||||
id=UUID("10000000-0000-4000-8000-000000000002"), |
||||
name="Synthetic Card", |
||||
kind=Account.Kind.CREDIT_CARD, |
||||
currency="BRL", |
||||
archived=True, |
||||
) |
||||
Account.objects.create( |
||||
id=UUID("10000000-0000-4000-8000-000000000001"), |
||||
name="Synthetic Bank", |
||||
kind=Account.Kind.BANK, |
||||
currency="USD", |
||||
) |
||||
payee = Payee.objects.create( |
||||
id=UUID("20000000-0000-4000-8000-000000000001"), |
||||
name="Synthetic Merchant", |
||||
) |
||||
category = Category.objects.create( |
||||
id=UUID("30000000-0000-4000-8000-000000000001"), |
||||
name="Synthetic Food", |
||||
kind=Category.Kind.EXPENSE, |
||||
) |
||||
statement = CardStatement.objects.create( |
||||
id=UUID("40000000-0000-4000-8000-000000000001"), |
||||
account=account, |
||||
period_start=date(2026, 1, 1), |
||||
period_end=date(2026, 1, 31), |
||||
due_date=date(2026, 2, 10), |
||||
statement_balance=Decimal("1234.50"), |
||||
) |
||||
plan = InstallmentPlan.objects.create( |
||||
id=UUID("50000000-0000-4000-8000-000000000001"), |
||||
account=account, |
||||
description="Synthetic appliance", |
||||
total_installments=3, |
||||
) |
||||
purchase = Transaction.objects.create( |
||||
id=UUID("60000000-0000-4000-8000-000000000001"), |
||||
account=account, |
||||
amount=Decimal("-411.50"), |
||||
posting_date=date(2026, 1, 12), |
||||
occurrence_date=date(2026, 1, 10), |
||||
description="Synthetic purchase, first part", |
||||
kind=Transaction.Kind.EXPENSE, |
||||
payee=payee, |
||||
category=category, |
||||
statement=statement, |
||||
installment_plan=plan, |
||||
installment_number=1, |
||||
) |
||||
refund = Transaction.objects.create( |
||||
id=UUID("60000000-0000-4000-8000-000000000002"), |
||||
account=account, |
||||
amount=Decimal("11.50"), |
||||
posting_date=date(2026, 2, 1), |
||||
description="", |
||||
kind=Transaction.Kind.REFUND, |
||||
category=category, |
||||
refund_of=purchase, |
||||
) |
||||
transfer_out = Transaction.objects.create( |
||||
id=UUID("60000000-0000-4000-8000-000000000003"), |
||||
account_id=UUID("10000000-0000-4000-8000-000000000001"), |
||||
amount=Decimal("-20.00"), |
||||
posting_date=date(2026, 2, 2), |
||||
kind=Transaction.Kind.TRANSFER, |
||||
) |
||||
transfer_in = Transaction.objects.create( |
||||
id=UUID("60000000-0000-4000-8000-000000000004"), |
||||
account=account, |
||||
amount=Decimal("20.00"), |
||||
posting_date=date(2026, 2, 2), |
||||
kind=Transaction.Kind.TRANSFER, |
||||
) |
||||
Transfer.objects.create( |
||||
id=UUID("70000000-0000-4000-8000-000000000001"), |
||||
outgoing=transfer_out, |
||||
incoming=transfer_in, |
||||
) |
||||
SourceRecord.objects.create( |
||||
id=UUID("80000000-0000-4000-8000-000000000001"), |
||||
transaction=purchase, |
||||
document_hash="a" * 64, |
||||
source_format=SourceRecord.Format.OUROCARD_TXT, |
||||
parser_version="synthetic-v1", |
||||
item_locator="line:2", |
||||
) |
||||
AuditEvent.objects.create( |
||||
actor_kind=AuditEvent.ActorKind.HUMAN, |
||||
actor_identifier="synthetic-owner", |
||||
operation="synthetic_operation", |
||||
outcome=AuditEvent.Outcome.SUCCESS, |
||||
) |
||||
return account, purchase, refund |
||||
|
||||
|
||||
def test_export_has_versioned_manifest_and_stable_csv_contract(): |
||||
seed_complete_export() |
||||
|
||||
artifact = build_export() |
||||
|
||||
with ZipFile(BytesIO(artifact.content)) as package: |
||||
assert package.namelist() == [*EXPECTED_FILES, "manifest.json"] |
||||
manifest = json.loads(package.read("manifest.json")) |
||||
assert manifest["format"] == "gizmoduck-export" |
||||
assert manifest["version"] == 1 |
||||
assert manifest["application_version"] == "0.1.0" |
||||
assert list(manifest["files"]) == EXPECTED_FILES |
||||
for name in EXPECTED_FILES: |
||||
content = package.read(name) |
||||
assert content.decode("utf-8").splitlines()[0] == EXPECTED_HEADERS[name] |
||||
assert manifest["files"][name]["sha256"] == sha256(content).hexdigest() |
||||
assert manifest["files"][name]["rows"] == len(rows(package, name)) |
||||
|
||||
|
||||
def test_export_retains_portable_values_relationships_and_provenance(): |
||||
account, purchase, refund = seed_complete_export() |
||||
|
||||
artifact = build_export() |
||||
|
||||
with ZipFile(BytesIO(artifact.content)) as package: |
||||
account_rows = rows(package, "accounts.csv") |
||||
assert [row["id"] for row in account_rows] == [ |
||||
"10000000-0000-4000-8000-000000000001", |
||||
"10000000-0000-4000-8000-000000000002", |
||||
] |
||||
assert account_rows[1]["archived"] == "true" |
||||
assert account_rows[1]["currency"] == "BRL" |
||||
|
||||
statement_row = rows(package, "statements.csv")[0] |
||||
assert statement_row["statement_balance"] == "1234.50" |
||||
assert statement_row["due_date"] == "2026-02-10" |
||||
|
||||
transaction_rows = rows(package, "transactions.csv") |
||||
purchase_row = next(row for row in transaction_rows if row["id"] == str(purchase.pk)) |
||||
assert purchase_row["amount"] == "-411.50" |
||||
assert purchase_row["occurrence_date"] == "2026-01-10" |
||||
assert purchase_row["description"] == "Synthetic purchase, first part" |
||||
assert purchase_row["account_id"] == str(account.pk) |
||||
assert purchase_row["installment_number"] == "1" |
||||
assert "T" in purchase_row["created_at"] |
||||
|
||||
refund_row = next(row for row in transaction_rows if row["id"] == str(refund.pk)) |
||||
assert refund_row["occurrence_date"] == "" |
||||
assert refund_row["payee_id"] == "" |
||||
assert refund_row["refund_of_id"] == str(purchase.pk) |
||||
|
||||
source_row = rows(package, "source_records.csv")[0] |
||||
assert source_row == { |
||||
"id": "80000000-0000-4000-8000-000000000001", |
||||
"transaction_id": str(purchase.pk), |
||||
"document_hash": "a" * 64, |
||||
"source_format": "OUROCARD_TXT", |
||||
"parser_version": "synthetic-v1", |
||||
"item_locator": "line:2", |
||||
"created_at": source_row["created_at"], |
||||
} |
||||
assert "audit_events.csv" not in package.namelist() |
||||
|
||||
|
||||
def test_export_is_byte_stable_for_the_same_snapshot(monkeypatch): |
||||
seed_complete_export() |
||||
fixed_now = datetime(2026, 9, 7, 12, 30, tzinfo=datetime_timezone.utc) |
||||
monkeypatch.setattr("finance.exports.timezone.now", lambda: fixed_now) |
||||
|
||||
first = build_export() |
||||
second = build_export() |
||||
|
||||
assert first.filename == second.filename == "gizmoduck-export-20260907T123000Z.zip" |
||||
assert first.content == second.content |
||||
|
||||
|
||||
def test_export_uses_one_repeatable_read_snapshot_across_related_files(monkeypatch): |
||||
seed_complete_export() |
||||
accounts_read = threading.Event() |
||||
writer_finished = threading.Event() |
||||
writer_errors = [] |
||||
new_account_id = UUID("10000000-0000-4000-8000-000000000003") |
||||
new_payee_id = UUID("20000000-0000-4000-8000-000000000002") |
||||
new_transaction_id = UUID("60000000-0000-4000-8000-000000000005") |
||||
new_source_id = UUID("80000000-0000-4000-8000-000000000002") |
||||
original_csv_content = exports._csv_content |
||||
|
||||
def pause_after_accounts(model, fields): |
||||
content = original_csv_content(model, fields) |
||||
if model is Account: |
||||
accounts_read.set() |
||||
assert writer_finished.wait(10), "separate writer did not finish" |
||||
return content |
||||
|
||||
def commit_related_rows(): |
||||
try: |
||||
assert accounts_read.wait(10), "export did not read accounts" |
||||
with transaction.atomic(): |
||||
account = Account.objects.create( |
||||
id=new_account_id, |
||||
name="Synthetic Concurrent Account", |
||||
kind=Account.Kind.BANK, |
||||
currency="BRL", |
||||
) |
||||
payee = Payee.objects.create( |
||||
id=new_payee_id, |
||||
name="Synthetic Concurrent Payee", |
||||
) |
||||
row = Transaction.objects.create( |
||||
id=new_transaction_id, |
||||
account=account, |
||||
amount=Decimal("-7.25"), |
||||
posting_date=date(2026, 3, 1), |
||||
description="Synthetic concurrent transaction", |
||||
kind=Transaction.Kind.EXPENSE, |
||||
payee=payee, |
||||
) |
||||
SourceRecord.objects.create( |
||||
id=new_source_id, |
||||
transaction=row, |
||||
document_hash="b" * 64, |
||||
source_format=SourceRecord.Format.BANK_CSV, |
||||
parser_version="synthetic-v2", |
||||
item_locator="line:3", |
||||
) |
||||
except BaseException as exc: |
||||
writer_errors.append(exc) |
||||
finally: |
||||
connections.close_all() |
||||
writer_finished.set() |
||||
|
||||
monkeypatch.setattr(exports, "_csv_content", pause_after_accounts) |
||||
writer = threading.Thread(target=commit_related_rows) |
||||
writer.start() |
||||
artifact = build_export() |
||||
writer.join(10) |
||||
|
||||
assert not writer.is_alive() |
||||
assert writer_errors == [] |
||||
with ZipFile(BytesIO(artifact.content)) as package: |
||||
assert str(new_account_id) not in {row["id"] for row in rows(package, "accounts.csv")} |
||||
assert str(new_payee_id) not in {row["id"] for row in rows(package, "payees.csv")} |
||||
assert str(new_transaction_id) not in { |
||||
row["id"] for row in rows(package, "transactions.csv") |
||||
} |
||||
assert str(new_source_id) not in { |
||||
row["id"] for row in rows(package, "source_records.csv") |
||||
} |
||||
@ -0,0 +1,126 @@
|
||||
from datetime import date |
||||
|
||||
import pytest |
||||
import finance.queries as queries |
||||
|
||||
from finance.models import Account, Category, Payee, Transaction |
||||
from finance.queries import ( |
||||
QueryInputError, |
||||
TransactionFilters, |
||||
filtered_transactions, |
||||
) |
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db |
||||
|
||||
|
||||
def make_account(name="Synthetic Bank", currency="BRL"): |
||||
return Account.objects.create(name=name, kind=Account.Kind.BANK, currency=currency) |
||||
|
||||
|
||||
def make_transaction(account, *, posting_date, amount="-10.00", **values): |
||||
return Transaction.objects.create( |
||||
account=account, |
||||
amount=amount, |
||||
posting_date=posting_date, |
||||
kind=values.pop("kind", Transaction.Kind.EXPENSE), |
||||
**values, |
||||
) |
||||
|
||||
|
||||
def test_filtered_transactions_combines_filters_and_orders_stably(): |
||||
account = make_account() |
||||
category = Category.objects.create(name="Synthetic Food", kind=Category.Kind.EXPENSE) |
||||
older = make_transaction(account, posting_date=date(2026, 1, 5)) |
||||
newer = make_transaction(account, posting_date=date(2026, 1, 20)) |
||||
make_transaction(account, posting_date=date(2026, 1, 25), category=category) |
||||
make_transaction(account, posting_date=date(2026, 2, 1)) |
||||
make_transaction(make_account("Synthetic USD", "USD"), posting_date=date(2026, 1, 22)) |
||||
|
||||
filters = TransactionFilters.from_mapping({ |
||||
"date_from": "2026-01-01", |
||||
"date_to": "2026-01-31", |
||||
"currency": "BRL", |
||||
"uncategorized": "yes", |
||||
}) |
||||
|
||||
rows = list(filtered_transactions(filters)) |
||||
|
||||
assert [row.pk for row in rows] == [newer.pk, older.pk] |
||||
|
||||
|
||||
def test_filtered_transactions_supports_reference_kind_and_categorized_filters(): |
||||
account = make_account() |
||||
other_account = make_account("Synthetic Other") |
||||
category = Category.objects.create(name="Synthetic Food", kind=Category.Kind.EXPENSE) |
||||
payee = Payee.objects.create(name="Synthetic Merchant") |
||||
expected = make_transaction( |
||||
account, |
||||
posting_date=date(2026, 1, 10), |
||||
category=category, |
||||
payee=payee, |
||||
) |
||||
make_transaction(account, posting_date=date(2026, 1, 11), category=category) |
||||
make_transaction(other_account, posting_date=date(2026, 1, 12), category=category, payee=payee) |
||||
|
||||
filters = TransactionFilters.from_mapping({ |
||||
"account": str(account.pk), |
||||
"category": str(category.pk), |
||||
"payee": str(payee.pk), |
||||
"kind": "EXPENSE", |
||||
"uncategorized": "no", |
||||
}) |
||||
|
||||
assert list(filtered_transactions(filters)) == [expected] |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
("mapping", "errors"), |
||||
[ |
||||
({"date_from": "31/01/2026"}, {"date_from": "invalid_date"}), |
||||
({"date_to": "not-a-date"}, {"date_to": "invalid_date"}), |
||||
({"account": "bad"}, {"account": "invalid_uuid"}), |
||||
({"category": "bad"}, {"category": "invalid_uuid"}), |
||||
({"payee": "bad"}, {"payee": "invalid_uuid"}), |
||||
({"kind": "UNKNOWN"}, {"kind": "invalid_choice"}), |
||||
({"currency": "brl"}, {"currency": "invalid_currency"}), |
||||
({"uncategorized": "maybe"}, {"uncategorized": "invalid_choice"}), |
||||
], |
||||
) |
||||
def test_invalid_filter_is_explicit(mapping, errors): |
||||
with pytest.raises(QueryInputError) as caught: |
||||
TransactionFilters.from_mapping(mapping) |
||||
|
||||
assert caught.value.errors == errors |
||||
|
||||
|
||||
def test_filter_errors_are_reported_together_and_date_order_is_checked(): |
||||
with pytest.raises(QueryInputError) as caught: |
||||
TransactionFilters.from_mapping({ |
||||
"date_from": "2026-02-01", |
||||
"date_to": "2026-01-01", |
||||
"account": "bad", |
||||
}) |
||||
|
||||
assert caught.value.errors == { |
||||
"account": "invalid_uuid", |
||||
"date_to": "before_date_from", |
||||
} |
||||
|
||||
|
||||
def test_reference_lists_include_history_after_active_rows_with_stable_names(): |
||||
active_account = make_account("Synthetic Active") |
||||
archived_account = make_account("Synthetic Archived") |
||||
archived_account.archived = True |
||||
archived_account.save(update_fields=["archived"]) |
||||
active_category = Category.objects.create(name="Synthetic Active", kind=Category.Kind.EXPENSE) |
||||
archived_category = Category.objects.create(name="Synthetic Archived", kind=Category.Kind.EXPENSE, archived=True) |
||||
active_payee = Payee.objects.create(name="Synthetic Active") |
||||
archived_payee = Payee.objects.create(name="Synthetic Archived", archived=True) |
||||
|
||||
assert list(queries.listed_accounts()) == [active_account, archived_account] |
||||
assert list(queries.listed_categories()) == [active_category, archived_category] |
||||
assert list(queries.listed_payees()) == [active_payee, archived_payee] |
||||
assert list(queries.active_accounts()) == [active_account] |
||||
assert list(queries.active_categories()) == [active_category] |
||||
assert list(queries.active_payees()) == [active_payee] |
||||
@ -0,0 +1,122 @@
|
||||
from datetime import date |
||||
from decimal import Decimal |
||||
|
||||
import pytest |
||||
|
||||
from finance.models import Account, Category, Transaction |
||||
from finance.reports import ( |
||||
ReportInputError, |
||||
account_balances, |
||||
available_currencies, |
||||
monthly_summary, |
||||
spending_by_category, |
||||
) |
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db |
||||
|
||||
|
||||
def make_account(name="Synthetic Bank", *, kind=Account.Kind.BANK, currency="BRL"): |
||||
return Account.objects.create(name=name, kind=kind, currency=currency) |
||||
|
||||
|
||||
def post(account, amount, kind, posting_date, *, category=None): |
||||
return Transaction.objects.create( |
||||
account=account, |
||||
amount=Decimal(amount), |
||||
kind=kind, |
||||
posting_date=posting_date, |
||||
category=category, |
||||
) |
||||
|
||||
|
||||
def test_monthly_summary_counts_card_purchase_once_and_refund_in_posting_month(): |
||||
bank = make_account() |
||||
card = make_account("Synthetic Card", kind=Account.Kind.CREDIT_CARD) |
||||
post(bank, "500.00", Transaction.Kind.OPENING, date(2025, 12, 31)) |
||||
post(bank, "1000.00", Transaction.Kind.INCOME, date(2026, 1, 2)) |
||||
post(card, "-300.00", Transaction.Kind.EXPENSE, date(2026, 1, 3)) |
||||
post(bank, "-300.00", Transaction.Kind.TRANSFER, date(2026, 1, 20)) |
||||
post(card, "300.00", Transaction.Kind.TRANSFER, date(2026, 1, 20)) |
||||
post(card, "50.00", Transaction.Kind.REFUND, date(2026, 1, 25)) |
||||
post(card, "20.00", Transaction.Kind.REFUND, date(2026, 2, 1)) |
||||
|
||||
result = monthly_summary(date(2026, 1, 1), "BRL") |
||||
|
||||
assert result.inflow == Decimal("1000.00") |
||||
assert result.expense == Decimal("300.00") |
||||
assert result.refund == Decimal("50.00") |
||||
assert result.net_spending == Decimal("250.00") |
||||
assert result.net_activity == Decimal("750.00") |
||||
|
||||
|
||||
def test_spending_has_uncategorized_bucket_and_never_combines_currency(): |
||||
bank = make_account() |
||||
usd = make_account("Synthetic USD", currency="USD") |
||||
food = Category.objects.create(name="Synthetic Food", kind=Category.Kind.EXPENSE) |
||||
post(bank, "-100.00", Transaction.Kind.EXPENSE, date(2026, 1, 2), category=food) |
||||
post(bank, "20.00", Transaction.Kind.REFUND, date(2026, 1, 3), category=food) |
||||
post(bank, "-20.00", Transaction.Kind.EXPENSE, date(2026, 1, 4)) |
||||
post(usd, "-999.00", Transaction.Kind.EXPENSE, date(2026, 1, 5), category=food) |
||||
|
||||
rows = spending_by_category(date(2026, 1, 1), "BRL") |
||||
|
||||
assert [(row.category_name, row.net_spending) for row in rows] == [ |
||||
("Synthetic Food", Decimal("80.00")), |
||||
("Uncategorized", Decimal("20.00")), |
||||
] |
||||
assert rows[0].expense == Decimal("100.00") |
||||
assert rows[0].refund == Decimal("20.00") |
||||
|
||||
|
||||
def test_empty_month_returns_explicit_zero_totals_and_no_spending_rows(): |
||||
result = monthly_summary(date(2026, 1, 1), "BRL") |
||||
|
||||
assert result.inflow == Decimal("0.00") |
||||
assert result.expense == Decimal("0.00") |
||||
assert result.refund == Decimal("0.00") |
||||
assert result.net_spending == Decimal("0.00") |
||||
assert result.net_activity == Decimal("0.00") |
||||
assert spending_by_category(date(2026, 1, 1), "BRL") == () |
||||
|
||||
|
||||
def test_account_balances_include_openings_and_transfers_through_month_end(): |
||||
beta = make_account("Synthetic Beta") |
||||
alpha = make_account("Synthetic Alpha") |
||||
usd = make_account("Synthetic USD", currency="USD") |
||||
post(alpha, "100.00", Transaction.Kind.OPENING, date(2025, 12, 1)) |
||||
post(alpha, "-20.00", Transaction.Kind.TRANSFER, date(2026, 1, 20)) |
||||
post(beta, "20.00", Transaction.Kind.TRANSFER, date(2026, 1, 20)) |
||||
post(beta, "-5.00", Transaction.Kind.EXPENSE, date(2026, 2, 1)) |
||||
post(usd, "1000.00", Transaction.Kind.OPENING, date(2025, 12, 1)) |
||||
|
||||
rows = account_balances(date(2026, 1, 1), "BRL") |
||||
|
||||
assert [(row.account_name, row.balance) for row in rows] == [ |
||||
("Synthetic Alpha", Decimal("80.00")), |
||||
("Synthetic Beta", Decimal("20.00")), |
||||
] |
||||
|
||||
|
||||
def test_available_currencies_are_distinct_and_sorted(): |
||||
make_account("Synthetic BRL", currency="BRL") |
||||
make_account("Synthetic USD", currency="USD") |
||||
make_account("Synthetic BRL 2", currency="BRL") |
||||
|
||||
assert available_currencies() == ("BRL", "USD") |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
("month", "currency", "errors"), |
||||
[ |
||||
(date(2026, 1, 2), "BRL", {"month": "must_be_first_day"}), |
||||
(date(9999, 12, 1), "BRL", {"month": "invalid_month"}), |
||||
(date(2026, 1, 1), "brl", {"currency": "invalid_currency"}), |
||||
("2026-01", "BRL", {"month": "invalid_date"}), |
||||
], |
||||
) |
||||
def test_report_inputs_are_validated(month, currency, errors): |
||||
with pytest.raises(ReportInputError) as caught: |
||||
monthly_summary(month, currency) |
||||
|
||||
assert caught.value.errors == errors |
||||
@ -0,0 +1,316 @@
|
||||
"""Synthetic acceptance cases for the public, audited write boundary.""" |
||||
|
||||
from contextlib import contextmanager |
||||
from datetime import date |
||||
from decimal import Decimal |
||||
|
||||
import pytest |
||||
from django.db import transaction |
||||
from django.db.models import Sum |
||||
|
||||
from finance.models import AuditEvent, SourceRecord, Transaction, Transfer |
||||
from finance.services import ( |
||||
Actor, DomainError, annotate_transaction, archive, attach_source, |
||||
create_account, create_category, create_payee, create_plan, create_statement, |
||||
create_transfer, match_transfer, record_transaction, |
||||
) |
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True) |
||||
ACTOR = Actor(kind="AGENT", identifier="synthetic-test-agent") |
||||
DAY = date(2026, 1, 15) |
||||
|
||||
|
||||
def account(name="Synthetic Bank", kind="BANK", currency="BRL"): |
||||
return create_account(name=name, kind=kind, currency=currency, actor=ACTOR) |
||||
|
||||
|
||||
def post(account, amount="-120.00", kind="EXPENSE", **fields): |
||||
return record_transaction( |
||||
account_id=account.pk, amount=Decimal(amount), kind=kind, |
||||
posting_date=DAY, actor=ACTOR, **fields, |
||||
) |
||||
|
||||
|
||||
@contextmanager |
||||
def rejected_without_posting(): |
||||
"""A rejected write must leave movement intact and retain a failure audit.""" |
||||
before = Transaction.objects.count() |
||||
previous_audits = set(AuditEvent.objects.values_list("pk", flat=True)) |
||||
with pytest.raises(DomainError) as caught: |
||||
yield |
||||
assert caught.value.code |
||||
assert Transaction.objects.count() == before |
||||
events = list(AuditEvent.objects.exclude(pk__in=previous_audits)) |
||||
assert len(events) == 1 |
||||
assert events[0].outcome == "FAILURE" |
||||
assert events[0].error_code == caught.value.code |
||||
assert events[0].actor_kind == "AGENT" |
||||
assert events[0].actor_identifier == ACTOR.identifier |
||||
|
||||
|
||||
def test_opening_and_expense_derive_bank_balance(): |
||||
bank = account() |
||||
post(bank, "1000.00", "OPENING") |
||||
expense = post(bank, occurrence_date=date(2026, 1, 12)) |
||||
assert Transaction.objects.filter(account=bank).aggregate(total=Sum("amount"))["total"] == Decimal("880.00") |
||||
assert expense.posting_date == DAY |
||||
assert expense.occurrence_date == date(2026, 1, 12) |
||||
with rejected_without_posting(): |
||||
post(bank, "1.00", "OPENING") |
||||
|
||||
|
||||
@pytest.mark.parametrize("amount", [0.1, Decimal("0"), Decimal("NaN"), Decimal("Infinity"), Decimal("-Infinity"), Decimal("-0.001"), Decimal("10000000000000000"), Decimal("1e999999999")]) |
||||
def test_invalid_money_is_rejected_without_rounding(amount): |
||||
bank = account() |
||||
with rejected_without_posting(): |
||||
record_transaction(account_id=bank.pk, amount=amount, kind="EXPENSE", posting_date=DAY, actor=ACTOR) |
||||
|
||||
|
||||
@pytest.mark.parametrize("kind,amount", [("INCOME", "-1"), ("EXPENSE", "1"), ("REFUND", "-1"), ("UNKNOWN", "1")]) |
||||
def test_transaction_sign_matches_kind(kind, amount): |
||||
bank = account() |
||||
with rejected_without_posting(): |
||||
post(bank, amount, kind) |
||||
|
||||
|
||||
@pytest.mark.parametrize("currency", ["brl", "BR", "BRLL", "B1L", "ÉUR"]) |
||||
def test_account_currency_requires_three_uppercase_ascii_letters(currency): |
||||
with rejected_without_posting(): |
||||
account(currency=currency) |
||||
|
||||
|
||||
def test_category_kind_and_archived_assignments_are_validated(): |
||||
bank = account() |
||||
income = create_category(name="Synthetic Salary", kind="INCOME", actor=ACTOR) |
||||
expense = create_category(name="Synthetic Food", kind="EXPENSE", actor=ACTOR) |
||||
payee = create_payee(name="Synthetic Merchant", actor=ACTOR) |
||||
with rejected_without_posting(): |
||||
post(bank, category_id=income.pk) |
||||
with rejected_without_posting(): |
||||
post(bank, "10", "INCOME", category_id=expense.pk) |
||||
historical = post(bank, category_id=expense.pk, payee_id=payee.pk) |
||||
archive(expense, actor=ACTOR) |
||||
archive(payee, actor=ACTOR) |
||||
for fields in ({"category_id": expense.pk}, {"payee_id": payee.pk}): |
||||
with rejected_without_posting(): |
||||
post(bank, **fields) |
||||
updated = annotate_transaction(transaction_id=historical.pk, description="Synthetic revised label", actor=ACTOR) |
||||
assert updated.category_id == expense.pk |
||||
assert updated.payee_id == payee.pk |
||||
assert updated.description == "Synthetic revised label" |
||||
archive(bank, actor=ACTOR) |
||||
with rejected_without_posting(): |
||||
post(bank) |
||||
historical.refresh_from_db() |
||||
assert historical.amount == Decimal("-120.00") |
||||
|
||||
|
||||
def test_annotations_change_only_permitted_fields_and_audit_names(): |
||||
bank = account() |
||||
row = post(bank) |
||||
category = create_category(name="Synthetic Food", kind="EXPENSE", actor=ACTOR) |
||||
payee = create_payee(name="Synthetic Merchant", actor=ACTOR) |
||||
updated = annotate_transaction(transaction_id=row.pk, description="Synthetic sensitive label", payee_id=payee.pk, category_id=category.pk, actor=ACTOR) |
||||
assert updated.description == "Synthetic sensitive label" |
||||
assert updated.category_id == category.pk |
||||
assert updated.payee_id == payee.pk |
||||
event = AuditEvent.objects.filter(target_id=row.pk).latest("created_at") |
||||
assert event.outcome == "SUCCESS" |
||||
assert set(event.changed_fields) == {"description", "payee", "category"} |
||||
assert "Synthetic sensitive label" not in str(event.__dict__) |
||||
cleared = annotate_transaction(transaction_id=row.pk, payee_id=None, category_id=None, actor=ACTOR) |
||||
assert cleared.payee_id is None and cleared.category_id is None |
||||
|
||||
|
||||
@pytest.mark.parametrize("field,value", [("amount", Decimal("-99")), ("account_id", None), ("posting_date", date(2026, 2, 1)), ("kind", "INCOME"), ("refund_of_id", None), ("installment_number", 1)]) |
||||
def test_annotation_api_rejects_financial_changes(field, value): |
||||
row = post(account()) |
||||
with pytest.raises((TypeError, DomainError)): |
||||
annotate_transaction(transaction_id=row.pk, actor=ACTOR, **{field: value}) |
||||
row.refresh_from_db() |
||||
assert row.amount == Decimal("-120.00") |
||||
assert row.posting_date == DAY |
||||
assert row.kind == "EXPENSE" |
||||
|
||||
|
||||
def test_card_payment_adds_no_spending_and_preserves_settlement_dates(): |
||||
bank = account() |
||||
card = account("Synthetic Card", "CREDIT_CARD") |
||||
post(bank, "1000", "OPENING") |
||||
post(card, "-300") |
||||
payment = create_transfer(outgoing_account_id=bank.pk, incoming_account_id=card.pk, amount=Decimal("300"), outgoing_date=DAY, incoming_date=date(2026, 1, 17), actor=ACTOR) |
||||
assert payment.outgoing.amount == Decimal("-300") |
||||
assert payment.incoming.amount == Decimal("300") |
||||
assert payment.outgoing.posting_date == DAY |
||||
assert payment.incoming.posting_date == date(2026, 1, 17) |
||||
assert Transaction.objects.filter(kind="EXPENSE").aggregate(total=Sum("amount"))["total"] == Decimal("-300") |
||||
assert Transaction.objects.filter(account=card).aggregate(total=Sum("amount"))["total"] == Decimal("0") |
||||
assert Transaction.objects.count() == 4 |
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ["same_account", "different_currency", "negative_amount"]) |
||||
def test_invalid_complete_transfer_is_atomic(scenario): |
||||
bank = account() |
||||
other = bank if scenario == "same_account" else account("Synthetic Other", currency="USD" if scenario == "different_currency" else "BRL") |
||||
with rejected_without_posting(): |
||||
create_transfer(outgoing_account_id=bank.pk, incoming_account_id=other.pk, amount=Decimal("-10" if scenario == "negative_amount" else "10"), outgoing_date=DAY, incoming_date=DAY, actor=ACTOR) |
||||
assert Transfer.objects.count() == 0 |
||||
|
||||
|
||||
def test_match_existing_transfer_legs_creates_no_extra_transactions(): |
||||
outgoing = post(account(), "-25", "TRANSFER") |
||||
incoming = post(account("Synthetic Other"), "25", "TRANSFER") |
||||
match = match_transfer(outgoing_id=outgoing.pk, incoming_id=incoming.pk, actor=ACTOR) |
||||
assert match.outgoing_id == outgoing.pk |
||||
assert match.incoming_id == incoming.pk |
||||
assert Transaction.objects.count() == 2 |
||||
with rejected_without_posting(): |
||||
match_transfer(outgoing_id=outgoing.pk, incoming_id=incoming.pk, actor=ACTOR) |
||||
assert Transfer.objects.count() == 1 |
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ["same_account", "currency", "sign", "magnitude", "kind"]) |
||||
def test_matching_validates_both_existing_legs(scenario): |
||||
bank = account() |
||||
other = bank if scenario == "same_account" else account("Synthetic Other", currency="USD" if scenario == "currency" else "BRL") |
||||
outgoing = post(bank, "-25", "EXPENSE" if scenario == "kind" else "TRANSFER") |
||||
incoming = post(other, "-25" if scenario == "sign" else "26" if scenario == "magnitude" else "25", "TRANSFER") |
||||
with rejected_without_posting(): |
||||
match_transfer(outgoing_id=outgoing.pk, incoming_id=incoming.pk, actor=ACTOR) |
||||
assert Transfer.objects.count() == 0 |
||||
|
||||
|
||||
def test_refunds_retain_purchase_and_allow_cross_account_partial_adjustments(): |
||||
bank = account() |
||||
category = create_category(name="Synthetic Food", kind="EXPENSE", actor=ACTOR) |
||||
purchase = post(bank, category_id=category.pk) |
||||
other = account("Synthetic Other") |
||||
refund = post(other, "50", "REFUND", refund_of_id=purchase.pk, category_id=category.pk) |
||||
assert refund.refund_of_id == purchase.pk |
||||
assert Transaction.objects.filter(kind__in=["EXPENSE", "REFUND"]).aggregate(total=Sum("amount"))["total"] == Decimal("-70") |
||||
post(other, "100", "REFUND", refund_of_id=purchase.pk) |
||||
unlinked = post(other, "5", "REFUND") |
||||
assert unlinked.refund_of_id is None |
||||
purchase.refresh_from_db() |
||||
assert purchase.amount == Decimal("-120") |
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ["currency", "target_kind", "wrong_row_kind"]) |
||||
def test_invalid_refund_links_are_rejected(scenario): |
||||
bank = account() |
||||
original = post(bank, "20" if scenario == "target_kind" else "-20", "INCOME" if scenario == "target_kind" else "EXPENSE") |
||||
other = account("Synthetic Other", currency="USD" if scenario == "currency" else "BRL") |
||||
with rejected_without_posting(): |
||||
post(other, "5", "INCOME" if scenario == "wrong_row_kind" else "REFUND", refund_of_id=original.pk) |
||||
|
||||
|
||||
def test_only_observed_installments_are_posted_and_numbers_are_unique_per_plan(): |
||||
card = account("Synthetic Card", "CREDIT_CARD") |
||||
first = create_plan(account_id=card.pk, description="Synthetic purchase", total_installments=10, actor=ACTOR) |
||||
second = create_plan(account_id=card.pk, description="Synthetic second purchase", total_installments=10, actor=ACTOR) |
||||
row = post(card, "-30.01", installment_plan_id=first.pk, installment_number=3) |
||||
assert Transaction.objects.count() == 1 |
||||
with rejected_without_posting(): |
||||
post(card, "-30.01", installment_plan_id=first.pk, installment_number=3) |
||||
post(card, "-30.00", installment_plan_id=first.pk, installment_number=4) |
||||
post(card, "-30.01", installment_plan_id=second.pk, installment_number=3) |
||||
refund = post(card, "5", "REFUND", refund_of_id=row.pk) |
||||
assert refund.refund_of_id == row.pk |
||||
assert Transaction.objects.filter(kind="EXPENSE").count() == 3 |
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ["account", "zero", "overflow", "missing_number", "missing_plan", "kind"]) |
||||
def test_invalid_installment_membership_is_rejected(scenario): |
||||
bank = account() |
||||
plan = create_plan(account_id=bank.pk, description="Synthetic purchase", total_installments=10, actor=ACTOR) |
||||
destination = account("Synthetic Other") if scenario == "account" else bank |
||||
fields = {"installment_plan_id": plan.pk, "installment_number": 3} |
||||
if scenario == "missing_number": |
||||
fields.pop("installment_number") |
||||
elif scenario == "missing_plan": |
||||
fields.pop("installment_plan_id") |
||||
elif scenario in {"zero", "overflow"}: |
||||
fields["installment_number"] = 0 if scenario == "zero" else 11 |
||||
with rejected_without_posting(): |
||||
post(destination, "10" if scenario == "kind" else "-10", "INCOME" if scenario == "kind" else "EXPENSE", **fields) |
||||
|
||||
|
||||
def test_statement_metadata_does_not_manufacture_transactions_or_reconciliation(): |
||||
card = account("Synthetic Card", "CREDIT_CARD") |
||||
statement = create_statement(account_id=card.pk, period_start=date(2026, 1, 1), period_end=date(2026, 1, 31), due_date=date(2026, 2, 10), statement_balance=Decimal("-900"), actor=ACTOR) |
||||
assert Transaction.objects.count() == 0 |
||||
row = post(card, "-30", statement_id=statement.pk) |
||||
assert row.statement_id == statement.pk |
||||
statement.refresh_from_db() |
||||
assert statement.statement_balance == Decimal("-900") |
||||
for destination in (account(), account("Synthetic Other Card", "CREDIT_CARD")): |
||||
with rejected_without_posting(): |
||||
post(destination, statement_id=statement.pk) |
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ["bank", "period", "due"]) |
||||
def test_statement_creation_validates_account_and_dates(scenario): |
||||
target = account(kind="BANK" if scenario == "bank" else "CREDIT_CARD") |
||||
with rejected_without_posting(): |
||||
create_statement(account_id=target.pk, period_start=date(2026, 2, 1) if scenario == "period" else date(2026, 1, 1), period_end=date(2026, 1, 31), due_date=date(2026, 1, 30) if scenario == "due" else date(2026, 2, 10), actor=ACTOR) |
||||
|
||||
|
||||
def test_source_identity_is_physical_item_and_supports_multiple_sources_per_transaction(): |
||||
row = post(account()) |
||||
other = post(row.account, "-40") |
||||
metadata = dict(document_hash="a" * 64, source_format="BANK_CSV", parser_version="synthetic-v1", item_locator="row:2", actor=ACTOR) |
||||
original = attach_source(transaction_id=row.pk, **metadata) |
||||
assert original.transaction_id == row.pk |
||||
with rejected_without_posting(): |
||||
attach_source(transaction_id=other.pk, **{**metadata, "parser_version": "synthetic-v2"}) |
||||
attach_source(transaction_id=row.pk, **{**metadata, "item_locator": "row:3"}) |
||||
attach_source(transaction_id=row.pk, **{**metadata, "document_hash": "b" * 64}) |
||||
assert SourceRecord.objects.filter(transaction=row).count() == 3 |
||||
assert SourceRecord.objects.filter(transaction=other).count() == 0 |
||||
assert Transaction.objects.count() == 2 |
||||
|
||||
|
||||
@pytest.mark.parametrize("fields", [{"document_hash": "invalid"}, {"source_format": "OFX"}, {"parser_version": " "}, {"item_locator": " "}]) |
||||
def test_invalid_source_metadata_is_rejected(fields): |
||||
row = post(account()) |
||||
values = dict(document_hash="a" * 64, source_format="BANK_CSV", parser_version="synthetic-v1", item_locator="row:2") |
||||
with rejected_without_posting(): |
||||
attach_source(transaction_id=row.pk, actor=ACTOR, **{**values, **fields}) |
||||
assert SourceRecord.objects.count() == 0 |
||||
|
||||
|
||||
def test_public_writes_refuse_caller_owned_atomic_blocks(): |
||||
bank = account() |
||||
before = Transaction.objects.count() |
||||
with transaction.atomic(): |
||||
with pytest.raises((DomainError, RuntimeError)): |
||||
post(bank) |
||||
assert Transaction.objects.count() == before |
||||
|
||||
|
||||
def test_complete_transfer_preserves_amount_under_low_decimal_precision(): |
||||
from decimal import localcontext |
||||
|
||||
bank = account() |
||||
other = account("Synthetic Other") |
||||
with localcontext() as context: |
||||
context.prec = 4 |
||||
payment = create_transfer(outgoing_account_id=bank.pk, incoming_account_id=other.pk, |
||||
amount=Decimal("123.45"), outgoing_date=DAY, |
||||
incoming_date=DAY, actor=ACTOR) |
||||
payment.outgoing.refresh_from_db() |
||||
payment.incoming.refresh_from_db() |
||||
assert payment.outgoing.amount == Decimal("-123.45") |
||||
assert payment.incoming.amount == Decimal("123.45") |
||||
|
||||
|
||||
def test_matching_rejects_amounts_that_only_match_after_context_rounding(): |
||||
from decimal import localcontext |
||||
|
||||
outgoing = post(account(), "-123.40", "TRANSFER") |
||||
incoming = post(account("Synthetic Other"), "123.45", "TRANSFER") |
||||
with localcontext() as context: |
||||
context.prec = 4 |
||||
with rejected_without_posting(): |
||||
match_transfer(outgoing_id=outgoing.pk, incoming_id=incoming.pk, actor=ACTOR) |
||||
assert Transfer.objects.count() == 0 |
||||
@ -0,0 +1,12 @@
|
||||
# Synthetic import fixtures |
||||
|
||||
Every name, amount, date, and identifier in this directory is invented for |
||||
tests. The files model the three versioned import profiles and contain no data |
||||
copied from Robopato or a financial institution. |
||||
|
||||
- `bank.csv`: UTF-8, comma-separated columns with date at index 0, |
||||
description at index 2, and signed decimal amount at index 5. |
||||
- `full.txt`: full `DD.MM.YYYY` Ourocard-style fixed-width rows, including a |
||||
credit adjustment and account-payment line. |
||||
- `partial.txt`: yearless `DD/MM` fixed-width rows crossing a year boundary; |
||||
parsing requires explicit start and end dates. |
||||
|
@ -0,0 +1,4 @@
|
||||
1 - SYNTHETIC OWNER |
||||
15.02.2026SYNTHETIC SHOP BR 12,34 0,00 |
||||
16.02.2026SYNTHETIC REFUND BR -8,11 0,00 |
||||
17.02.2026PGTO DEBITO CONTA 200211 33,00 0,00 |
||||
@ -0,0 +1,3 @@
|
||||
1 - SYNTHETIC OWNER |
||||
31/12SYNTHETIC PARTIAL BR 12,34 0,00 |
||||
01/01SYNTHETIC NEW YEAR BR 12,34 0,00 |
||||
@ -0,0 +1,565 @@
|
||||
import json |
||||
import os |
||||
import subprocess |
||||
from contextlib import contextmanager |
||||
from hashlib import sha256 |
||||
from pathlib import Path |
||||
from types import SimpleNamespace |
||||
|
||||
import pytest |
||||
import psycopg |
||||
from django.db import connection |
||||
from psycopg import connect, sql |
||||
|
||||
from core.backup import ( |
||||
BACKUP_FORMAT, |
||||
BACKUP_VERSION, |
||||
BackupError, |
||||
DatabaseSnapshot, |
||||
DatabaseTarget, |
||||
create_backup, |
||||
verify_backup, |
||||
) |
||||
|
||||
|
||||
class RecordingRunner: |
||||
def __init__(self, output=b"PGDMP synthetic"): |
||||
self.calls = [] |
||||
self.output = output |
||||
|
||||
def __call__(self, argv, *, env, check, stdout=None, capture_output=False): |
||||
self.calls.append((list(argv), dict(env), check, stdout, capture_output)) |
||||
if argv[0] == "pg_dump" and argv != ["pg_dump", "--version"]: |
||||
output_name = next(value.removeprefix("--file=") for value in argv if value.startswith("--file=")) |
||||
Path(output_name).write_bytes(self.output) |
||||
return SimpleNamespace(stdout=b"pg_dump (PostgreSQL) 17.6\n") |
||||
|
||||
|
||||
@pytest.fixture |
||||
def database_settings(settings, monkeypatch): |
||||
configured = { |
||||
"ENGINE": "django.db.backends.postgresql", |
||||
"HOST": "live-db", |
||||
"PORT": "5432", |
||||
"NAME": "gizmoduck", |
||||
"USER": "gizmoduck_migrate", |
||||
"PASSWORD": "synthetic-secret", |
||||
} |
||||
monkeypatch.setitem(settings.DATABASES, "default", configured) |
||||
return configured |
||||
|
||||
|
||||
@pytest.fixture |
||||
def backup_root(tmp_path, settings): |
||||
settings.BACKUP_ROOT = tmp_path |
||||
return tmp_path |
||||
|
||||
|
||||
@pytest.fixture |
||||
def synthetic_snapshot(monkeypatch): |
||||
snapshot = DatabaseSnapshot( |
||||
migrations=(("finance", "0001_initial"), ("ingestion", "0001_initial")), |
||||
row_counts={"finance_account": 2, "ingestion_importbatch": 1}, |
||||
) |
||||
@contextmanager |
||||
def exported_snapshot(_target): |
||||
yield "synthetic-snapshot", snapshot |
||||
|
||||
monkeypatch.setattr("core.backup._exported_snapshot", exported_snapshot) |
||||
monkeypatch.setattr("core.backup._read_snapshot", lambda _target: snapshot) |
||||
return snapshot |
||||
|
||||
|
||||
def test_pg_dump_receives_password_only_in_environment( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
runner = RecordingRunner() |
||||
|
||||
result = create_backup("synthetic.dump", runner=runner) |
||||
|
||||
argv, env, check, stdout, capture_output = runner.calls[0] |
||||
assert argv[:4] == ["pg_dump", "--format=custom", "--no-owner", "--no-acl"] |
||||
assert database_settings["PASSWORD"] not in argv |
||||
assert env["PGPASSWORD"] == database_settings["PASSWORD"] |
||||
assert check is True |
||||
assert stdout is None |
||||
assert capture_output is True |
||||
assert result.path.read_bytes() == runner.output |
||||
|
||||
|
||||
def test_pg_dump_uses_the_exported_snapshot( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
runner = RecordingRunner() |
||||
|
||||
create_backup("synthetic.dump", runner=runner) |
||||
|
||||
assert "--snapshot=synthetic-snapshot" in runner.calls[0][0] |
||||
assert runner.calls[0][4] is True |
||||
|
||||
|
||||
def test_backup_writes_versioned_checksum_manifest( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
runner = RecordingRunner() |
||||
|
||||
result = create_backup("synthetic.dump", runner=runner) |
||||
|
||||
manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) |
||||
assert manifest == { |
||||
"application_version": "0.1.0", |
||||
"byte_size": len(runner.output), |
||||
"created_at": result.created_at.isoformat(), |
||||
"database_name": "gizmoduck", |
||||
"format": BACKUP_FORMAT, |
||||
"migrations": [ |
||||
["finance", "0001_initial"], |
||||
["ingestion", "0001_initial"], |
||||
], |
||||
"postgresql_client_version": result.postgresql_client_version, |
||||
"row_counts": { |
||||
"finance_account": 2, |
||||
"ingestion_importbatch": 1, |
||||
}, |
||||
"sha256": sha256(runner.output).hexdigest(), |
||||
"version": BACKUP_VERSION, |
||||
} |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"name", |
||||
["../outside.dump", "nested/file.dump", "nested\\file.dump", ".dump", "file.zip"], |
||||
) |
||||
def test_backup_rejects_non_plain_dump_names( |
||||
backup_root, database_settings, synthetic_snapshot, name |
||||
): |
||||
with pytest.raises(BackupError, match="invalid_name"): |
||||
create_backup(name, runner=RecordingRunner()) |
||||
|
||||
assert list(backup_root.iterdir()) == [] |
||||
|
||||
|
||||
def test_backup_does_not_replace_existing_artifacts( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
dump = backup_root / "synthetic.dump" |
||||
manifest = backup_root / "synthetic.dump.json" |
||||
dump.write_bytes(b"existing dump") |
||||
manifest.write_text("existing manifest", encoding="utf-8") |
||||
|
||||
with pytest.raises(BackupError, match="existing_destination"): |
||||
create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
|
||||
assert dump.read_bytes() == b"existing dump" |
||||
assert manifest.read_text(encoding="utf-8") == "existing manifest" |
||||
|
||||
|
||||
def test_backup_cleans_private_temporary_files_on_dump_failure( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
def failing_runner(argv, **kwargs): |
||||
raise OSError("synthetic private detail") |
||||
|
||||
with pytest.raises(BackupError, match="backup_failed"): |
||||
create_backup("synthetic.dump", runner=failing_runner) |
||||
|
||||
assert list(backup_root.iterdir()) == [] |
||||
|
||||
|
||||
def test_backup_sanitizes_subprocess_failure_and_captures_diagnostics( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
def failing_runner(argv, *, capture_output, **kwargs): |
||||
assert capture_output is True |
||||
raise subprocess.CalledProcessError( |
||||
1, argv, output=b"SYNTHETIC PRIVATE STDOUT", stderr=b"SYNTHETIC PRIVATE STDERR" |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="backup_failed") as error: |
||||
create_backup("synthetic.dump", runner=failing_runner) |
||||
|
||||
assert "PRIVATE" not in str(error.value) |
||||
assert list(backup_root.iterdir()) == [] |
||||
|
||||
|
||||
def test_second_temporary_unlink_failure_preserves_complete_committed_pair( |
||||
backup_root, database_settings, synthetic_snapshot, monkeypatch |
||||
): |
||||
original_unlink = Path.unlink |
||||
failed = False |
||||
|
||||
def fail_once(path, *args, **kwargs): |
||||
nonlocal failed |
||||
if not failed and path.name.endswith(".dump.tmp"): |
||||
failed = True |
||||
raise OSError("synthetic unlink failure") |
||||
return original_unlink(path, *args, **kwargs) |
||||
|
||||
monkeypatch.setattr(Path, "unlink", fail_once) |
||||
|
||||
with pytest.raises(BackupError, match="cleanup_failed"): |
||||
create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
|
||||
assert (backup_root / "synthetic.dump").read_bytes() == b"PGDMP synthetic" |
||||
assert (backup_root / "synthetic.dump.json").is_file() |
||||
assert sorted(path.name for path in backup_root.iterdir()) == [ |
||||
"synthetic.dump", |
||||
"synthetic.dump.json", |
||||
] |
||||
|
||||
|
||||
def test_directory_fsync_failure_rolls_back_both_owned_finals( |
||||
backup_root, database_settings, synthetic_snapshot, monkeypatch |
||||
): |
||||
monkeypatch.setattr( |
||||
"core.backup._fsync_directory", |
||||
lambda _root: (_ for _ in ()).throw(OSError("synthetic fsync failure")), |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="publication_failed"): |
||||
create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
|
||||
assert list(backup_root.iterdir()) == [] |
||||
|
||||
|
||||
def test_publication_never_deletes_concurrent_final_files(tmp_path, monkeypatch): |
||||
import core.backup as backup |
||||
|
||||
manifest_temp = tmp_path / ".manifest.tmp" |
||||
dump_temp = tmp_path / ".dump.tmp" |
||||
manifest_final = tmp_path / "synthetic.dump.json" |
||||
dump_final = tmp_path / "synthetic.dump" |
||||
manifest_temp.write_bytes(b"our manifest") |
||||
dump_temp.write_bytes(b"our dump") |
||||
original_link = os.link |
||||
calls = 0 |
||||
|
||||
def competing_link(source, destination): |
||||
nonlocal calls |
||||
calls += 1 |
||||
if calls == 2: |
||||
manifest_final.unlink() |
||||
manifest_final.write_bytes(b"other manifest") |
||||
dump_final.write_bytes(b"other dump") |
||||
raise FileExistsError |
||||
return original_link(source, destination) |
||||
|
||||
monkeypatch.setattr(os, "link", competing_link) |
||||
|
||||
with pytest.raises(BackupError, match="publication_failed"): |
||||
backup._publish_pair( |
||||
tmp_path, |
||||
((manifest_temp, manifest_final), (dump_temp, dump_final)), |
||||
) |
||||
|
||||
assert manifest_final.read_bytes() == b"other manifest" |
||||
assert dump_final.read_bytes() == b"other dump" |
||||
assert sorted(path.name for path in tmp_path.iterdir()) == [ |
||||
"synthetic.dump", |
||||
"synthetic.dump.json", |
||||
] |
||||
|
||||
|
||||
def test_verification_rejects_live_database( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
target = DatabaseTarget.from_default(database_settings) |
||||
|
||||
with pytest.raises(BackupError, match="live_target"): |
||||
verify_backup("synthetic.dump", target, runner=RecordingRunner()) |
||||
|
||||
|
||||
def test_verification_requires_explicit_allow_flag( |
||||
backup_root, database_settings, synthetic_snapshot, settings |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = False |
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="restore_user", |
||||
password="restore-secret", |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="verification_not_allowed"): |
||||
verify_backup("synthetic.dump", target, runner=RecordingRunner()) |
||||
|
||||
|
||||
def test_verification_rejects_changed_dump_before_restore( |
||||
backup_root, database_settings, synthetic_snapshot, settings |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
result.path.write_bytes(b"PGDMP tampered!") |
||||
restore_runner = RecordingRunner() |
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="restore_user", |
||||
password="restore-secret", |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="digest_mismatch"): |
||||
verify_backup(result.name, target, runner=restore_runner) |
||||
|
||||
assert restore_runner.calls == [] |
||||
|
||||
|
||||
def test_verification_sanitizes_digest_read_failure( |
||||
backup_root, database_settings, synthetic_snapshot, settings, monkeypatch |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
monkeypatch.setattr( |
||||
"core.backup._file_digest", |
||||
lambda _path: (_ for _ in ()).throw(OSError("SYNTHETIC PRIVATE PATH")), |
||||
) |
||||
target = DatabaseTarget( |
||||
host="restore-db", port="5432", name="restore", user="user", password="secret" |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="verification_failed") as error: |
||||
verify_backup(result.name, target, runner=RecordingRunner()) |
||||
|
||||
assert "PRIVATE" not in str(error.value) |
||||
|
||||
|
||||
def test_verification_rejects_manifest_with_wrong_field_type( |
||||
backup_root, database_settings, synthetic_snapshot, settings |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) |
||||
manifest["application_version"] = 1 |
||||
result.manifest_path.write_text(json.dumps(manifest), encoding="utf-8") |
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="restore_user", |
||||
password="restore-secret", |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="invalid_manifest"): |
||||
verify_backup(result.name, target, runner=RecordingRunner()) |
||||
|
||||
|
||||
def test_backup_rejects_nonmatching_postgresql_client_major( |
||||
backup_root, database_settings, synthetic_snapshot |
||||
): |
||||
class WrongVersionRunner(RecordingRunner): |
||||
def __call__(self, argv, **kwargs): |
||||
result = super().__call__(argv, **kwargs) |
||||
if argv == ["pg_dump", "--version"]: |
||||
result.stdout = b"pg_dump (PostgreSQL) 16.9\n" |
||||
return result |
||||
|
||||
with pytest.raises(BackupError, match="client_version_failed"): |
||||
create_backup("synthetic.dump", runner=WrongVersionRunner()) |
||||
|
||||
assert list(backup_root.iterdir()) == [] |
||||
|
||||
|
||||
def test_pg_restore_receives_password_only_in_environment( |
||||
backup_root, database_settings, synthetic_snapshot, settings, monkeypatch |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
snapshots = iter( |
||||
[DatabaseSnapshot(migrations=(), row_counts={}), synthetic_snapshot] |
||||
) |
||||
monkeypatch.setattr("core.backup._read_snapshot", lambda _target: next(snapshots)) |
||||
runner = RecordingRunner() |
||||
target = DatabaseTarget( |
||||
host="restore-db", |
||||
port="5432", |
||||
name="gizmoduck_restore", |
||||
user="restore_user", |
||||
password="synthetic-restore-secret", |
||||
) |
||||
|
||||
verification = verify_backup(result.name, target, runner=runner) |
||||
|
||||
argv, env, check, stdout, capture_output = runner.calls[0] |
||||
assert argv[:5] == [ |
||||
"pg_restore", |
||||
"--exit-on-error", |
||||
"--single-transaction", |
||||
"--no-owner", |
||||
"--no-acl", |
||||
] |
||||
assert target.password not in argv |
||||
assert env["PGPASSWORD"] == target.password |
||||
assert check is True |
||||
assert stdout is None |
||||
assert capture_output is True |
||||
assert verification.row_counts == synthetic_snapshot.row_counts |
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True) |
||||
def test_snapshot_counts_tables_in_non_system_schemas(settings): |
||||
from core.backup import _read_snapshot |
||||
|
||||
target = DatabaseTarget.from_default(settings.DATABASES["default"]) |
||||
with connection.cursor() as cursor: |
||||
cursor.execute("CREATE SCHEMA synthetic_extra") |
||||
cursor.execute("CREATE TABLE synthetic_extra.marker (id integer)") |
||||
cursor.execute("INSERT INTO synthetic_extra.marker VALUES (1)") |
||||
try: |
||||
snapshot = _read_snapshot(target) |
||||
assert snapshot.row_counts["synthetic_extra.marker"] == 1 |
||||
finally: |
||||
with connection.cursor() as cursor: |
||||
cursor.execute("DROP SCHEMA synthetic_extra CASCADE") |
||||
|
||||
|
||||
def test_manifest_rejects_boolean_integer_fields( |
||||
backup_root, database_settings, synthetic_snapshot, settings |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) |
||||
manifest["byte_size"] = True |
||||
result.manifest_path.write_text(json.dumps(manifest), encoding="utf-8") |
||||
target = DatabaseTarget( |
||||
host="restore-db", port="5432", name="restore", user="user", password="secret" |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="invalid_manifest"): |
||||
verify_backup(result.name, target, runner=RecordingRunner()) |
||||
|
||||
|
||||
def test_manifest_rejects_boolean_version( |
||||
backup_root, database_settings, synthetic_snapshot, settings |
||||
): |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
result = create_backup("synthetic.dump", runner=RecordingRunner()) |
||||
manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) |
||||
manifest["version"] = True |
||||
result.manifest_path.write_text(json.dumps(manifest), encoding="utf-8") |
||||
target = DatabaseTarget( |
||||
host="restore-db", port="5432", name="restore", user="user", password="secret" |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="invalid_manifest"): |
||||
verify_backup(result.name, target, runner=RecordingRunner()) |
||||
|
||||
|
||||
def test_synthetic_preflight_database_error_is_sanitized(monkeypatch, settings): |
||||
import core.backup as backup |
||||
|
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
for name, value in { |
||||
"BACKUP_SOURCE_HOST": "source", |
||||
"BACKUP_SOURCE_NAME": "source", |
||||
"BACKUP_SOURCE_USER": "user", |
||||
"BACKUP_SOURCE_PASSWORD": "SYNTHETIC PRIVATE PASSWORD", |
||||
}.items(): |
||||
monkeypatch.setenv(name, value) |
||||
monkeypatch.setattr( |
||||
backup, |
||||
"_read_snapshot", |
||||
lambda _target: (_ for _ in ()).throw( |
||||
psycopg.OperationalError("SYNTHETIC PRIVATE DIAGNOSTIC") |
||||
), |
||||
) |
||||
|
||||
with pytest.raises(BackupError, match="integration_verification_failed") as error: |
||||
backup.run_synthetic_restore_verification() |
||||
|
||||
assert "PRIVATE" not in str(error.value) |
||||
|
||||
|
||||
def test_cleanup_attempts_every_resource_and_reports_safe_failure(monkeypatch, tmp_path): |
||||
import core.backup as backup |
||||
|
||||
attempts = [] |
||||
dump = tmp_path / "synthetic.dump" |
||||
manifest = tmp_path / "synthetic.dump.json" |
||||
dump.write_bytes(b"dump") |
||||
manifest.write_text("{}", encoding="utf-8") |
||||
artifact = SimpleNamespace(path=dump, manifest_path=manifest) |
||||
source = DatabaseTarget("source", "5432", "source", "user", "password") |
||||
target = DatabaseTarget("target", "5432", "target", "user", "password") |
||||
|
||||
original_unlink = Path.unlink |
||||
|
||||
def unlink(path, *args, **kwargs): |
||||
attempts.append(f"unlink:{path.name}") |
||||
if path == dump: |
||||
raise OSError("private path") |
||||
return original_unlink(path, *args, **kwargs) |
||||
|
||||
def clear(database): |
||||
attempts.append(f"clear:{database.name}") |
||||
if database == source: |
||||
raise BackupError("cleanup_failed") |
||||
|
||||
monkeypatch.setattr(Path, "unlink", unlink) |
||||
monkeypatch.setattr(backup, "_clear_disposable_database", clear) |
||||
|
||||
with pytest.raises(BackupError, match="cleanup_failed"): |
||||
backup._cleanup_synthetic_resources(artifact, source, target) |
||||
|
||||
assert attempts == [ |
||||
"unlink:synthetic.dump", |
||||
"unlink:synthetic.dump.json", |
||||
"clear:source", |
||||
"clear:target", |
||||
] |
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True) |
||||
def test_concurrent_insert_is_excluded_from_both_manifest_and_dump(tmp_path, settings): |
||||
from finance.models import Account |
||||
|
||||
settings.BACKUP_ROOT = tmp_path |
||||
settings.BACKUP_VERIFY_ALLOWED = True |
||||
source = DatabaseTarget.from_default(settings.DATABASES["default"]) |
||||
Account.objects.create(name="Synthetic Snapshot First", kind="BANK", currency="BRL") |
||||
inserted = False |
||||
|
||||
def runner(argv, **kwargs): |
||||
nonlocal inserted |
||||
if argv[0] == "pg_dump" and argv != ["pg_dump", "--version"]: |
||||
assert any(value.startswith("--snapshot=") for value in argv) |
||||
Account.objects.create( |
||||
name="Synthetic Snapshot Concurrent", kind="BANK", currency="BRL" |
||||
) |
||||
inserted = True |
||||
return subprocess.run(argv, **kwargs) |
||||
|
||||
database_name = f"snapshot_restore_{os.getpid()}" |
||||
admin = dict( |
||||
host=source.host, |
||||
port=source.port, |
||||
dbname="postgres", |
||||
user=source.user, |
||||
password=source.password, |
||||
autocommit=True, |
||||
) |
||||
with connect(**admin) as database: |
||||
database.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database_name))) |
||||
target = DatabaseTarget( |
||||
host=source.host, |
||||
port=source.port, |
||||
name=database_name, |
||||
user=source.user, |
||||
password=source.password, |
||||
) |
||||
try: |
||||
result = create_backup("concurrent.dump", runner=runner) |
||||
verification = verify_backup(result.name, target) |
||||
|
||||
assert inserted is True |
||||
assert result.row_counts["finance_account"] == 1 |
||||
assert verification.row_counts["finance_account"] == 1 |
||||
assert Account.objects.count() == 2 |
||||
finally: |
||||
with connect(**admin) as database: |
||||
database.execute( |
||||
sql.SQL("DROP DATABASE {} WITH (FORCE)").format( |
||||
sql.Identifier(database_name) |
||||
) |
||||
) |
||||
@ -0,0 +1,143 @@
|
||||
from datetime import datetime, timezone |
||||
from io import StringIO |
||||
from pathlib import Path |
||||
|
||||
import pytest |
||||
from django.core.management import call_command |
||||
from django.core.management.base import CommandError |
||||
|
||||
from core.backup import BackupError, BackupResult, VerificationResult |
||||
|
||||
|
||||
def backup_result(): |
||||
return BackupResult( |
||||
name="synthetic.dump", |
||||
path=Path("/private/backup/synthetic.dump"), |
||||
manifest_path=Path("/private/backup/synthetic.dump.json"), |
||||
version=1, |
||||
created_at=datetime(2026, 9, 8, tzinfo=timezone.utc), |
||||
postgresql_client_version="pg_dump (PostgreSQL) 17.6", |
||||
row_counts={"finance_account": 2, "finance_transaction": 4}, |
||||
) |
||||
|
||||
|
||||
def test_backup_database_prints_only_safe_summary(monkeypatch): |
||||
monkeypatch.setattr( |
||||
"core.management.commands.backup_database.create_backup", |
||||
lambda name: backup_result(), |
||||
) |
||||
stdout = StringIO() |
||||
|
||||
call_command("backup_database", output="synthetic.dump", stdout=stdout) |
||||
|
||||
assert stdout.getvalue().splitlines() == [ |
||||
"backup=synthetic.dump format=1", |
||||
"table=finance_account rows=2", |
||||
"table=finance_transaction rows=4", |
||||
] |
||||
assert "/private/backup" not in stdout.getvalue() |
||||
|
||||
|
||||
def test_backup_database_converts_service_error_to_safe_code(monkeypatch): |
||||
def fail(_name): |
||||
raise BackupError("backup_failed") |
||||
|
||||
monkeypatch.setattr("core.management.commands.backup_database.create_backup", fail) |
||||
|
||||
with pytest.raises(CommandError, match=r"^backup_failed$"): |
||||
call_command("backup_database", output="synthetic.dump") |
||||
|
||||
|
||||
def test_verify_backup_uses_configured_password_and_prints_safe_summary( |
||||
monkeypatch, settings |
||||
): |
||||
monkeypatch.setitem(settings.DATABASES, "default", { |
||||
"HOST": "restore-db", |
||||
"PORT": "5432", |
||||
"NAME": "gizmoduck_restore", |
||||
"USER": "restore_user", |
||||
"PASSWORD": "synthetic-restore-secret", |
||||
}) |
||||
captured = {} |
||||
|
||||
def verify(name, target): |
||||
captured["name"] = name |
||||
captured["target"] = target |
||||
return VerificationResult( |
||||
name=name, |
||||
version=1, |
||||
row_counts={"django_migrations": 12, "finance_account": 2}, |
||||
) |
||||
|
||||
monkeypatch.setattr("core.management.commands.verify_backup.verify_backup", verify) |
||||
stdout = StringIO() |
||||
|
||||
call_command( |
||||
"verify_backup", |
||||
input="synthetic.dump", |
||||
target_name="gizmoduck_restore", |
||||
target_host="restore-db", |
||||
target_user="restore_user", |
||||
stdout=stdout, |
||||
) |
||||
|
||||
assert captured["name"] == "synthetic.dump" |
||||
assert captured["target"].password == "synthetic-restore-secret" |
||||
assert stdout.getvalue().splitlines() == [ |
||||
"backup=synthetic.dump format=1 verified", |
||||
"table=django_migrations rows=12", |
||||
"table=finance_account rows=2", |
||||
] |
||||
assert "synthetic-restore-secret" not in stdout.getvalue() |
||||
|
||||
|
||||
def test_verify_backup_converts_service_error_to_safe_code(monkeypatch, settings): |
||||
monkeypatch.setitem(settings.DATABASES, "default", { |
||||
"HOST": "restore-db", |
||||
"PORT": "5432", |
||||
"NAME": "gizmoduck_restore", |
||||
"USER": "restore_user", |
||||
"PASSWORD": "synthetic-restore-secret", |
||||
}) |
||||
|
||||
def fail(_name, _target): |
||||
raise BackupError("target_not_empty") |
||||
|
||||
monkeypatch.setattr("core.management.commands.verify_backup.verify_backup", fail) |
||||
|
||||
with pytest.raises(CommandError, match=r"^target_not_empty$"): |
||||
call_command( |
||||
"verify_backup", |
||||
input="synthetic.dump", |
||||
target_name="gizmoduck_restore", |
||||
target_host="restore-db", |
||||
target_user="restore_user", |
||||
) |
||||
|
||||
|
||||
def test_backup_restore_acceptance_command_prints_only_safe_summary(monkeypatch): |
||||
calls = 0 |
||||
|
||||
def verify(): |
||||
nonlocal calls |
||||
calls += 1 |
||||
return VerificationResult( |
||||
name="synthetic-restore-check.dump", |
||||
version=1, |
||||
row_counts={"finance_account": 1, "finance_transaction": 1}, |
||||
) |
||||
|
||||
monkeypatch.setattr( |
||||
"core.management.commands.test_backup_restore.run_synthetic_restore_verification", |
||||
verify, |
||||
) |
||||
stdout = StringIO() |
||||
|
||||
call_command("test_backup_restore", stdout=stdout) |
||||
|
||||
assert calls == 2 |
||||
assert stdout.getvalue().splitlines() == [ |
||||
"backup=synthetic-restore-check.dump format=1 verified", |
||||
"table=finance_account rows=1", |
||||
"table=finance_transaction rows=1", |
||||
] |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue