Initial deploy setup

config/settings.py

@@ -0,0 +1,348 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+from django.core.exceptions import ImproperlyConfigured
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+def env_bool(name: str, default: bool = False) -> bool:
+    return os.getenv(name, "1" if default else "0").lower() in {"1", "true", "yes", "on"}
+
+
+def env_list(name: str, default: str = "") -> list[str]:
+    return [item.strip() for item in os.getenv(name, default).split(",") if item.strip()]
+
+
+def _validate_production_security(
+    *,
+    debug: bool,
+    secret_key: str,
+    allowed_hosts: list[str],
+    csrf_trusted_origins: list[str],
+    session_cookie_secure: bool,
+    csrf_cookie_secure: bool,
+    secure_ssl_redirect: bool,
+) -> None:
+    if debug:
+        return
+    if secret_key.strip().lower() in {"", "change-me", "dev-only-change-me", "secret", "insecure"}:
+        raise ImproperlyConfigured(
+            "Onveilige DJANGO_SECRET_KEY in productie. Stel een sterke waarde in via environment."
+        )
+    if len(secret_key.strip()) < 50:
+        raise ImproperlyConfigured(
+            "DJANGO_SECRET_KEY is te kort voor een productieomgeving."
+        )
+    if not allowed_hosts:
+        raise ImproperlyConfigured("ALLOWED_HOSTS mag in productie niet leeg zijn.")
+    if not csrf_trusted_origins:
+        raise ImproperlyConfigured(
+            "CSRF_TRUSTED_ORIGINS is verplicht bij DEBUG=False."
+        )
+    if not any(origin.lower().startswith("https://") for origin in csrf_trusted_origins):
+        raise ImproperlyConfigured(
+            "CSRF_TRUSTED_ORIGINS moet HTTPS-origins bevatten in productie."
+        )
+    if not session_cookie_secure or not csrf_cookie_secure:
+        raise ImproperlyConfigured(
+            "Session- en CSRF-cookies moeten Secure=True zijn in productie."
+        )
+    if not secure_ssl_redirect:
+        raise ImproperlyConfigured("SECURE_SSL_REDIRECT moet True zijn in productie.")
+
+
+SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "dev-only-change-me")
+DEBUG = env_bool("DJANGO_DEBUG", True)
+ALLOWED_HOSTS = env_list("DJANGO_ALLOWED_HOSTS", "localhost,127.0.0.1,testserver")
+CSRF_TRUSTED_ORIGINS = env_list("DJANGO_CSRF_TRUSTED_ORIGINS")
+TIME_ZONE = os.getenv("DJANGO_TIME_ZONE", "Europe/Brussels")
+LANGUAGE_CODE = "nl-be"
+USE_I18N = True
+USE_TZ = True
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "apps.core",
+    "apps.profiles",
+    "apps.sources",
+    "apps.jobs",
+    "apps.notifications",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "whitenoise.middleware.WhiteNoiseMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.locale.LocaleMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "apps.core.middleware.SecurityHeadersMiddleware",
+]
+
+ROOT_URLCONF = "config.urls"
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [BASE_DIR / "templates"],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+                "apps.core.context_processors.navigation_context",
+            ],
+        },
+    }
+]
+WSGI_APPLICATION = "config.wsgi.application"
+ASGI_APPLICATION = "config.asgi.application"
+
+DATABASE_URL = os.getenv("DATABASE_URL", "").strip()
+POSTGRES_DB = os.getenv("POSTGRES_DB", "vacatureradar")
+POSTGRES_USER = os.getenv("POSTGRES_USER", "vacatureradar")
+POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", "")
+POSTGRES_HOST = os.getenv("POSTGRES_HOST", "postgres")
+POSTGRES_PORT = os.getenv("POSTGRES_PORT", "5432")
+
+
+def _postgres_database_from_env() -> dict[str, str | int]:
+    if POSTGRES_PASSWORD.lower().startswith("change_me") or POSTGRES_PASSWORD == "CHANGE_ME":
+        return {}
+    if not (POSTGRES_PASSWORD and POSTGRES_HOST and POSTGRES_DB and POSTGRES_USER):
+        return {}
+    return {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": POSTGRES_DB,
+        "USER": POSTGRES_USER,
+        "PASSWORD": POSTGRES_PASSWORD,
+        "HOST": POSTGRES_HOST,
+        "PORT": POSTGRES_PORT,
+        "CONN_MAX_AGE": 60,
+        "OPTIONS": {"connect_timeout": 10},
+    }
+
+
+if DATABASE_URL:
+    parsed = urlparse(DATABASE_URL)
+    DATABASES = {
+        "default": {
+            "ENGINE": "django.db.backends.postgresql",
+            "NAME": parsed.path.lstrip("/"),
+            "USER": parsed.username,
+            "PASSWORD": parsed.password,
+            "HOST": parsed.hostname,
+            "PORT": parsed.port or 5432,
+            "CONN_MAX_AGE": 60,
+            "OPTIONS": {"connect_timeout": 10},
+        }
+    }
+elif (postgres_cfg := _postgres_database_from_env()):
+    DATABASES = {"default": postgres_cfg}
+else:
+    DATABASES = {
+        "default": {
+            "ENGINE": "django.db.backends.sqlite3",
+            "NAME": BASE_DIR / "local" / "db.sqlite3",
+        }
+    }
+
+AUTH_PASSWORD_VALIDATORS = [
+    {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
+    {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
+    {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
+    {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
+]
+
+STATIC_URL = "/static/"
+STATIC_ROOT = BASE_DIR / "staticfiles"
+STATICFILES_DIRS = [BASE_DIR / "static"]
+STORAGES = {
+    "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"},
+    "staticfiles": {
+        "BACKEND": (
+            "whitenoise.storage.CompressedStaticFilesStorage"
+            if DEBUG
+            else "whitenoise.storage.CompressedManifestStaticFilesStorage"
+        )
+    },
+}
+MEDIA_URL = "/media/"
+MEDIA_ROOT = BASE_DIR / "media"
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+LOGIN_URL = "login"
+LOGIN_REDIRECT_URL = "dashboard:today"
+LOGOUT_REDIRECT_URL = "login"
+
+EMAIL_BACKEND = os.getenv("EMAIL_BACKEND", "django.core.mail.backends.console.EmailBackend")
+DEFAULT_FROM_EMAIL = os.getenv("DEFAULT_FROM_EMAIL", "VacatureRadar <vacatureradar@localhost>")
+EMAIL_HOST = os.getenv("SMTP_HOST", "")
+EMAIL_PORT = int(os.getenv("SMTP_PORT", "587"))
+EMAIL_HOST_USER = os.getenv("SMTP_USER", "")
+EMAIL_HOST_PASSWORD = os.getenv("SMTP_PASSWORD", "")
+EMAIL_USE_TLS = env_bool("SMTP_USE_TLS", True)
+
+SESSION_COOKIE_HTTPONLY = True
+SESSION_COOKIE_SAMESITE = "Lax"
+SESSION_COOKIE_SECURE = env_bool("SESSION_COOKIE_SECURE", False)
+CSRF_COOKIE_SECURE = env_bool("CSRF_COOKIE_SECURE", False)
+CSRF_COOKIE_SAMESITE = "Lax"
+SECURE_SSL_REDIRECT = env_bool("SECURE_SSL_REDIRECT", False)
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+SECURE_HSTS_SECONDS = int(os.getenv("SECURE_HSTS_SECONDS", "0"))
+SECURE_HSTS_INCLUDE_SUBDOMAINS = env_bool("SECURE_HSTS_INCLUDE_SUBDOMAINS", True)
+SECURE_HSTS_PRELOAD = env_bool("SECURE_HSTS_PRELOAD", False)
+USE_X_FORWARDED_HOST = True
+SECURE_CONTENT_TYPE_NOSNIFF = True
+SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
+X_FRAME_OPTIONS = "DENY"
+
+CACHES = {
+    "default": {
+        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        "LOCATION": "vacatureradar-rate-limit",
+    }
+}
+
+REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
+CELERY_BROKER_URL = REDIS_URL
+CELERY_RESULT_BACKEND = REDIS_URL
+CELERY_TASK_ALWAYS_EAGER = env_bool("CELERY_TASK_ALWAYS_EAGER", False)
+CELERY_TASK_EAGER_PROPAGATES = True
+CELERY_TASK_ACKS_LATE = True
+CELERY_WORKER_PREFETCH_MULTIPLIER = 1
+CELERY_TASK_TIME_LIMIT = 300
+CELERY_TASK_SOFT_TIME_LIMIT = 270
+CELERY_TIMEZONE = TIME_ZONE
+CELERY_BEAT_SCHEDULE = {
+    "poll-imap": {
+        "task": "apps.sources.tasks.poll_imap_inbox",
+        "schedule": 600.0,
+    },
+    "schedule-due-sources": {
+        "task": "apps.sources.tasks.schedule_due_sources",
+        "schedule": 900.0,
+    },
+    "expire-old-jobs": {
+        "task": "apps.jobs.tasks.update_job_lifecycle",
+        "schedule": 3600.0,
+    },
+    "build-daily-digests": {
+        "task": "apps.notifications.tasks.build_due_digests",
+        "schedule": 900.0,
+    },
+    "build-due-reminders": {
+        "task": "apps.notifications.tasks.build_due_reminders",
+        "schedule": 900.0,
+    },
+    "enforce-source-health": {
+        "task": "apps.sources.tasks.enforce_source_health",
+        "schedule": 900.0,
+    },
+    "recover-source-health": {
+        "task": "apps.sources.tasks.recover_source_health",
+        "schedule": 3600.0,
+    },
+    "cleanup-raw-documents": {
+        "task": "apps.sources.tasks.cleanup_raw_documents",
+        "schedule": 3600.0,
+    },
+}
+
+FETCHER_USER_AGENT = os.getenv("FETCHER_USER_AGENT", "VacatureRadar/0.1 (+local-personal-use)")
+FETCHER_TIMEOUT_SECONDS = float(os.getenv("FETCHER_TIMEOUT_SECONDS", "20"))
+FETCHER_MAX_BYTES = int(os.getenv("FETCHER_MAX_BYTES", "3000000"))
+FETCHER_MAX_REDIRECTS = int(os.getenv("FETCHER_MAX_REDIRECTS", "5"))
+FETCHER_ALLOW_NONSTANDARD_PORTS = env_bool("FETCHER_ALLOW_NONSTANDARD_PORTS", False)
+SOURCE_POLICY_MODE = os.getenv("SOURCE_POLICY_MODE", "strict")
+SOURCE_LEASE_TTL_SECONDS = float(os.getenv("SOURCE_LEASE_TTL_SECONDS", "180"))
+SOURCE_ORIGIN_MIN_INTERVAL_SECONDS = int(os.getenv("SOURCE_ORIGIN_MIN_INTERVAL_SECONDS", "0"))
+SOURCE_SUCCESS_JITTER_SECONDS = int(os.getenv("SOURCE_SUCCESS_JITTER_SECONDS", "120"))
+SOURCE_FAILURE_BACKOFF_BASE_SECONDS = int(os.getenv("SOURCE_FAILURE_BACKOFF_BASE_SECONDS", "60"))
+SOURCE_FAILURE_BACKOFF_MAX_SECONDS = int(os.getenv("SOURCE_FAILURE_BACKOFF_MAX_SECONDS", "3600"))
+SOURCE_FAILURE_TIMEOUT_BASE_SECONDS = int(os.getenv("SOURCE_FAILURE_TIMEOUT_BASE_SECONDS", "120"))
+SOURCE_FAILURE_JITTER_SECONDS = int(os.getenv("SOURCE_FAILURE_JITTER_SECONDS", "45"))
+SOURCE_REVIEW_TTL_DAYS = int(os.getenv("SOURCE_REVIEW_TTL_DAYS", "90"))
+ROBOTS_CACHE_TTL_SECONDS = float(os.getenv("ROBOTS_CACHE_TTL_SECONDS", "3600"))
+ROBOTS_FETCH_TIMEOUT_SECONDS = float(os.getenv("ROBOTS_FETCH_TIMEOUT_SECONDS", "5"))
+ROBOTS_MAX_BYTES = int(os.getenv("ROBOTS_MAX_BYTES", "131072"))
+RAW_DOCUMENT_RETENTION_DAYS = int(os.getenv("RAW_DOCUMENT_RETENTION_DAYS", "7"))
+MANUAL_IMPORT_PASTE_MAX_BYTES = int(os.getenv("MANUAL_IMPORT_PASTE_MAX_BYTES", "12000"))
+DATA_UPLOAD_MAX_MEMORY_SIZE = int(os.getenv("DATA_UPLOAD_MAX_MEMORY_SIZE", str(8 * 1024 * 1024)))
+FILE_UPLOAD_MAX_MEMORY_SIZE = 0
+DATA_UPLOAD_MAX_NUMBER_FILES = 0
+
+IMAP_ENABLED = env_bool("IMAP_ENABLED", False)
+IMAP_HOST = os.getenv("IMAP_HOST", "")
+IMAP_PORT = int(os.getenv("IMAP_PORT", "993"))
+IMAP_USER = os.getenv("IMAP_USER", "")
+IMAP_PASSWORD = os.getenv("IMAP_PASSWORD", "")
+IMAP_MAILBOX = os.getenv("IMAP_MAILBOX", "INBOX")
+IMAP_USE_SSL = env_bool("IMAP_USE_SSL", True)
+IMAP_MARK_SEEN = env_bool("IMAP_MARK_SEEN", False)
+
+OLLAMA_ENABLED = env_bool("OLLAMA_ENABLED", False)
+OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
+OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "")
+OLLAMA_TIMEOUT_SECONDS = float(os.getenv("OLLAMA_TIMEOUT_SECONDS", "60"))
+
+DIGEST_RECIPIENT = os.getenv("DIGEST_RECIPIENT", "")
+
+AUTH_LOGIN_RATE_LIMIT_MAX_ATTEMPTS = int(
+    os.getenv("AUTH_LOGIN_RATE_LIMIT_MAX_ATTEMPTS", "8")
+)
+AUTH_LOGIN_RATE_LIMIT_WINDOW_SECONDS = int(
+    os.getenv("AUTH_LOGIN_RATE_LIMIT_WINDOW_SECONDS", "300")
+)
+AUTH_LOGIN_RATE_LIMIT_BLOCK_SECONDS = int(
+    os.getenv("AUTH_LOGIN_RATE_LIMIT_BLOCK_SECONDS", "300")
+)
+MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS = int(
+    os.getenv("MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS", "12")
+)
+MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS = int(
+    os.getenv("MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS", "120")
+)
+MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS = int(
+    os.getenv("MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS", "300")
+)
+
+_validate_production_security(
+    debug=DEBUG,
+    secret_key=SECRET_KEY,
+    allowed_hosts=ALLOWED_HOSTS,
+    csrf_trusted_origins=CSRF_TRUSTED_ORIGINS,
+    session_cookie_secure=SESSION_COOKIE_SECURE,
+    csrf_cookie_secure=CSRF_COOKIE_SECURE,
+    secure_ssl_redirect=SECURE_SSL_REDIRECT,
+)
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "jsonish": {
+            "format": "%(asctime)s %(levelname)s %(name)s %(message)s",
+        }
+    },
+    "handlers": {
+        "console": {"class": "logging.StreamHandler", "formatter": "jsonish"},
+    },
+    "root": {"handlers": ["console"], "level": os.getenv("LOG_LEVEL", "INFO")},
+}
+
+# Testomgeving hoeft geen Redis te hebben.
+if os.getenv("PYTEST_CURRENT_TEST") or "pytest" in os.sys.modules:
+    CELERY_TASK_ALWAYS_EAGER = True
+    EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"