357 lines
13 KiB
Python
357 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
from django.core.exceptions import ImproperlyConfigured
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def env_bool(name: str, default: bool = False) -> bool:
|
|
return os.getenv(name, "1" if default else "0").lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def env_list(name: str, default: str = "") -> list[str]:
|
|
return [item.strip() for item in os.getenv(name, default).split(",") if item.strip()]
|
|
|
|
|
|
def env_int_range(name: str, default: int, *, minimum: int, maximum: int) -> int:
|
|
try:
|
|
value = int(os.getenv(name, str(default)))
|
|
except ValueError as exc:
|
|
raise ImproperlyConfigured(f"{name} moet een geheel getal zijn.") from exc
|
|
if not minimum <= value <= maximum:
|
|
raise ImproperlyConfigured(f"{name} moet tussen {minimum} en {maximum} liggen.")
|
|
return value
|
|
|
|
|
|
def _validate_production_security(
|
|
*,
|
|
debug: bool,
|
|
secret_key: str,
|
|
allowed_hosts: list[str],
|
|
csrf_trusted_origins: list[str],
|
|
session_cookie_secure: bool,
|
|
csrf_cookie_secure: bool,
|
|
secure_ssl_redirect: bool,
|
|
) -> None:
|
|
if debug:
|
|
return
|
|
if secret_key.strip().lower() in {"", "change-me", "dev-only-change-me", "secret", "insecure"}:
|
|
raise ImproperlyConfigured(
|
|
"Onveilige DJANGO_SECRET_KEY in productie. Stel een sterke waarde in via environment."
|
|
)
|
|
if len(secret_key.strip()) < 50:
|
|
raise ImproperlyConfigured("DJANGO_SECRET_KEY is te kort voor een productieomgeving.")
|
|
if not allowed_hosts:
|
|
raise ImproperlyConfigured("ALLOWED_HOSTS mag in productie niet leeg zijn.")
|
|
if not csrf_trusted_origins:
|
|
raise ImproperlyConfigured("CSRF_TRUSTED_ORIGINS is verplicht bij DEBUG=False.")
|
|
if not any(origin.lower().startswith("https://") for origin in csrf_trusted_origins):
|
|
raise ImproperlyConfigured("CSRF_TRUSTED_ORIGINS moet HTTPS-origins bevatten in productie.")
|
|
if not session_cookie_secure or not csrf_cookie_secure:
|
|
raise ImproperlyConfigured("Session- en CSRF-cookies moeten Secure=True zijn in productie.")
|
|
if not secure_ssl_redirect:
|
|
raise ImproperlyConfigured("SECURE_SSL_REDIRECT moet True zijn in productie.")
|
|
|
|
|
|
SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "dev-only-change-me")
|
|
DEBUG = env_bool("DJANGO_DEBUG", True)
|
|
ALLOWED_HOSTS = env_list("DJANGO_ALLOWED_HOSTS", "localhost,127.0.0.1,testserver")
|
|
CSRF_TRUSTED_ORIGINS = env_list("DJANGO_CSRF_TRUSTED_ORIGINS")
|
|
TIME_ZONE = os.getenv("DJANGO_TIME_ZONE", "Europe/Brussels")
|
|
LANGUAGE_CODE = "nl-be"
|
|
USE_I18N = True
|
|
USE_TZ = True
|
|
|
|
INSTALLED_APPS = [
|
|
"django.contrib.admin",
|
|
"django.contrib.auth",
|
|
"django.contrib.contenttypes",
|
|
"django.contrib.sessions",
|
|
"django.contrib.messages",
|
|
"django.contrib.staticfiles",
|
|
"apps.core",
|
|
"apps.profiles",
|
|
"apps.sources",
|
|
"apps.jobs",
|
|
"apps.notifications",
|
|
]
|
|
|
|
MIDDLEWARE = [
|
|
"django.middleware.security.SecurityMiddleware",
|
|
"whitenoise.middleware.WhiteNoiseMiddleware",
|
|
"django.contrib.sessions.middleware.SessionMiddleware",
|
|
"django.middleware.locale.LocaleMiddleware",
|
|
"django.middleware.common.CommonMiddleware",
|
|
"django.middleware.csrf.CsrfViewMiddleware",
|
|
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
|
"django.contrib.messages.middleware.MessageMiddleware",
|
|
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
|
"apps.core.middleware.SecurityHeadersMiddleware",
|
|
]
|
|
|
|
ROOT_URLCONF = "config.urls"
|
|
TEMPLATES = [
|
|
{
|
|
"BACKEND": "django.template.backends.django.DjangoTemplates",
|
|
"DIRS": [BASE_DIR / "templates"],
|
|
"APP_DIRS": True,
|
|
"OPTIONS": {
|
|
"context_processors": [
|
|
"django.template.context_processors.request",
|
|
"django.contrib.auth.context_processors.auth",
|
|
"django.contrib.messages.context_processors.messages",
|
|
"apps.core.context_processors.navigation_context",
|
|
],
|
|
},
|
|
}
|
|
]
|
|
WSGI_APPLICATION = "config.wsgi.application"
|
|
ASGI_APPLICATION = "config.asgi.application"
|
|
|
|
DATABASE_URL = os.getenv("DATABASE_URL", "").strip()
|
|
POSTGRES_DB = os.getenv("POSTGRES_DB", "vacatureradar")
|
|
POSTGRES_USER = os.getenv("POSTGRES_USER", "vacatureradar")
|
|
POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", "")
|
|
POSTGRES_HOST = os.getenv("POSTGRES_HOST", "postgres")
|
|
POSTGRES_PORT = os.getenv("POSTGRES_PORT", "5432")
|
|
|
|
|
|
def _postgres_database_from_env() -> dict[str, str | int]:
|
|
if POSTGRES_PASSWORD.lower().startswith("change_me") or POSTGRES_PASSWORD == "CHANGE_ME":
|
|
return {}
|
|
if not (POSTGRES_PASSWORD and POSTGRES_HOST and POSTGRES_DB and POSTGRES_USER):
|
|
return {}
|
|
return {
|
|
"ENGINE": "django.db.backends.postgresql",
|
|
"NAME": POSTGRES_DB,
|
|
"USER": POSTGRES_USER,
|
|
"PASSWORD": POSTGRES_PASSWORD,
|
|
"HOST": POSTGRES_HOST,
|
|
"PORT": POSTGRES_PORT,
|
|
"CONN_MAX_AGE": 60,
|
|
"OPTIONS": {"connect_timeout": 10},
|
|
}
|
|
|
|
|
|
if DATABASE_URL:
|
|
parsed = urlparse(DATABASE_URL)
|
|
DATABASES = {
|
|
"default": {
|
|
"ENGINE": "django.db.backends.postgresql",
|
|
"NAME": parsed.path.lstrip("/"),
|
|
"USER": parsed.username,
|
|
"PASSWORD": parsed.password,
|
|
"HOST": parsed.hostname,
|
|
"PORT": parsed.port or 5432,
|
|
"CONN_MAX_AGE": 60,
|
|
"OPTIONS": {"connect_timeout": 10},
|
|
}
|
|
}
|
|
elif postgres_cfg := _postgres_database_from_env():
|
|
DATABASES = {"default": postgres_cfg}
|
|
else:
|
|
DATABASES = {
|
|
"default": {
|
|
"ENGINE": "django.db.backends.sqlite3",
|
|
"NAME": BASE_DIR / "local" / "db.sqlite3",
|
|
}
|
|
}
|
|
|
|
AUTH_PASSWORD_VALIDATORS = [
|
|
{"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
|
|
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
|
|
{"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
|
|
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
|
|
]
|
|
|
|
STATIC_URL = "/static/"
|
|
STATIC_ROOT = BASE_DIR / "staticfiles"
|
|
STATICFILES_DIRS = [BASE_DIR / "static"]
|
|
STORAGES = {
|
|
"default": {"BACKEND": "django.core.files.storage.FileSystemStorage"},
|
|
"staticfiles": {
|
|
"BACKEND": (
|
|
"whitenoise.storage.CompressedStaticFilesStorage"
|
|
if DEBUG
|
|
else "whitenoise.storage.CompressedManifestStaticFilesStorage"
|
|
)
|
|
},
|
|
}
|
|
MEDIA_URL = "/media/"
|
|
MEDIA_ROOT = BASE_DIR / "media"
|
|
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
|
LOGIN_URL = "login"
|
|
LOGIN_REDIRECT_URL = "dashboard:today"
|
|
LOGOUT_REDIRECT_URL = "login"
|
|
|
|
EMAIL_BACKEND = os.getenv("EMAIL_BACKEND", "django.core.mail.backends.console.EmailBackend")
|
|
DEFAULT_FROM_EMAIL = os.getenv("DEFAULT_FROM_EMAIL", "VacatureRadar <vacatureradar@localhost>")
|
|
EMAIL_HOST = os.getenv("SMTP_HOST", "")
|
|
EMAIL_PORT = int(os.getenv("SMTP_PORT", "587"))
|
|
EMAIL_HOST_USER = os.getenv("SMTP_USER", "")
|
|
EMAIL_HOST_PASSWORD = os.getenv("SMTP_PASSWORD", "")
|
|
EMAIL_USE_TLS = env_bool("SMTP_USE_TLS", True)
|
|
|
|
SESSION_COOKIE_HTTPONLY = True
|
|
SESSION_COOKIE_SAMESITE = "Lax"
|
|
SESSION_COOKIE_SECURE = env_bool("SESSION_COOKIE_SECURE", False)
|
|
CSRF_COOKIE_SECURE = env_bool("CSRF_COOKIE_SECURE", False)
|
|
CSRF_COOKIE_SAMESITE = "Lax"
|
|
SECURE_SSL_REDIRECT = env_bool("SECURE_SSL_REDIRECT", False)
|
|
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
|
|
SECURE_HSTS_SECONDS = int(os.getenv("SECURE_HSTS_SECONDS", "0"))
|
|
SECURE_HSTS_INCLUDE_SUBDOMAINS = env_bool("SECURE_HSTS_INCLUDE_SUBDOMAINS", True)
|
|
SECURE_HSTS_PRELOAD = env_bool("SECURE_HSTS_PRELOAD", False)
|
|
USE_X_FORWARDED_HOST = True
|
|
SECURE_CONTENT_TYPE_NOSNIFF = True
|
|
SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
|
|
X_FRAME_OPTIONS = "DENY"
|
|
|
|
CACHES = {
|
|
"default": {
|
|
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
|
|
"LOCATION": "vacatureradar-rate-limit",
|
|
}
|
|
}
|
|
|
|
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
|
|
CELERY_BROKER_URL = REDIS_URL
|
|
CELERY_RESULT_BACKEND = REDIS_URL
|
|
CELERY_TASK_ALWAYS_EAGER = env_bool("CELERY_TASK_ALWAYS_EAGER", False)
|
|
CELERY_TASK_EAGER_PROPAGATES = True
|
|
CELERY_TASK_ACKS_LATE = True
|
|
CELERY_WORKER_PREFETCH_MULTIPLIER = 1
|
|
CELERY_TASK_TIME_LIMIT = 300
|
|
CELERY_TASK_SOFT_TIME_LIMIT = 270
|
|
CELERY_TIMEZONE = TIME_ZONE
|
|
|
|
IMAP_MAX_MESSAGES_PER_POLL = env_int_range(
|
|
"IMAP_MAX_MESSAGES_PER_POLL", 200, minimum=1, maximum=1000
|
|
)
|
|
IMAP_MAX_MESSAGE_BYTES = env_int_range(
|
|
"IMAP_MAX_MESSAGE_BYTES", 1000000, minimum=1024, maximum=10000000
|
|
)
|
|
IMAP_CONNECT_TIMEOUT_SECONDS = env_int_range(
|
|
"IMAP_CONNECT_TIMEOUT_SECONDS", 15, minimum=3, maximum=60
|
|
)
|
|
MAILBOX_CREDENTIAL_KEYS = env_list("MAILBOX_CREDENTIAL_KEYS")
|
|
if DEBUG and not MAILBOX_CREDENTIAL_KEYS:
|
|
MAILBOX_CREDENTIAL_KEYS = [
|
|
base64.urlsafe_b64encode(
|
|
hashlib.sha256(f"vacatureradar-mailbox:{SECRET_KEY}".encode()).digest()
|
|
).decode("ascii")
|
|
]
|
|
|
|
CELERY_BEAT_SCHEDULE = {
|
|
"schedule-mailbox-polls": {
|
|
"task": "apps.sources.tasks.schedule_mailbox_polls",
|
|
"schedule": 300.0,
|
|
},
|
|
"schedule-due-sources": {
|
|
"task": "apps.sources.tasks.schedule_due_sources",
|
|
"schedule": 900.0,
|
|
},
|
|
"expire-old-jobs": {
|
|
"task": "apps.jobs.tasks.update_job_lifecycle",
|
|
"schedule": 3600.0,
|
|
},
|
|
"build-daily-digests": {
|
|
"task": "apps.notifications.tasks.build_due_digests",
|
|
"schedule": 900.0,
|
|
},
|
|
"build-due-reminders": {
|
|
"task": "apps.notifications.tasks.build_due_reminders",
|
|
"schedule": 900.0,
|
|
},
|
|
"enforce-source-health": {
|
|
"task": "apps.sources.tasks.enforce_source_health",
|
|
"schedule": 900.0,
|
|
},
|
|
"recover-source-health": {
|
|
"task": "apps.sources.tasks.recover_source_health",
|
|
"schedule": 3600.0,
|
|
},
|
|
"cleanup-raw-documents": {
|
|
"task": "apps.sources.tasks.cleanup_raw_documents",
|
|
"schedule": 3600.0,
|
|
},
|
|
}
|
|
|
|
FETCHER_USER_AGENT = os.getenv("FETCHER_USER_AGENT", "VacatureRadar/0.1 (+local-personal-use)")
|
|
FETCHER_TIMEOUT_SECONDS = float(os.getenv("FETCHER_TIMEOUT_SECONDS", "20"))
|
|
FETCHER_MAX_BYTES = int(os.getenv("FETCHER_MAX_BYTES", "3000000"))
|
|
FETCHER_MAX_REDIRECTS = int(os.getenv("FETCHER_MAX_REDIRECTS", "5"))
|
|
FETCHER_ALLOW_NONSTANDARD_PORTS = env_bool("FETCHER_ALLOW_NONSTANDARD_PORTS", False)
|
|
SOURCE_POLICY_MODE = os.getenv("SOURCE_POLICY_MODE", "strict")
|
|
SOURCE_LEASE_TTL_SECONDS = float(os.getenv("SOURCE_LEASE_TTL_SECONDS", "180"))
|
|
SOURCE_ORIGIN_MIN_INTERVAL_SECONDS = int(os.getenv("SOURCE_ORIGIN_MIN_INTERVAL_SECONDS", "0"))
|
|
SOURCE_SUCCESS_JITTER_SECONDS = int(os.getenv("SOURCE_SUCCESS_JITTER_SECONDS", "120"))
|
|
SOURCE_FAILURE_BACKOFF_BASE_SECONDS = int(os.getenv("SOURCE_FAILURE_BACKOFF_BASE_SECONDS", "60"))
|
|
SOURCE_FAILURE_BACKOFF_MAX_SECONDS = int(os.getenv("SOURCE_FAILURE_BACKOFF_MAX_SECONDS", "3600"))
|
|
SOURCE_FAILURE_TIMEOUT_BASE_SECONDS = int(os.getenv("SOURCE_FAILURE_TIMEOUT_BASE_SECONDS", "120"))
|
|
SOURCE_FAILURE_JITTER_SECONDS = int(os.getenv("SOURCE_FAILURE_JITTER_SECONDS", "45"))
|
|
SOURCE_REVIEW_TTL_DAYS = int(os.getenv("SOURCE_REVIEW_TTL_DAYS", "90"))
|
|
ROBOTS_CACHE_TTL_SECONDS = float(os.getenv("ROBOTS_CACHE_TTL_SECONDS", "3600"))
|
|
ROBOTS_FETCH_TIMEOUT_SECONDS = float(os.getenv("ROBOTS_FETCH_TIMEOUT_SECONDS", "5"))
|
|
ROBOTS_MAX_BYTES = int(os.getenv("ROBOTS_MAX_BYTES", "131072"))
|
|
RAW_DOCUMENT_RETENTION_DAYS = int(os.getenv("RAW_DOCUMENT_RETENTION_DAYS", "7"))
|
|
MANUAL_IMPORT_PASTE_MAX_BYTES = int(os.getenv("MANUAL_IMPORT_PASTE_MAX_BYTES", "12000"))
|
|
DATA_UPLOAD_MAX_MEMORY_SIZE = int(os.getenv("DATA_UPLOAD_MAX_MEMORY_SIZE", str(8 * 1024 * 1024)))
|
|
FILE_UPLOAD_MAX_MEMORY_SIZE = 0
|
|
DATA_UPLOAD_MAX_NUMBER_FILES = 0
|
|
|
|
OLLAMA_ENABLED = env_bool("OLLAMA_ENABLED", False)
|
|
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://localhost:11434")
|
|
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "")
|
|
OLLAMA_TIMEOUT_SECONDS = float(os.getenv("OLLAMA_TIMEOUT_SECONDS", "60"))
|
|
|
|
DIGEST_RECIPIENT = os.getenv("DIGEST_RECIPIENT", "")
|
|
|
|
AUTH_LOGIN_RATE_LIMIT_MAX_ATTEMPTS = int(os.getenv("AUTH_LOGIN_RATE_LIMIT_MAX_ATTEMPTS", "8"))
|
|
AUTH_LOGIN_RATE_LIMIT_WINDOW_SECONDS = int(os.getenv("AUTH_LOGIN_RATE_LIMIT_WINDOW_SECONDS", "300"))
|
|
AUTH_LOGIN_RATE_LIMIT_BLOCK_SECONDS = int(os.getenv("AUTH_LOGIN_RATE_LIMIT_BLOCK_SECONDS", "300"))
|
|
MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS = int(
|
|
os.getenv("MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS", "12")
|
|
)
|
|
MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS = int(
|
|
os.getenv("MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS", "120")
|
|
)
|
|
MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS = int(
|
|
os.getenv("MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS", "300")
|
|
)
|
|
|
|
_validate_production_security(
|
|
debug=DEBUG,
|
|
secret_key=SECRET_KEY,
|
|
allowed_hosts=ALLOWED_HOSTS,
|
|
csrf_trusted_origins=CSRF_TRUSTED_ORIGINS,
|
|
session_cookie_secure=SESSION_COOKIE_SECURE,
|
|
csrf_cookie_secure=CSRF_COOKIE_SECURE,
|
|
secure_ssl_redirect=SECURE_SSL_REDIRECT,
|
|
)
|
|
|
|
LOGGING = {
|
|
"version": 1,
|
|
"disable_existing_loggers": False,
|
|
"formatters": {
|
|
"jsonish": {
|
|
"format": "%(asctime)s %(levelname)s %(name)s %(message)s",
|
|
}
|
|
},
|
|
"handlers": {
|
|
"console": {"class": "logging.StreamHandler", "formatter": "jsonish"},
|
|
},
|
|
"root": {"handlers": ["console"], "level": os.getenv("LOG_LEVEL", "INFO")},
|
|
}
|
|
|
|
# Testomgeving hoeft geen Redis te hebben.
|
|
if os.getenv("PYTEST_CURRENT_TEST") or "pytest" in os.sys.modules:
|
|
CELERY_TASK_ALWAYS_EAGER = True
|
|
EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
|