Initial deploy setup

2026-07-21 14:00:00 +02:00
commit b8091e59bd
285 changed files with 27854 additions and 0 deletions
@@ -0,0 +1,136 @@
+# Generated by Django 5.2.16 on 2026-07-20 23:57
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RawDocument',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('url', models.URLField(blank=True, max_length=2000)),
+                ('final_url', models.URLField(blank=True, max_length=2000)),
+                ('kind', models.CharField(choices=[('html', 'HTML'), ('xml', 'XML'), ('email', 'E-mail'), ('json', 'JSON'), ('text', 'Tekst')], max_length=16)),
+                ('content_type', models.CharField(blank=True, max_length=200)),
+                ('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('response_headers', models.JSONField(blank=True, default=dict)),
+                ('content_hash', models.CharField(db_index=True, max_length=64)),
+                ('body_text', models.TextField(blank=True)),
+                ('byte_length', models.PositiveIntegerField(default=0)),
+                ('parser_key', models.CharField(blank=True, max_length=120)),
+                ('parser_version', models.CharField(blank=True, max_length=80)),
+                ('extraction_confidence', models.DecimalField(decimal_places=3, default=0, max_digits=4)),
+                ('retain_until', models.DateTimeField(blank=True, null=True)),
+                ('quarantined', models.BooleanField(default=False)),
+                ('quarantine_reason', models.CharField(blank=True, max_length=500)),
+                ('metadata', models.JSONField(blank=True, default=dict)),
+            ],
+            options={
+                'ordering': ['-created_at'],
+            },
+        ),
+        migrations.CreateModel(
+            name='EmailMessageRecord',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('message_id', models.CharField(max_length=998, unique=True)),
+                ('mailbox', models.CharField(default='INBOX', max_length=255)),
+                ('sender', models.CharField(blank=True, max_length=500)),
+                ('subject', models.CharField(blank=True, max_length=998)),
+                ('received_at', models.DateTimeField(blank=True, null=True)),
+                ('links', models.JSONField(blank=True, default=list)),
+                ('processed', models.BooleanField(default=False)),
+                ('error_message', models.CharField(blank=True, max_length=1000)),
+                ('raw_document', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='email_record', to='sources.rawdocument')),
+            ],
+            options={
+                'ordering': ['-received_at', '-created_at'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Source',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=200)),
+                ('source_type', models.CharField(choices=[('employer', 'Werkgeverspagina'), ('ats', 'Publieke ATS-pagina'), ('rss', 'RSS/Atom'), ('sitemap', 'Sitemap'), ('email', 'Vacaturemail'), ('manual', 'Handmatige import')], max_length=24)),
+                ('base_url', models.URLField(blank=True, max_length=1000)),
+                ('domain', models.CharField(db_index=True, max_length=255)),
+                ('status', models.CharField(choices=[('candidate', 'Kandidaat'), ('trial', 'Proefrun'), ('active', 'Actief'), ('quarantined', 'Quarantaine'), ('paused', 'Gepauzeerd'), ('disabled', 'Uitgeschakeld')], default='candidate', max_length=24)),
+                ('policy', models.CharField(choices=[('allow', 'Toestaan'), ('review', 'Te beoordelen'), ('deny', 'Blokkeren')], default='review', max_length=16)),
+                ('policy_reason', models.CharField(blank=True, max_length=500)),
+                ('parser_key', models.CharField(default='auto', max_length=120)),
+                ('strict_mode', models.BooleanField(default=True)),
+                ('allow_public_endpoint', models.BooleanField(default=False)),
+                ('honor_robots', models.BooleanField(default=True)),
+                ('crawl_interval_minutes', models.PositiveIntegerField(default=720)),
+                ('minimum_interval_seconds', models.PositiveIntegerField(default=30)),
+                ('max_concurrency', models.PositiveSmallIntegerField(default=1)),
+                ('next_run_at', models.DateTimeField(blank=True, db_index=True, null=True)),
+                ('last_success_at', models.DateTimeField(blank=True, null=True)),
+                ('last_failure_at', models.DateTimeField(blank=True, null=True)),
+                ('failure_count', models.PositiveIntegerField(default=0)),
+                ('etag', models.CharField(blank=True, max_length=500)),
+                ('last_modified', models.CharField(blank=True, max_length=500)),
+                ('robots_checked_at', models.DateTimeField(blank=True, null=True)),
+                ('terms_checked_at', models.DateTimeField(blank=True, null=True)),
+                ('metadata', models.JSONField(blank=True, default=dict)),
+            ],
+            options={
+                'ordering': ['name'],
+                'constraints': [models.UniqueConstraint(fields=('domain', 'source_type'), name='unique_domain_source_type')],
+            },
+        ),
+        migrations.AddField(
+            model_name='rawdocument',
+            name='source',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.source'),
+        ),
+        migrations.CreateModel(
+            name='SourceRun',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('status', models.CharField(choices=[('running', 'Bezig'), ('success', 'Geslaagd'), ('partial', 'Gedeeltelijk'), ('failed', 'Mislukt'), ('skipped', 'Overgeslagen')], default='running', max_length=16)),
+                ('started_at', models.DateTimeField(default=django.utils.timezone.now)),
+                ('finished_at', models.DateTimeField(blank=True, null=True)),
+                ('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('discovered_count', models.PositiveIntegerField(default=0)),
+                ('extracted_count', models.PositiveIntegerField(default=0)),
+                ('created_count', models.PositiveIntegerField(default=0)),
+                ('updated_count', models.PositiveIntegerField(default=0)),
+                ('duplicate_count', models.PositiveIntegerField(default=0)),
+                ('error_category', models.CharField(blank=True, max_length=80)),
+                ('error_message', models.CharField(blank=True, max_length=1000)),
+                ('metrics', models.JSONField(blank=True, default=dict)),
+                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='sources.source')),
+            ],
+            options={
+                'ordering': ['-started_at'],
+            },
+        ),
+        migrations.AddField(
+            model_name='rawdocument',
+            name='source_run',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.sourcerun'),
+        ),
+        migrations.AddIndex(
+            model_name='rawdocument',
+            index=models.Index(fields=['source', 'content_hash'], name='sources_raw_source__4a7b2a_idx'),
+        ),
+    ]
@@ -0,0 +1,51 @@
+# Generated by Django 5.2.16 on 2026-07-21 for VR-102
+
+import django.conf
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sources", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SourcePolicyReview",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("scope", models.CharField(choices=[("source", "Bronspecifiek"), ("domain", "Domeinbrede controle")], default="source", max_length=20)),
+                ("decision", models.CharField(choices=[("allow", "Toestaan"), ("trial", "Proefrun"), ("pause", "Pauzeren"), ("deny", "Blokkeren"),], max_length=16)),
+                ("reason", models.CharField(max_length=500)),
+                ("evidence_link", models.URLField(blank=True, max_length=500)),
+                ("notes", models.TextField(blank=True)),
+                ("expires_at", models.DateTimeField(blank=True, null=True)),
+                ("metadata", models.JSONField(blank=True, default=dict)),
+                ("actor", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="policy_reviews", to=django.conf.settings.AUTH_USER_MODEL)),
+                ("source", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name="policy_reviews", to="sources.source")),
+            ],
+            options={"ordering": ["-created_at"]},
+        ),
+        migrations.CreateModel(
+            name="SourceRobotsCache",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("origin", models.CharField(max_length=255, unique=True)),
+                ("expires_at", models.DateTimeField()),
+                ("etag", models.CharField(blank=True, max_length=255)),
+                ("last_modified", models.CharField(blank=True, max_length=255)),
+                ("allow_rules", models.JSONField(blank=True, default=dict)),
+                ("disallow_rules", models.JSONField(blank=True, default=dict)),
+                ("error", models.CharField(blank=True, max_length=500)),
+                ("byte_length", models.PositiveIntegerField(default=0)),
+            ],
+            options={"ordering": ["origin"]},
+        ),
+    ]
@@ -0,0 +1,45 @@
+# Generated by Django 5.2.16 on 2026-07-21 for VR-103
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sources", "0002_policy_review_and_robots_cache"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SourceLease",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("token", models.CharField(default="", max_length=64)),
+                ("worker_id", models.CharField(blank=True, max_length=128)),
+                ("expires_at", models.DateTimeField(db_index=True)),
+                (
+                    "source",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="lease",
+                        to="sources.source",
+                    ),
+                ),
+            ],
+            options={"ordering": ["source"]},
+        ),
+        migrations.CreateModel(
+            name="SourceOriginState",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("domain", models.CharField(max_length=255, unique=True)),
+                ("next_allowed_at", models.DateTimeField()),
+            ],
+            options={"ordering": ["domain"]},
+        ),
+    ]