137 lines
8.3 KiB
Python
137 lines
8.3 KiB
Python
# Generated by Django 5.2.16 on 2026-07-20 23:57
|
|
|
|
import django.db.models.deletion
|
|
import django.utils.timezone
|
|
from django.db import migrations, models
|
|
|
|
|
|
class Migration(migrations.Migration):
|
|
|
|
initial = True
|
|
|
|
dependencies = [
|
|
]
|
|
|
|
operations = [
|
|
migrations.CreateModel(
|
|
name='RawDocument',
|
|
fields=[
|
|
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
('created_at', models.DateTimeField(auto_now_add=True)),
|
|
('updated_at', models.DateTimeField(auto_now=True)),
|
|
('url', models.URLField(blank=True, max_length=2000)),
|
|
('final_url', models.URLField(blank=True, max_length=2000)),
|
|
('kind', models.CharField(choices=[('html', 'HTML'), ('xml', 'XML'), ('email', 'E-mail'), ('json', 'JSON'), ('text', 'Tekst')], max_length=16)),
|
|
('content_type', models.CharField(blank=True, max_length=200)),
|
|
('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
|
|
('response_headers', models.JSONField(blank=True, default=dict)),
|
|
('content_hash', models.CharField(db_index=True, max_length=64)),
|
|
('body_text', models.TextField(blank=True)),
|
|
('byte_length', models.PositiveIntegerField(default=0)),
|
|
('parser_key', models.CharField(blank=True, max_length=120)),
|
|
('parser_version', models.CharField(blank=True, max_length=80)),
|
|
('extraction_confidence', models.DecimalField(decimal_places=3, default=0, max_digits=4)),
|
|
('retain_until', models.DateTimeField(blank=True, null=True)),
|
|
('quarantined', models.BooleanField(default=False)),
|
|
('quarantine_reason', models.CharField(blank=True, max_length=500)),
|
|
('metadata', models.JSONField(blank=True, default=dict)),
|
|
],
|
|
options={
|
|
'ordering': ['-created_at'],
|
|
},
|
|
),
|
|
migrations.CreateModel(
|
|
name='EmailMessageRecord',
|
|
fields=[
|
|
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
('created_at', models.DateTimeField(auto_now_add=True)),
|
|
('updated_at', models.DateTimeField(auto_now=True)),
|
|
('message_id', models.CharField(max_length=998, unique=True)),
|
|
('mailbox', models.CharField(default='INBOX', max_length=255)),
|
|
('sender', models.CharField(blank=True, max_length=500)),
|
|
('subject', models.CharField(blank=True, max_length=998)),
|
|
('received_at', models.DateTimeField(blank=True, null=True)),
|
|
('links', models.JSONField(blank=True, default=list)),
|
|
('processed', models.BooleanField(default=False)),
|
|
('error_message', models.CharField(blank=True, max_length=1000)),
|
|
('raw_document', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='email_record', to='sources.rawdocument')),
|
|
],
|
|
options={
|
|
'ordering': ['-received_at', '-created_at'],
|
|
},
|
|
),
|
|
migrations.CreateModel(
|
|
name='Source',
|
|
fields=[
|
|
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
('created_at', models.DateTimeField(auto_now_add=True)),
|
|
('updated_at', models.DateTimeField(auto_now=True)),
|
|
('name', models.CharField(max_length=200)),
|
|
('source_type', models.CharField(choices=[('employer', 'Werkgeverspagina'), ('ats', 'Publieke ATS-pagina'), ('rss', 'RSS/Atom'), ('sitemap', 'Sitemap'), ('email', 'Vacaturemail'), ('manual', 'Handmatige import')], max_length=24)),
|
|
('base_url', models.URLField(blank=True, max_length=1000)),
|
|
('domain', models.CharField(db_index=True, max_length=255)),
|
|
('status', models.CharField(choices=[('candidate', 'Kandidaat'), ('trial', 'Proefrun'), ('active', 'Actief'), ('quarantined', 'Quarantaine'), ('paused', 'Gepauzeerd'), ('disabled', 'Uitgeschakeld')], default='candidate', max_length=24)),
|
|
('policy', models.CharField(choices=[('allow', 'Toestaan'), ('review', 'Te beoordelen'), ('deny', 'Blokkeren')], default='review', max_length=16)),
|
|
('policy_reason', models.CharField(blank=True, max_length=500)),
|
|
('parser_key', models.CharField(default='auto', max_length=120)),
|
|
('strict_mode', models.BooleanField(default=True)),
|
|
('allow_public_endpoint', models.BooleanField(default=False)),
|
|
('honor_robots', models.BooleanField(default=True)),
|
|
('crawl_interval_minutes', models.PositiveIntegerField(default=720)),
|
|
('minimum_interval_seconds', models.PositiveIntegerField(default=30)),
|
|
('max_concurrency', models.PositiveSmallIntegerField(default=1)),
|
|
('next_run_at', models.DateTimeField(blank=True, db_index=True, null=True)),
|
|
('last_success_at', models.DateTimeField(blank=True, null=True)),
|
|
('last_failure_at', models.DateTimeField(blank=True, null=True)),
|
|
('failure_count', models.PositiveIntegerField(default=0)),
|
|
('etag', models.CharField(blank=True, max_length=500)),
|
|
('last_modified', models.CharField(blank=True, max_length=500)),
|
|
('robots_checked_at', models.DateTimeField(blank=True, null=True)),
|
|
('terms_checked_at', models.DateTimeField(blank=True, null=True)),
|
|
('metadata', models.JSONField(blank=True, default=dict)),
|
|
],
|
|
options={
|
|
'ordering': ['name'],
|
|
'constraints': [models.UniqueConstraint(fields=('domain', 'source_type'), name='unique_domain_source_type')],
|
|
},
|
|
),
|
|
migrations.AddField(
|
|
model_name='rawdocument',
|
|
name='source',
|
|
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.source'),
|
|
),
|
|
migrations.CreateModel(
|
|
name='SourceRun',
|
|
fields=[
|
|
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
('created_at', models.DateTimeField(auto_now_add=True)),
|
|
('updated_at', models.DateTimeField(auto_now=True)),
|
|
('status', models.CharField(choices=[('running', 'Bezig'), ('success', 'Geslaagd'), ('partial', 'Gedeeltelijk'), ('failed', 'Mislukt'), ('skipped', 'Overgeslagen')], default='running', max_length=16)),
|
|
('started_at', models.DateTimeField(default=django.utils.timezone.now)),
|
|
('finished_at', models.DateTimeField(blank=True, null=True)),
|
|
('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
|
|
('discovered_count', models.PositiveIntegerField(default=0)),
|
|
('extracted_count', models.PositiveIntegerField(default=0)),
|
|
('created_count', models.PositiveIntegerField(default=0)),
|
|
('updated_count', models.PositiveIntegerField(default=0)),
|
|
('duplicate_count', models.PositiveIntegerField(default=0)),
|
|
('error_category', models.CharField(blank=True, max_length=80)),
|
|
('error_message', models.CharField(blank=True, max_length=1000)),
|
|
('metrics', models.JSONField(blank=True, default=dict)),
|
|
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='sources.source')),
|
|
],
|
|
options={
|
|
'ordering': ['-started_at'],
|
|
},
|
|
),
|
|
migrations.AddField(
|
|
model_name='rawdocument',
|
|
name='source_run',
|
|
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.sourcerun'),
|
|
),
|
|
migrations.AddIndex(
|
|
model_name='rawdocument',
|
|
index=models.Index(fields=['source', 'content_hash'], name='sources_raw_source__4a7b2a_idx'),
|
|
),
|
|
]
|