288 lines
10 KiB
Python
288 lines
10 KiB
Python
from datetime import timedelta
|
|
|
|
import pytest
|
|
from django.urls import reverse
|
|
from django.utils import timezone
|
|
|
|
from apps.jobs.models import Application, Feedback, ScoreRun
|
|
from apps.jobs.tasks import rescore_active_jobs, update_job_lifecycle
|
|
from apps.notifications.tasks import build_due_digests
|
|
from apps.sources.models import RawDocument, Source, SourceLease, SourcePolicyReview, SourceRun
|
|
from apps.sources.services.fetcher import FetchedDocument, RateLimitedError
|
|
from apps.sources.tasks import cleanup_raw_documents, fetch_source, schedule_due_sources
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_task_helpers(profile, job, source, monkeypatch):
|
|
result = rescore_active_jobs(profile.pk)
|
|
assert result["scores_created"] == 1
|
|
assert ScoreRun.objects.filter(job=job, profile=profile).exists()
|
|
assert update_job_lifecycle() == {"expired": 0, "uncertain": 0, "removed": 0}
|
|
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
"apps.sources.tasks.fetch_source.delay", lambda source_id: calls.append(source_id)
|
|
)
|
|
source.next_run_at = timezone.now() - timedelta(minutes=1)
|
|
source.save(update_fields=["next_run_at"])
|
|
assert schedule_due_sources()["scheduled"] == 1
|
|
assert calls == [source.pk]
|
|
|
|
old = RawDocument.objects.create(
|
|
source=source,
|
|
kind=RawDocument.Kind.TEXT,
|
|
content_hash="8" * 64,
|
|
retain_until=timezone.now() - timedelta(days=1),
|
|
)
|
|
assert cleanup_raw_documents()["deleted"] >= 1
|
|
assert not RawDocument.objects.filter(pk=old.pk).exists()
|
|
|
|
source.policy = Source.Policy.DENY
|
|
source.policy_reason = "test"
|
|
source.save(update_fields=["policy", "policy_reason"])
|
|
assert fetch_source(source.pk)["status"] == "skipped"
|
|
assert SourceRun.objects.filter(source=source, status=SourceRun.Status.SKIPPED).exists()
|
|
|
|
|
|
def test_fetch_source_skips_when_lease_blocked(db, source, monkeypatch):
|
|
SourceLease.objects.create(
|
|
source=source,
|
|
token="active-worker",
|
|
worker_id="active-worker",
|
|
expires_at=timezone.now() + timedelta(minutes=2),
|
|
)
|
|
calls = {"count": 0}
|
|
|
|
def fake_fetch_url(*args, **kwargs):
|
|
calls["count"] += 1
|
|
return FetchedDocument(
|
|
requested_url=source.base_url,
|
|
final_url=source.base_url,
|
|
status_code=200,
|
|
headers={},
|
|
content=b"Vacature",
|
|
)
|
|
|
|
monkeypatch.setattr("apps.sources.tasks.fetch_url", fake_fetch_url)
|
|
assert fetch_source(source.pk)["status"] == "skipped"
|
|
assert calls["count"] == 0
|
|
assert SourceRun.objects.filter(source=source, status=SourceRun.Status.SKIPPED).exists()
|
|
|
|
|
|
def test_fetch_source_handles_rate_limit_backoff(db, source, monkeypatch):
|
|
called = {"count": 0}
|
|
|
|
def fake_fetch_url(*args, **kwargs):
|
|
called["count"] += 1
|
|
raise RateLimitedError(30)
|
|
|
|
monkeypatch.setattr("apps.sources.tasks.fetch_url", fake_fetch_url)
|
|
result = fetch_source(source.pk)
|
|
assert result["status"] == "rate_limited"
|
|
assert called["count"] == 1
|
|
source.refresh_from_db()
|
|
assert source.next_run_at is not None
|
|
assert SourceRun.objects.filter(source=source, status=SourceRun.Status.FAILED).exists()
|
|
|
|
|
|
def test_fetch_source_resets_fail_state_on_304(db, source, monkeypatch):
|
|
source.failure_count = 4
|
|
source.save(update_fields=["failure_count"])
|
|
|
|
def fake_fetch_url(*args, **kwargs):
|
|
return FetchedDocument(
|
|
requested_url=source.base_url,
|
|
final_url=source.base_url,
|
|
status_code=304,
|
|
headers={},
|
|
content=b"",
|
|
)
|
|
|
|
monkeypatch.setattr("apps.sources.tasks.fetch_url", fake_fetch_url)
|
|
assert fetch_source(source.pk)["status"] == "not_modified"
|
|
source.refresh_from_db()
|
|
assert source.failure_count == 0
|
|
|
|
|
|
def test_fetch_source_is_blocked_by_conflicting_review(db):
|
|
source = Source.objects.create(
|
|
name="Blocked source",
|
|
source_type=Source.Type.EMPLOYER,
|
|
base_url="https://jobs.example.org/vacatures/",
|
|
domain="jobs.example.org",
|
|
status=Source.Status.ACTIVE,
|
|
policy=Source.Policy.ALLOW,
|
|
)
|
|
SourcePolicyReview.objects.create(
|
|
source=source,
|
|
decision=SourcePolicyReview.Decision.DENY,
|
|
reason="Review blokkeert deze bron",
|
|
)
|
|
result = fetch_source(source.pk)
|
|
assert result["status"] == "skipped"
|
|
assert SourceRun.objects.filter(source=source, status=SourceRun.Status.SKIPPED).exists()
|
|
|
|
|
|
def test_fetch_source_is_blocked_by_expired_review(db):
|
|
source = Source.objects.create(
|
|
name="Expired review source",
|
|
source_type=Source.Type.EMPLOYER,
|
|
base_url="https://jobs.example.org/vacatures/",
|
|
domain="jobs.example.org",
|
|
status=Source.Status.ACTIVE,
|
|
policy=Source.Policy.ALLOW,
|
|
)
|
|
SourcePolicyReview.objects.create(
|
|
source=source,
|
|
decision=SourcePolicyReview.Decision.ALLOW,
|
|
reason="Review verlopen",
|
|
expires_at=timezone.now() - timedelta(days=1),
|
|
)
|
|
result = fetch_source(source.pk)
|
|
assert result["status"] == "skipped"
|
|
assert SourceRun.objects.filter(source=source, status=SourceRun.Status.SKIPPED).exists()
|
|
|
|
# Outside the digest window this is a safe no-op.
|
|
assert build_due_digests()["sent"] == 0
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.django_db
|
|
def test_job_profile_source_and_application_views(client, user, profile, job, source):
|
|
client.force_login(user)
|
|
assert client.get(reverse("jobs:list"), {"q": "Infrastructure"}).status_code == 200
|
|
assert client.get(reverse("jobs:detail", kwargs={"pk": job.pk})).status_code == 200
|
|
response = client.post(
|
|
reverse("jobs:feedback", kwargs={"pk": job.pk}),
|
|
{"action": Feedback.Action.INTERESTING},
|
|
)
|
|
assert response.status_code == 302
|
|
assert Feedback.objects.filter(user=user, job=job).exists()
|
|
|
|
unsafe_redirect = client.post(
|
|
reverse("jobs:feedback", kwargs={"pk": job.pk}),
|
|
{
|
|
"action": Feedback.Action.INTERESTING,
|
|
"next": "https://malicious.example/steal-session",
|
|
},
|
|
)
|
|
assert unsafe_redirect.url == reverse("jobs:detail", kwargs={"pk": job.pk})
|
|
|
|
assert client.get(reverse("profiles:list")).status_code == 200
|
|
assert client.get(reverse("profiles:edit", kwargs={"pk": profile.pk})).status_code == 200
|
|
assert client.get(reverse("sources:list")).status_code == 200
|
|
assert client.get(reverse("dashboard:system")).status_code == 200
|
|
applications_response = client.get(reverse("jobs:applications"))
|
|
assert applications_response.status_code == 200
|
|
assert [column["status"] for column in applications_response.context["pipeline_columns"]] == [
|
|
status for status, _label in Application.Status.choices
|
|
]
|
|
|
|
source.policy = Source.Policy.DENY
|
|
source.save(update_fields=["policy"])
|
|
response = client.post(reverse("sources:retry", kwargs={"pk": source.pk}))
|
|
assert response.status_code == 302
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.django_db
|
|
def test_sources_candidate_bulk_actions_and_evidence_view(client, user, profile):
|
|
profile.home_postal_code = "2400"
|
|
profile.max_distance_km = 40
|
|
profile.home_latitude = None
|
|
profile.home_longitude = None
|
|
profile.save(
|
|
update_fields=[
|
|
"home_postal_code",
|
|
"max_distance_km",
|
|
"home_latitude",
|
|
"home_longitude",
|
|
"updated_at",
|
|
]
|
|
)
|
|
candidate = Source.objects.create(
|
|
name="Discoverd candidate",
|
|
source_type=Source.Type.EMPLOYER,
|
|
base_url="https://jobs.example.org/vacatures/",
|
|
domain="jobs.example.org",
|
|
status=Source.Status.CANDIDATE,
|
|
policy=Source.Policy.REVIEW,
|
|
metadata={
|
|
"discovery": [
|
|
{
|
|
"url": "https://jobs.example.org/careers",
|
|
"discovered_from": "html",
|
|
"reason": "career-link",
|
|
"confidence": 0.9,
|
|
"label": "Werken bij",
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
other = Source.objects.create(
|
|
name="Another source",
|
|
source_type=Source.Type.EMPLOYER,
|
|
base_url="https://other.example.org/",
|
|
domain="other.example.org",
|
|
status=Source.Status.CANDIDATE,
|
|
policy=Source.Policy.REVIEW,
|
|
)
|
|
watchlist = Source.objects.create(
|
|
name="Interessante werkgever zonder stabiele feed",
|
|
source_type=Source.Type.EMPLOYER,
|
|
base_url="https://watch.example.org/jobs/",
|
|
domain="watch.example.org",
|
|
status=Source.Status.PAUSED,
|
|
policy=Source.Policy.REVIEW,
|
|
metadata={
|
|
"watchlist": True,
|
|
"municipality_scope": "Olen",
|
|
"watchlist_reason": "Regionaal interessante werkgever",
|
|
"current_observation": "Momenteel geen lokale vacature bevestigd.",
|
|
"public_job_page": "https://watch.example.org/jobs/",
|
|
},
|
|
)
|
|
|
|
client.force_login(user)
|
|
response = client.get(reverse("sources:list"))
|
|
assert response.status_code == 200
|
|
content = response.content.decode("utf-8")
|
|
assert "jobs.example.org/careers" in content
|
|
assert "Platformmailboxen" in content
|
|
assert "Mailbox koppelen en testen" in content
|
|
assert "elke vijf minuten" in content
|
|
assert "2400 Mol" in content
|
|
assert "maximaal 40 km" in content
|
|
assert "Coördinaten ontbreken nog" in content
|
|
assert "Interessante werkgeverswaaklijst" in content
|
|
assert watchlist.name in content
|
|
assert response.context["watchlist_count"] == 1
|
|
assert {item.pk for item in response.context["sources"]} == {candidate.pk, other.pk}
|
|
|
|
response = client.post(
|
|
reverse("sources:bulk"),
|
|
{"action": "approve_to_trial", "source_ids": [str(candidate.pk)]},
|
|
)
|
|
assert response.status_code == 302
|
|
candidate.refresh_from_db()
|
|
assert candidate.status == Source.Status.TRIAL
|
|
|
|
response = client.post(
|
|
reverse("sources:bulk"),
|
|
{"action": "dismiss", "source_ids": [str(other.pk)]},
|
|
)
|
|
assert response.status_code == 302
|
|
other.refresh_from_db()
|
|
assert other.status == Source.Status.DISABLED
|
|
assert other.policy == Source.Policy.DENY
|
|
|
|
response = client.post(
|
|
reverse("sources:bulk"),
|
|
{"action": "dismiss", "source_ids": [str(watchlist.pk)]},
|
|
)
|
|
assert response.status_code == 302
|
|
watchlist.refresh_from_db()
|
|
assert watchlist.status == Source.Status.PAUSED
|
|
assert watchlist.policy == Source.Policy.REVIEW
|