137 lines
5.1 KiB
Python
137 lines
5.1 KiB
Python
import hashlib
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from django.utils import timezone
|
|
|
|
from apps.jobs.models import Employer, JobPosting, JobSourceAlias, ScoreRun
|
|
from apps.jobs.services.pipeline import process_raw_document
|
|
from apps.sources.models import RawDocument, Source
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.django_db
|
|
def test_pipeline_is_idempotent_and_scores(source, profile):
|
|
content = Path("fixtures/pages/sample_jsonld_job.html").read_text(encoding="utf-8")
|
|
document = RawDocument.objects.create(
|
|
source=source,
|
|
url="https://jobs.example.org/vacatures/infrastructure-engineer",
|
|
final_url="https://jobs.example.org/vacatures/infrastructure-engineer",
|
|
kind=RawDocument.Kind.HTML,
|
|
content_type="text/html",
|
|
content_hash=hashlib.sha256(content.encode()).hexdigest(),
|
|
body_text=content,
|
|
byte_length=len(content.encode()),
|
|
retain_until=timezone.now() + timedelta(days=7),
|
|
)
|
|
first = process_raw_document(document)
|
|
second = process_raw_document(document)
|
|
assert first["created"] == 1
|
|
assert second["created"] == 0
|
|
assert JobPosting.objects.count() == 1
|
|
assert JobSourceAlias.objects.count() == 1
|
|
assert ScoreRun.objects.filter(profile=profile).count() == 2
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_pipeline_resolves_recruiter_alias_to_direct_employer():
|
|
source = Source.objects.create(
|
|
name="SmartRecruiters",
|
|
source_type=Source.Type.ATS,
|
|
base_url="https://jobs.smartrecruiters.com/example",
|
|
domain="jobs.smartrecruiters.com",
|
|
status=Source.Status.ACTIVE,
|
|
policy=Source.Policy.ALLOW,
|
|
)
|
|
employer = Employer.objects.create(
|
|
name="Example Public IT",
|
|
normalized_name="example public it",
|
|
domain="jobs.example.org",
|
|
is_direct_employer=True,
|
|
is_recruiter=False,
|
|
confidence=0.9,
|
|
)
|
|
direct_job = JobPosting.objects.create(
|
|
employer=employer,
|
|
original_title="Security Engineer",
|
|
normalized_title="security engineer",
|
|
canonical_url="https://jobs.example.org/vacatures/security-engineer",
|
|
canonical_key="8" * 64,
|
|
content_hash="e" * 64,
|
|
description_text="Beveiliger, hybride werk met support op locatie.",
|
|
raw_location="Antwerpen",
|
|
region="Antwerpen",
|
|
municipality="Antwerpen",
|
|
workplace_type=JobPosting.Workplace.HYBRID,
|
|
employment_types=["full_time"],
|
|
direct_employer=True,
|
|
)
|
|
content = """
|
|
<html lang="nl">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>Security Engineer | Example Public IT</title>
|
|
<link rel="canonical" href="https://jobs.smartrecruiters.com/example/security-engineer">
|
|
<script type="application/ld+json">
|
|
{
|
|
"@context": "https://schema.org",
|
|
"@type": "JobPosting",
|
|
"identifier": {
|
|
"@type": "PropertyValue",
|
|
"name": "Example Public IT",
|
|
"value": "VR-DEMO-SEC-001"
|
|
},
|
|
"title": "Security Engineer",
|
|
"description": "<p>Beveiliging, monitoring en hybride support.</p>",
|
|
"datePosted": "2026-07-20",
|
|
"validThrough": "2026-08-31T23:59:00+02:00",
|
|
"employmentType": ["FULL_TIME"],
|
|
"hiringOrganization": {
|
|
"@type": "Organization",
|
|
"name": "Example Public IT",
|
|
"sameAs": "https://www.example.org"
|
|
},
|
|
"jobLocation": {
|
|
"@type": "Place",
|
|
"address": {
|
|
"@type": "PostalAddress",
|
|
"streetAddress": "Voorbeeldstraat 1",
|
|
"addressLocality": "Antwerpen",
|
|
"addressRegion": "Antwerpen",
|
|
"postalCode": "2000",
|
|
"addressCountry": "BE"
|
|
}
|
|
},
|
|
"jobLocationType": "HYBRID",
|
|
"url": "/example/security-engineer"
|
|
}
|
|
</script>
|
|
</head>
|
|
<body><main><h1>Security Engineer</h1></main></body>
|
|
</html>
|
|
""".strip()
|
|
body_hash = hashlib.sha256(content.encode()).hexdigest()
|
|
document = RawDocument.objects.create(
|
|
source=source,
|
|
url="https://jobs.smartrecruiters.com/example/security-engineer",
|
|
final_url="https://jobs.smartrecruiters.com/example/security-engineer",
|
|
kind=RawDocument.Kind.HTML,
|
|
content_type="text/html",
|
|
content_hash=body_hash,
|
|
body_text=content,
|
|
byte_length=len(content.encode()),
|
|
retain_until=timezone.now() + timedelta(days=7),
|
|
)
|
|
result = process_raw_document(document)
|
|
direct_job.refresh_from_db()
|
|
assert direct_job.canonical_url == "https://jobs.example.org/vacatures/security-engineer"
|
|
assert result["created"] == 0
|
|
assert result["duplicates"] == 1
|
|
alias = JobSourceAlias.objects.get(job=direct_job, source=source)
|
|
payload = alias.payload.get("employer_resolution")
|
|
assert payload is not None
|
|
assert payload["reason"] == "resolved_direct_match"
|
|
assert payload["resolved_direct"] is True
|
|
assert payload["canonical_url"] == "https://jobs.example.org/vacatures/security-engineer"
|