86 lines
3.0 KiB
Python
86 lines
3.0 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from apps.sources.models import Source, SourceRun
|
|
from apps.sources.services.fetcher import FetchedDocument
|
|
from apps.sources.services.manual_import import ManualImportError, import_manual_source
|
|
|
|
|
|
def test_manual_import_url_uses_fetch_and_succeeds(monkeypatch, user):
|
|
def fake_fetch(*_args, **_kwargs):
|
|
return FetchedDocument(
|
|
requested_url="https://jobs.example.org/jobs",
|
|
final_url="https://jobs.example.org/jobs",
|
|
status_code=200,
|
|
headers={
|
|
"content-type": "text/html; charset=utf-8",
|
|
"etag": "abc",
|
|
"last-modified": "Mon, 01 Jan 2000 00:00:00 GMT",
|
|
},
|
|
content=b"<html><body>vacature</body></html>",
|
|
)
|
|
|
|
def fake_process(document):
|
|
return {
|
|
"extracted": 2,
|
|
"created": 1,
|
|
"updated": 1,
|
|
"duplicates": 0,
|
|
"parser": "manual-test",
|
|
"warnings": [],
|
|
}
|
|
|
|
monkeypatch.setattr("apps.sources.services.manual_import.fetch_url", fake_fetch)
|
|
monkeypatch.setattr("apps.sources.services.manual_import.process_raw_document", fake_process)
|
|
|
|
summary = import_manual_source(actor=user, source_url="https://jobs.example.org/jobs")
|
|
source = Source.objects.get(pk=summary.source_id)
|
|
|
|
assert summary.mode == "url"
|
|
assert summary.extracted_count == 2
|
|
assert summary.created_count == 1
|
|
assert source.source_type == Source.Type.MANUAL
|
|
assert source.domain == "jobs.example.org"
|
|
assert SourceRun.objects.filter(source=source, status=SourceRun.Status.SUCCESS).exists()
|
|
|
|
|
|
def test_manual_import_paste_text_sanitizes_and_skips_fetch(monkeypatch, user):
|
|
def fail_fetch(*_args, **_kwargs):
|
|
raise AssertionError("Fetcher should not be called for pasted text.")
|
|
|
|
def fake_process(document):
|
|
return {
|
|
"extracted": 0,
|
|
"created": 0,
|
|
"updated": 0,
|
|
"duplicates": 0,
|
|
"parser": "manual-test",
|
|
"warnings": ["noop"],
|
|
}
|
|
|
|
monkeypatch.setattr("apps.sources.services.manual_import.fetch_url", fail_fetch)
|
|
monkeypatch.setattr("apps.sources.services.manual_import.process_raw_document", fake_process)
|
|
|
|
summary = import_manual_source(
|
|
actor=user, pasted_text="<script>alert('x')</script>\nVacaturetekst"
|
|
)
|
|
source = Source.objects.get(pk=summary.source_id)
|
|
|
|
assert summary.mode == "paste"
|
|
assert source.source_type == Source.Type.MANUAL
|
|
assert source.base_url.startswith("https://manual-")
|
|
|
|
|
|
def test_manual_import_rejects_denied_domain(user):
|
|
with pytest.raises(ManualImportError):
|
|
import_manual_source(actor=user, source_url="https://linkedin.com/jobs/123")
|
|
|
|
|
|
def test_manual_import_rejects_paste_payload_when_too_large(user, settings):
|
|
settings.MANUAL_IMPORT_PASTE_MAX_BYTES = 12
|
|
huge_text = "a" * 20
|
|
|
|
with pytest.raises(ManualImportError):
|
|
import_manual_source(actor=user, pasted_text=huge_text)
|