This commit is contained in:
@@ -18,9 +18,9 @@ from apps.jobs.services.pipeline import process_raw_document
|
||||
from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceRun
|
||||
from apps.sources.services.canonicalize import canonicalize_url
|
||||
from apps.sources.services.fetcher import (
|
||||
FetchedDocument,
|
||||
FetchError,
|
||||
FetchTimeoutError,
|
||||
FetchedDocument,
|
||||
PolicyBlockedError,
|
||||
RateLimitedError,
|
||||
fetch_url,
|
||||
@@ -88,11 +88,7 @@ def _mode_source_name(domain: str, *, mode: str) -> str:
|
||||
|
||||
def _ensure_manual_review(source: Source, actor) -> None:
|
||||
review = source.latest_policy_review
|
||||
if (
|
||||
review
|
||||
and not review.is_expired
|
||||
and review.decision == SourcePolicyReview.Decision.ALLOW
|
||||
):
|
||||
if review and not review.is_expired and review.decision == SourcePolicyReview.Decision.ALLOW:
|
||||
return
|
||||
create_policy_review(
|
||||
source,
|
||||
@@ -120,9 +116,7 @@ def _ensure_manual_source(*, domain: str, source_url: str, actor, mode: str) ->
|
||||
source.base_url = source_url
|
||||
source.status = Source.Status.CANDIDATE
|
||||
source.policy = Source.Policy.ALLOW
|
||||
source.save(
|
||||
update_fields=["name", "base_url", "status", "policy", "updated_at"]
|
||||
)
|
||||
source.save(update_fields=["name", "base_url", "status", "policy", "updated_at"])
|
||||
_ensure_manual_review(source, actor=actor)
|
||||
return source
|
||||
|
||||
@@ -175,7 +169,7 @@ def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImpo
|
||||
source = document.source
|
||||
source_name = source.name if source else ""
|
||||
warnings: list[str] = list(metrics.get("warnings", []))
|
||||
warnings_count = len(warnings)
|
||||
len(warnings)
|
||||
source_run.finish(
|
||||
SourceRun.Status.SUCCESS,
|
||||
http_status=document.source_run.http_status if document.source_run else None,
|
||||
@@ -186,9 +180,8 @@ def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImpo
|
||||
metrics={"parser": metrics["parser"], "warnings": warnings},
|
||||
)
|
||||
jobs = _build_jobs_from_document(document)
|
||||
if metrics["created"] == 0 and metrics["updated"] == 0:
|
||||
if not warnings:
|
||||
warnings.append("De bron leverde geen herkenbare vacaturedata op.")
|
||||
if metrics["created"] == 0 and metrics["updated"] == 0 and not warnings:
|
||||
warnings.append("De bron leverde geen herkenbare vacaturedata op.")
|
||||
if not warnings:
|
||||
# keep stable, machine-readable payload shape
|
||||
warnings = []
|
||||
|
||||
Reference in New Issue
Block a user