88 lines
3.4 KiB
Python
88 lines
3.4 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from urllib.parse import urljoin
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
|
|
|
LABEL_PATTERNS = {
|
|
"location": re.compile(r"^(locatie|location|lieu|plaats|standplaats)\s*:?$", re.I),
|
|
"employer": re.compile(r"^(werkgever|employer|company|organisatie|société)\s*:?$", re.I),
|
|
}
|
|
|
|
|
|
class GenericHtmlAdapter:
|
|
parser_key = "generic-html"
|
|
parser_version = "1.0.0"
|
|
|
|
@staticmethod
|
|
def _meta(soup: BeautifulSoup, *names: str) -> str:
|
|
for name in names:
|
|
tag = soup.find("meta", attrs={"property": name}) or soup.find(
|
|
"meta", attrs={"name": name}
|
|
)
|
|
if tag and tag.get("content"):
|
|
return str(tag["content"]).strip()
|
|
return ""
|
|
|
|
@staticmethod
|
|
def _label_value(soup: BeautifulSoup, pattern: re.Pattern[str]) -> str:
|
|
label = soup.find(string=lambda value: bool(value and pattern.match(value.strip())))
|
|
if not label:
|
|
return ""
|
|
parent = label.parent
|
|
if not parent:
|
|
return ""
|
|
sibling = parent.find_next_sibling()
|
|
if sibling:
|
|
return sibling.get_text(" ", strip=True)
|
|
text = parent.get_text(" ", strip=True)
|
|
return pattern.sub("", text).strip(" :-")
|
|
|
|
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
|
soup = BeautifulSoup(content, "lxml")
|
|
for element in soup(["script", "style", "noscript", "template"]):
|
|
element.decompose()
|
|
h1 = soup.find("h1")
|
|
title = (h1.get_text(" ", strip=True) if h1 else "") or self._meta(
|
|
soup, "og:title", "twitter:title"
|
|
)
|
|
if not title and soup.title:
|
|
title = soup.title.get_text(" ", strip=True)
|
|
title = re.sub(r"\s+[|\\u2013\\u2014-]\s+.*$", "", title).strip()
|
|
if not title:
|
|
return ExtractionResult([], self.parser_key, self.parser_version, 0.0, ["Geen titel"])
|
|
|
|
employer = self._meta(soup, "og:site_name", "application-name") or self._label_value(
|
|
soup, LABEL_PATTERNS["employer"]
|
|
)
|
|
location = self._label_value(soup, LABEL_PATTERNS["location"])
|
|
main = soup.find("main") or soup.find("article") or soup.body
|
|
description_html = str(main) if main else ""
|
|
description_text = (
|
|
main.get_text("\n", strip=True) if main else soup.get_text("\n", strip=True)
|
|
)
|
|
canonical = soup.find("link", rel=lambda value: value and "canonical" in value)
|
|
job_url = (
|
|
urljoin(url, canonical.get("href")) if canonical and canonical.get("href") else url
|
|
)
|
|
evidence = [
|
|
FieldEvidence("title", "html-heading", 0.78, title[:240]),
|
|
FieldEvidence("employer_name", "html-meta-or-label", 0.65, employer[:240]),
|
|
FieldEvidence("location_text", "html-label", 0.62, location[:240]),
|
|
FieldEvidence("description", "html-main", 0.70, description_text[:300]),
|
|
]
|
|
job = ExtractedJob(
|
|
url=job_url,
|
|
title=title,
|
|
employer_name=employer,
|
|
location_text=location,
|
|
description_html=description_html,
|
|
description_text=description_text,
|
|
raw={"generic_html": True},
|
|
evidence=evidence,
|
|
)
|
|
return ExtractionResult([job], self.parser_key, self.parser_version, 0.70)
|