from __future__ import annotations import re from urllib.parse import urljoin from bs4 import BeautifulSoup from .base import ExtractedJob, ExtractionResult, FieldEvidence LABEL_PATTERNS = { "location": re.compile(r"^(locatie|location|lieu|plaats|standplaats)\s*:?$", re.I), "date_posted": re.compile(r"^(publicatiedatum|geplaatst|date posted|published)\s*:?$", re.I), "employment_type": re.compile(r"^(dienstverband|contract|employment type)\s*:?$", re.I), "employer": re.compile(r"^(werkgever|employer|company|organisatie|société)\s*:?$", re.I), } class GenericHtmlAdapter: parser_key = "generic-html" parser_version = "1.1.0" @staticmethod def _meta(soup: BeautifulSoup, *names: str) -> str: for name in names: tag = soup.find("meta", attrs={"property": name}) or soup.find( "meta", attrs={"name": name} ) if tag and tag.get("content"): return str(tag["content"]).strip() return "" @staticmethod def _label_value(soup: BeautifulSoup, pattern: re.Pattern[str]) -> str: label = soup.find(string=lambda value: bool(value and pattern.match(value.strip()))) if not label: return "" parent = label.parent if not parent: return "" sibling = parent.find_next_sibling() if sibling: return sibling.get_text(" ", strip=True) text = parent.get_text(" ", strip=True) return pattern.sub("", text).strip(" :-") def extract(self, content: str, *, url: str) -> ExtractionResult: soup = BeautifulSoup(content, "lxml") for element in soup(["script", "style", "noscript", "template"]): element.decompose() h1 = soup.find("h1") title = (h1.get_text(" ", strip=True) if h1 else "") or self._meta( soup, "og:title", "twitter:title" ) if not title and soup.title: title = soup.title.get_text(" ", strip=True) title = re.sub(r"\s+[|\\u2013\\u2014-]\s+.*$", "", title).strip() if not title: return ExtractionResult([], self.parser_key, self.parser_version, 0.0, ["Geen titel"]) employer = self._meta(soup, "og:site_name", "application-name") or self._label_value( soup, LABEL_PATTERNS["employer"] ) location = self._label_value(soup, LABEL_PATTERNS["location"]) date_posted = self._label_value(soup, LABEL_PATTERNS["date_posted"]) employment_type = self._label_value(soup, LABEL_PATTERNS["employment_type"]) main = soup.find("main") or soup.find("article") or soup.body description_html = str(main) if main else "" description_text = ( main.get_text("\n", strip=True) if main else soup.get_text("\n", strip=True) ) canonical = soup.find("link", rel=lambda value: value and "canonical" in value) job_url = ( urljoin(url, canonical.get("href")) if canonical and canonical.get("href") else url ) evidence = [ FieldEvidence("title", "html-heading", 0.78, title[:240]), FieldEvidence("employer_name", "html-meta-or-label", 0.65, employer[:240]), FieldEvidence("location_text", "html-label", 0.62, location[:240]), FieldEvidence("description", "html-main", 0.70, description_text[:300]), ] if date_posted: evidence.append(FieldEvidence("date_posted", "html-label", 0.62, date_posted[:40])) if employment_type: evidence.append( FieldEvidence("employment_types", "html-label", 0.62, employment_type[:120]) ) job = ExtractedJob( url=job_url, title=title, employer_name=employer, location_text=location, description_html=description_html, description_text=description_text, date_posted=date_posted, employment_types=[employment_type] if employment_type else [], raw={"generic_html": True}, evidence=evidence, ) return ExtractionResult([job], self.parser_key, self.parser_version, 0.70)