54 lines
1.8 KiB
Python
54 lines
1.8 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from time import mktime
|
|
|
|
import feedparser
|
|
from bs4 import BeautifulSoup
|
|
|
|
from .base import ExtractedJob, ExtractionResult, FieldEvidence
|
|
|
|
|
|
class RssAdapter:
|
|
parser_key = "rss-atom"
|
|
parser_version = "1.0.0"
|
|
|
|
def extract(self, content: str, *, url: str) -> ExtractionResult:
|
|
feed = feedparser.parse(content)
|
|
jobs: list[ExtractedJob] = []
|
|
for entry in feed.entries:
|
|
title = str(entry.get("title") or "").strip()
|
|
link = str(entry.get("link") or "").strip()
|
|
if not title or not link:
|
|
continue
|
|
summary = str(entry.get("summary") or entry.get("description") or "")
|
|
text = BeautifulSoup(summary, "lxml").get_text("\n", strip=True)
|
|
published = ""
|
|
if entry.get("published_parsed"):
|
|
published = datetime.fromtimestamp(mktime(entry.published_parsed)).isoformat()
|
|
jobs.append(
|
|
ExtractedJob(
|
|
url=link,
|
|
title=title,
|
|
external_id=str(entry.get("id") or ""),
|
|
description_html=summary,
|
|
description_text=text,
|
|
date_posted=published,
|
|
raw=dict(entry),
|
|
evidence=[
|
|
FieldEvidence("title", "rss", 0.88, title[:240]),
|
|
FieldEvidence("description", "rss", 0.80, text[:300]),
|
|
],
|
|
)
|
|
)
|
|
warnings: list[str] = []
|
|
if getattr(feed, "bozo", False):
|
|
warnings.append(str(getattr(feed, "bozo_exception", "Ongeldige feed")))
|
|
return ExtractionResult(
|
|
jobs,
|
|
self.parser_key,
|
|
self.parser_version,
|
|
0.82 if jobs else 0.0,
|
|
warnings,
|
|
)
|