26 lines
506 B
YAML
26 lines
506 B
YAML
mode: strict
|
|
honor_robots: true
|
|
crawl_logged_in_pages: false
|
|
solve_captcha: false
|
|
public_ats_endpoints: false
|
|
max_concurrency_per_domain: 1
|
|
minimum_interval_seconds: 30
|
|
max_redirects: 5
|
|
max_document_bytes: 3000000
|
|
deny_domains:
|
|
- linkedin.com
|
|
- indeed.com
|
|
- indeed.be
|
|
- stepstone.be
|
|
- jobat.be
|
|
- vdab.be
|
|
- ictjob.be
|
|
- eurobrussels.com
|
|
- brusselsjobs.com
|
|
review_on:
|
|
- login_required
|
|
- captcha_or_challenge
|
|
- unexpected_cross_domain_redirect
|
|
- terms_unclear
|
|
- repeated_403_or_429
|