This commit is contained in:
@@ -6,10 +6,24 @@ from django.utils import timezone
|
||||
|
||||
from apps.sources.models import Source, SourceRobotsCache
|
||||
from apps.sources.services.robots import assess_robots
|
||||
from apps.sources.services.url_security import ValidatedUrl
|
||||
|
||||
|
||||
def _allow_test_dns(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"apps.sources.services.robots.validate_public_url",
|
||||
lambda url, **kwargs: ValidatedUrl(
|
||||
url=url,
|
||||
hostname="jobs.example.org",
|
||||
port=443,
|
||||
addresses=("93.184.216.34",),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.security
|
||||
def test_robots_uses_user_agent_matching_and_allows_default(db):
|
||||
def test_robots_uses_user_agent_matching_and_allows_default(db, monkeypatch):
|
||||
_allow_test_dns(monkeypatch)
|
||||
source = Source.objects.create(
|
||||
name="Example jobs",
|
||||
source_type=Source.Type.EMPLOYER,
|
||||
@@ -45,7 +59,8 @@ def test_robots_uses_user_agent_matching_and_allows_default(db):
|
||||
|
||||
|
||||
@pytest.mark.security
|
||||
def test_robots_cache_refreshes_after_ttl(db, settings):
|
||||
def test_robots_cache_refreshes_after_ttl(db, settings, monkeypatch):
|
||||
_allow_test_dns(monkeypatch)
|
||||
source = Source.objects.create(
|
||||
name="Example jobs",
|
||||
source_type=Source.Type.EMPLOYER,
|
||||
@@ -63,7 +78,9 @@ def test_robots_cache_refreshes_after_ttl(db, settings):
|
||||
return httpx.Response(200, text="User-agent: *\nAllow: /")
|
||||
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
assess_robots(source.base_url, source=source, user_agent="test-agent", client=client, now=timezone.now())
|
||||
assess_robots(
|
||||
source.base_url, source=source, user_agent="test-agent", client=client, now=timezone.now()
|
||||
)
|
||||
assert calls["count"] == 1
|
||||
assess_robots(
|
||||
source.base_url,
|
||||
|
||||
Reference in New Issue
Block a user