This commit is contained in:
+8
-10
@@ -20,7 +20,11 @@ from apps.sources.services.fetcher import (
|
||||
RateLimitedError,
|
||||
fetch_url,
|
||||
)
|
||||
from apps.sources.services.health import canary_recovery_sources, evaluate_source_health, start_health_canary
|
||||
from apps.sources.services.health import (
|
||||
canary_recovery_sources,
|
||||
evaluate_source_health,
|
||||
start_health_canary,
|
||||
)
|
||||
from apps.sources.services.policy import assess_url
|
||||
from apps.sources.services.scheduling import (
|
||||
acquire_source_lease,
|
||||
@@ -82,9 +86,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
|
||||
try:
|
||||
fetched = fetch_url(source.base_url, source=source, conditional_headers=headers)
|
||||
if fetched.status_code == 304:
|
||||
source.schedule_after_success(
|
||||
jitter_seconds=calculate_success_jitter_seconds(source)
|
||||
)
|
||||
source.schedule_after_success(jitter_seconds=calculate_success_jitter_seconds(source))
|
||||
run.finish(SourceRun.Status.SUCCESS, http_status=304)
|
||||
return {"status": "not_modified"}
|
||||
content_type = fetched.headers.get("content-type", "")
|
||||
@@ -110,9 +112,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
|
||||
source.etag = fetched.headers.get("etag", source.etag)
|
||||
source.last_modified = fetched.headers.get("last-modified", source.last_modified)
|
||||
source.save(update_fields=["etag", "last_modified", "updated_at"])
|
||||
source.schedule_after_success(
|
||||
jitter_seconds=calculate_success_jitter_seconds(source)
|
||||
)
|
||||
source.schedule_after_success(jitter_seconds=calculate_success_jitter_seconds(source))
|
||||
run.finish(
|
||||
SourceRun.Status.SUCCESS,
|
||||
http_status=fetched.status_code,
|
||||
@@ -159,9 +159,7 @@ def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]
|
||||
)
|
||||
return {"status": "timeout", "backoff": backoff}
|
||||
except FetchError as exc:
|
||||
backoff = calculate_failure_backoff_seconds(
|
||||
source, failure_count=source.failure_count + 1
|
||||
)
|
||||
backoff = calculate_failure_backoff_seconds(source, failure_count=source.failure_count + 1)
|
||||
source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
|
||||
run.finish(
|
||||
SourceRun.Status.FAILED,
|
||||
|
||||
Reference in New Issue
Block a user