317 lines
12 KiB
Python
317 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import shutil
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from app.core.config import get_settings
|
|
from app.core.errors import AppError
|
|
|
|
|
|
class StorageService:
|
|
UPLOAD_CHUNK_SIZE = 8 * 1024 * 1024
|
|
|
|
@staticmethod
|
|
def _base_dir() -> Path:
|
|
return Path(get_settings().storage_root).resolve()
|
|
|
|
@staticmethod
|
|
def assert_within_storage_root(
|
|
path: str | Path,
|
|
*,
|
|
label: str = "artifact",
|
|
settings: Any = None,
|
|
) -> Path:
|
|
"""Resolve a path and refuse anything outside the configured storage root.
|
|
|
|
``tile_manifest_path`` arrives in the analysis request and a manifest
|
|
entry may name an absolute tile path, so without this an API field is an
|
|
unbounded reference to the host filesystem. It is also the persistence
|
|
rule the product already states: only a governed, runtime-produced
|
|
artifact may be consumed, and a file outside the root is not one.
|
|
|
|
Resolution happens before the comparison, so ``..`` cannot climb out and
|
|
a sibling that merely shares a name prefix does not pass.
|
|
"""
|
|
|
|
from app.core.config import get_settings
|
|
|
|
settings = settings or get_settings()
|
|
raw = str(path or "").strip()
|
|
if not raw:
|
|
raise AppError(
|
|
code="STORAGE_PATH_OUTSIDE_ROOT",
|
|
message=f"A {label} path is required.",
|
|
status_code=400,
|
|
)
|
|
|
|
resolved = Path(raw).expanduser().resolve()
|
|
if getattr(settings, "allow_external_artifact_paths", False):
|
|
return resolved
|
|
|
|
root = Path(settings.storage_root).resolve()
|
|
if resolved != root and root not in resolved.parents:
|
|
raise AppError(
|
|
code="STORAGE_PATH_OUTSIDE_ROOT",
|
|
message=(
|
|
f"The {label} path lies outside the configured storage root and is therefore not a "
|
|
"governed artifact."
|
|
),
|
|
details={"storage_root": str(root)},
|
|
status_code=400,
|
|
)
|
|
return resolved
|
|
|
|
@staticmethod
|
|
def normalize_dataset_type(dataset_type: str) -> str:
|
|
normalized = dataset_type.strip().lower()
|
|
if normalized == "geojson":
|
|
return "vector"
|
|
return normalized
|
|
|
|
@staticmethod
|
|
def _safe_filename(value: str) -> str:
|
|
value = value.strip().replace("\\", "/").split("/")[-1]
|
|
fallback = "upload"
|
|
if not value:
|
|
return fallback
|
|
allowed = []
|
|
for char in value:
|
|
if char.isalnum() or char in "-_ .":
|
|
allowed.append(char)
|
|
else:
|
|
allowed.append("_")
|
|
cleaned = "".join(allowed)
|
|
cleaned = cleaned.strip(" .")
|
|
return cleaned or fallback
|
|
|
|
@staticmethod
|
|
def dataset_root(project_id: str, dataset_id: str, dataset_type: str) -> Path:
|
|
return StorageService._base_dir() / "uploads" / project_id / dataset_type / dataset_id
|
|
|
|
@staticmethod
|
|
def derived_raster_root(project_id: str, dataset_id: str) -> Path:
|
|
return StorageService._base_dir() / "rasters" / "derived" / project_id / dataset_id
|
|
|
|
@staticmethod
|
|
def preview_root(project_id: str, dataset_id: str) -> Path:
|
|
return StorageService._base_dir() / "previews" / project_id / dataset_id
|
|
|
|
@staticmethod
|
|
def raster_tiles_root(project_id: str, source_dataset_id: str, tile_set_id: str) -> Path:
|
|
return StorageService._base_dir() / "tiles" / project_id / source_dataset_id / tile_set_id
|
|
|
|
@staticmethod
|
|
def dataset_file_path(
|
|
project_id: str,
|
|
dataset_id: str,
|
|
dataset_type: str,
|
|
original_filename: str,
|
|
) -> str:
|
|
safe_original = StorageService._safe_filename(original_filename)
|
|
stored_filename = f"{dataset_id}_{safe_original}"
|
|
return str(StorageService.dataset_root(project_id, dataset_id, dataset_type) / stored_filename)
|
|
|
|
@staticmethod
|
|
def calculate_checksum_sha256(content: bytes) -> str:
|
|
digest = hashlib.sha256()
|
|
digest.update(content)
|
|
return digest.hexdigest()
|
|
|
|
@staticmethod
|
|
def persist_dataset_file(
|
|
project_id: str,
|
|
dataset_id: str,
|
|
dataset_type: str,
|
|
original_filename: str,
|
|
content: bytes,
|
|
content_type: str | None,
|
|
) -> dict[str, Any]:
|
|
normalized_type = StorageService.normalize_dataset_type(dataset_type)
|
|
file_path = Path(StorageService.dataset_file_path(project_id, dataset_id, normalized_type, original_filename))
|
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
with file_path.open("wb") as stream:
|
|
stream.write(content)
|
|
|
|
metadata: dict[str, Any] = {
|
|
"original_filename": StorageService._safe_filename(original_filename),
|
|
"stored_filename": file_path.name,
|
|
"content_type": content_type or "application/octet-stream",
|
|
"size_bytes": len(content),
|
|
"checksum_sha256": StorageService.calculate_checksum_sha256(content),
|
|
"storage_path": str(file_path),
|
|
}
|
|
return metadata
|
|
|
|
@staticmethod
|
|
async def persist_upload_file(
|
|
*,
|
|
project_id: str,
|
|
dataset_id: str,
|
|
dataset_type: str,
|
|
original_filename: str,
|
|
upload: Any,
|
|
content_type: str | None,
|
|
max_bytes: int,
|
|
chunk_size: int | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Stream an UploadFile to governed storage with a hard byte limit.
|
|
|
|
The reverse proxy limit is defense in depth. This backend boundary is
|
|
authoritative as direct/loopback requests can bypass that proxy.
|
|
"""
|
|
|
|
if max_bytes <= 0:
|
|
raise ValueError("max_bytes must be positive")
|
|
resolved_chunk_size = chunk_size or StorageService.UPLOAD_CHUNK_SIZE
|
|
if resolved_chunk_size <= 0:
|
|
raise ValueError("chunk_size must be positive")
|
|
normalized_type = StorageService.normalize_dataset_type(dataset_type)
|
|
file_path = Path(
|
|
StorageService.dataset_file_path(
|
|
project_id,
|
|
dataset_id,
|
|
normalized_type,
|
|
original_filename,
|
|
)
|
|
)
|
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
digest = hashlib.sha256()
|
|
size_bytes = 0
|
|
try:
|
|
with file_path.open("wb") as stream:
|
|
while True:
|
|
chunk = await upload.read(resolved_chunk_size)
|
|
if not chunk:
|
|
break
|
|
size_bytes += len(chunk)
|
|
if size_bytes > max_bytes:
|
|
raise AppError(
|
|
code="UPLOAD_TOO_LARGE",
|
|
message="Upload exceeds the configured backend size limit.",
|
|
details={
|
|
"max_bytes": max_bytes,
|
|
"max_upload_mb": max_bytes // (1024 * 1024),
|
|
},
|
|
status_code=413,
|
|
)
|
|
stream.write(chunk)
|
|
digest.update(chunk)
|
|
except Exception:
|
|
file_path.unlink(missing_ok=True)
|
|
parent = file_path.parent
|
|
if parent.exists() and parent.is_dir() and not any(parent.iterdir()):
|
|
parent.rmdir()
|
|
raise
|
|
return {
|
|
"original_filename": StorageService._safe_filename(original_filename),
|
|
"stored_filename": file_path.name,
|
|
"content_type": content_type or "application/octet-stream",
|
|
"size_bytes": size_bytes,
|
|
"checksum_sha256": digest.hexdigest(),
|
|
"storage_path": str(file_path),
|
|
}
|
|
|
|
@staticmethod
|
|
def persist_dataset_file_from_path(
|
|
project_id: str,
|
|
dataset_id: str,
|
|
dataset_type: str,
|
|
original_filename: str,
|
|
source_path: str | Path,
|
|
content_type: str | None,
|
|
) -> dict[str, Any]:
|
|
source = Path(source_path).resolve()
|
|
if not source.is_file():
|
|
raise FileNotFoundError(f"Dataset source artifact does not exist: {source}")
|
|
|
|
normalized_type = StorageService.normalize_dataset_type(dataset_type)
|
|
file_path = Path(StorageService.dataset_file_path(project_id, dataset_id, normalized_type, original_filename))
|
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
digest = hashlib.sha256()
|
|
size_bytes = 0
|
|
with source.open("rb") as input_stream, file_path.open("wb") as output_stream:
|
|
for chunk in iter(lambda: input_stream.read(8 * 1024 * 1024), b""):
|
|
output_stream.write(chunk)
|
|
digest.update(chunk)
|
|
size_bytes += len(chunk)
|
|
|
|
return {
|
|
"original_filename": StorageService._safe_filename(original_filename),
|
|
"stored_filename": file_path.name,
|
|
"content_type": content_type or "application/octet-stream",
|
|
"size_bytes": size_bytes,
|
|
"checksum_sha256": digest.hexdigest(),
|
|
"storage_path": str(file_path),
|
|
}
|
|
|
|
@staticmethod
|
|
def persist_file(
|
|
storage_path: str,
|
|
content: bytes,
|
|
original_filename: str,
|
|
content_type: str | None,
|
|
) -> dict[str, Any]:
|
|
target = Path(storage_path)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with target.open("wb") as stream:
|
|
stream.write(content)
|
|
|
|
metadata: dict[str, Any] = {
|
|
"original_filename": StorageService._safe_filename(original_filename),
|
|
"stored_filename": target.name,
|
|
"content_type": content_type or "application/octet-stream",
|
|
"size_bytes": len(content),
|
|
"checksum_sha256": StorageService.calculate_checksum_sha256(content),
|
|
"storage_path": str(target),
|
|
}
|
|
return metadata
|
|
|
|
@staticmethod
|
|
def persist_file_from_path(
|
|
storage_path: str,
|
|
source_path: str | Path,
|
|
original_filename: str,
|
|
content_type: str | None,
|
|
) -> dict[str, Any]:
|
|
source = Path(source_path).resolve()
|
|
if not source.is_file():
|
|
raise FileNotFoundError(f"Source artifact does not exist: {source}")
|
|
target = Path(storage_path)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
digest = hashlib.sha256()
|
|
size_bytes = 0
|
|
with source.open("rb") as input_stream, target.open("wb") as output_stream:
|
|
for chunk in iter(lambda: input_stream.read(StorageService.UPLOAD_CHUNK_SIZE), b""):
|
|
output_stream.write(chunk)
|
|
digest.update(chunk)
|
|
size_bytes += len(chunk)
|
|
return {
|
|
"original_filename": StorageService._safe_filename(original_filename),
|
|
"stored_filename": target.name,
|
|
"content_type": content_type or "application/octet-stream",
|
|
"size_bytes": size_bytes,
|
|
"checksum_sha256": digest.hexdigest(),
|
|
"storage_path": str(target),
|
|
}
|
|
|
|
@staticmethod
|
|
def remove_dataset_file(path: str) -> None:
|
|
target = Path(path)
|
|
if target.exists():
|
|
target.unlink(missing_ok=True)
|
|
|
|
dataset_parent = target.parent
|
|
if dataset_parent.exists() and dataset_parent.is_dir():
|
|
has_files = any(dataset_parent.iterdir())
|
|
if not has_files:
|
|
shutil.rmtree(dataset_parent, ignore_errors=True)
|
|
|
|
@staticmethod
|
|
def dataset_export_path(project_id: str, dataset_id: str, filename: str) -> str:
|
|
output_dir = StorageService._base_dir() / "exports" / project_id / "datasets"
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
return str(output_dir / f"{dataset_id}_{StorageService._safe_filename(filename)}")
|