510 lines
18 KiB
Python
510 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
"""Read-only, host-only parser for bounded PS5 SIECAF structural metadata.
|
|
|
|
This module never decrypts, extracts content, invokes ps5-bar-tool, opens a
|
|
network connection, or writes an output file. The byte layout is based on:
|
|
|
|
* https://www.psdevwiki.com/ps5/Archive.dat
|
|
* c0w-ar/ps5-bar-tool include/bar_file.h at
|
|
36d014672bc87577a6e0d750c2cccadc3fae0854
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import struct
|
|
from typing import Any, BinaryIO, Iterator
|
|
import zipfile
|
|
|
|
|
|
MAGIC = b"SIECAF\x00\x00"
|
|
HEADER = struct.Struct("<8sQiIiI16s12sIQQQ")
|
|
SEGMENT_META = struct.Struct("<iHHQQQQ12sIQ")
|
|
SECTION_HASH = struct.Struct("<ii16s24s")
|
|
ALIGNMENT = 0x10000
|
|
UINT64_MAX = (1 << 64) - 1
|
|
SUPPORTED_VERSIONS = frozenset({3, 6})
|
|
|
|
|
|
class SiecafError(ValueError):
|
|
"""A fail-closed SIECAF structural error."""
|
|
|
|
|
|
def _sha256(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def _checked_add(left: int, right: int, label: str) -> int:
|
|
result = left + right
|
|
if left < 0 or right < 0 or result > UINT64_MAX:
|
|
raise SiecafError(f"{label}: uint64 addition overflow")
|
|
return result
|
|
|
|
|
|
def _checked_mul(left: int, right: int, label: str) -> int:
|
|
result = left * right
|
|
if left < 0 or right < 0 or result > UINT64_MAX:
|
|
raise SiecafError(f"{label}: uint64 multiplication overflow")
|
|
return result
|
|
|
|
|
|
def _read_exact(stream: BinaryIO, size: int, label: str) -> bytes:
|
|
value = stream.read(size)
|
|
if len(value) != size:
|
|
raise SiecafError(f"{label}: truncated input")
|
|
return value
|
|
|
|
|
|
def _canonical_hash(records: list[dict[str, Any]]) -> str:
|
|
encoded = json.dumps(
|
|
records, sort_keys=True, separators=(",", ":"), ensure_ascii=True
|
|
).encode("ascii")
|
|
return _sha256(encoded)
|
|
|
|
|
|
def inspect_siecaf(
|
|
stream: BinaryIO, source_size: int, *, source_label: str = "<stream>"
|
|
) -> dict[str, Any]:
|
|
"""Parse structural metadata from an already-open, read-only stream."""
|
|
if source_size < HEADER.size:
|
|
return {
|
|
"source": source_label,
|
|
"source_size": source_size,
|
|
"classification": "SIECAF_MALFORMED",
|
|
"errors": ["header: truncated input"],
|
|
"warnings": [],
|
|
}
|
|
stream.seek(0)
|
|
header_raw = _read_exact(stream, HEADER.size, "header")
|
|
(
|
|
magic,
|
|
unknown_u64,
|
|
mode,
|
|
pad1,
|
|
version,
|
|
pad2,
|
|
key,
|
|
iv12,
|
|
pad3,
|
|
segment_count,
|
|
file_offset,
|
|
file_size,
|
|
) = HEADER.unpack(header_raw)
|
|
header = {
|
|
"magic_hex": magic.hex(),
|
|
"magic_ascii": magic.rstrip(b"\x00").decode("ascii", errors="replace"),
|
|
"unknown_u64": unknown_u64,
|
|
"mode_i32": mode,
|
|
"padding_1_u32": pad1,
|
|
"version_i32": version,
|
|
"padding_2_u32": pad2,
|
|
"key_or_unknown_16_redacted": True,
|
|
"key_or_unknown_16_sha256": _sha256(key),
|
|
"iv_12_hex": iv12.hex(),
|
|
"padding_3_u32": pad3,
|
|
"segment_count": segment_count,
|
|
"file_offset": file_offset,
|
|
"file_size": file_size,
|
|
"raw_sha256": _sha256(header_raw),
|
|
}
|
|
if magic != MAGIC:
|
|
return {
|
|
"source": source_label,
|
|
"source_size": source_size,
|
|
"classification": "SIECAF_MALFORMED",
|
|
"header": header,
|
|
"errors": ["magic: expected SIECAF\\0\\0"],
|
|
"warnings": [],
|
|
}
|
|
if version not in SUPPORTED_VERSIONS:
|
|
return {
|
|
"source": source_label,
|
|
"source_size": source_size,
|
|
"classification": "SIECAF_UNSUPPORTED_VERSION",
|
|
"header": header,
|
|
"errors": [f"unsupported SIECAF header version: {version}"],
|
|
"warnings": [],
|
|
}
|
|
errors: list[str] = []
|
|
warnings: list[str] = []
|
|
if segment_count == 0:
|
|
errors.append("segment_count: zero")
|
|
try:
|
|
metadata_bytes = _checked_mul(
|
|
segment_count, SEGMENT_META.size, "segment metadata table"
|
|
)
|
|
hash_bytes = _checked_mul(
|
|
segment_count, SECTION_HASH.size, "section hash table"
|
|
)
|
|
metadata_end = _checked_add(HEADER.size, metadata_bytes, "metadata end")
|
|
tables_end = _checked_add(metadata_end, hash_bytes, "tables end")
|
|
declared_end = _checked_add(file_offset, file_size, "declared file end")
|
|
except SiecafError as error:
|
|
return {
|
|
"source": source_label,
|
|
"source_size": source_size,
|
|
"classification": "SIECAF_MALFORMED",
|
|
"header": header,
|
|
"errors": [str(error)],
|
|
"warnings": [],
|
|
}
|
|
if tables_end > source_size:
|
|
errors.append("tables: outside source file")
|
|
if tables_end > file_offset:
|
|
errors.append("tables: overlap declared data region")
|
|
if file_offset % ALIGNMENT != 0:
|
|
errors.append(f"file_offset: not aligned to {ALIGNMENT}")
|
|
if declared_end > source_size:
|
|
errors.append("declared file range: outside source file")
|
|
elif declared_end < source_size:
|
|
warnings.append(
|
|
f"trailing data after declared file range: {source_size - declared_end}"
|
|
)
|
|
if errors:
|
|
return {
|
|
"source": source_label,
|
|
"source_size": source_size,
|
|
"classification": "SIECAF_MALFORMED",
|
|
"header": header,
|
|
"tables_end": tables_end,
|
|
"errors": errors,
|
|
"warnings": warnings,
|
|
}
|
|
|
|
metadata: list[dict[str, Any]] = []
|
|
for index in range(segment_count):
|
|
raw = _read_exact(stream, SEGMENT_META.size, f"segment metadata {index}")
|
|
(
|
|
section_id,
|
|
padding_1,
|
|
part_number,
|
|
data_offset,
|
|
aligned_length,
|
|
hash_key_id,
|
|
encryption_key_id,
|
|
segment_iv12,
|
|
segment_iv_padding,
|
|
unaligned_length,
|
|
) = SEGMENT_META.unpack(raw)
|
|
iv16 = segment_iv12 + struct.pack("<I", segment_iv_padding)
|
|
metadata.append(
|
|
{
|
|
"table_index": index,
|
|
"section_id": section_id,
|
|
"padding_1_u16": padding_1,
|
|
"part_number": part_number,
|
|
"data_offset": data_offset,
|
|
"aligned_length": aligned_length,
|
|
"unaligned_length": unaligned_length,
|
|
"hash_key_id_or_algorithm_type": hash_key_id,
|
|
"encryption_key_id_or_algorithm_version": encryption_key_id,
|
|
"iv_hex": iv16.hex(),
|
|
"raw_sha256": _sha256(raw),
|
|
}
|
|
)
|
|
hashes: list[dict[str, Any]] = []
|
|
for index in range(segment_count):
|
|
raw = _read_exact(stream, SECTION_HASH.size, f"section hash {index}")
|
|
section_id, section_type, section_hash, padding = SECTION_HASH.unpack(raw)
|
|
hashes.append(
|
|
{
|
|
"table_index": index,
|
|
"section_id": section_id,
|
|
"section_type": section_type,
|
|
"section_hash_128_hex": section_hash.hex(),
|
|
"padding_hex": padding.hex(),
|
|
"raw_sha256": _sha256(raw),
|
|
}
|
|
)
|
|
|
|
metadata_keys = [(item["section_id"], item["part_number"]) for item in metadata]
|
|
metadata_ids = [item["section_id"] for item in metadata]
|
|
hash_ids = [item["section_id"] for item in hashes]
|
|
duplicate_metadata_ids = sorted(
|
|
{
|
|
section_id
|
|
for section_id, part_number in metadata_keys
|
|
if metadata_keys.count((section_id, part_number)) > 1
|
|
}
|
|
)
|
|
duplicate_metadata_keys = sorted(
|
|
{
|
|
(section_id, part_number)
|
|
for section_id, part_number in metadata_keys
|
|
if metadata_keys.count((section_id, part_number)) > 1
|
|
}
|
|
)
|
|
repeated_metadata_ids = sorted(
|
|
{item for item in metadata_ids if metadata_ids.count(item) > 1}
|
|
)
|
|
duplicate_hash_ids = sorted({item for item in hash_ids if hash_ids.count(item) > 1})
|
|
if duplicate_metadata_ids:
|
|
errors.append(
|
|
f"duplicate metadata section ID/part keys: {duplicate_metadata_keys}"
|
|
)
|
|
if duplicate_hash_ids:
|
|
errors.append(f"duplicate hash section IDs: {duplicate_hash_ids}")
|
|
if set(hash_ids) != set(range(segment_count)):
|
|
errors.append("hash section IDs do not cover metadata table indexes")
|
|
|
|
hash_by_id = {
|
|
item["section_id"]: item
|
|
for item in hashes
|
|
if item["section_id"] not in duplicate_hash_ids
|
|
}
|
|
ranges: list[tuple[int, int, int]] = []
|
|
for item in metadata:
|
|
section_id = item["section_id"]
|
|
start = item["data_offset"]
|
|
length = item["aligned_length"]
|
|
unaligned = item["unaligned_length"]
|
|
try:
|
|
end = _checked_add(start, length, f"section {section_id} end")
|
|
except SiecafError as error:
|
|
errors.append(str(error))
|
|
continue
|
|
if section_id < 0:
|
|
errors.append(f"section {section_id}: negative ID")
|
|
if start % ALIGNMENT != 0:
|
|
errors.append(f"section {section_id}: unaligned data offset")
|
|
if length == 0 and unaligned != 0:
|
|
errors.append(f"section {section_id}: incoherent unaligned length")
|
|
if length > 0 and length % ALIGNMENT != 0:
|
|
errors.append(f"section {section_id}: incoherent aligned length")
|
|
if length > 0 and (unaligned > length or length - unaligned >= ALIGNMENT):
|
|
errors.append(f"section {section_id}: incoherent unaligned length")
|
|
if start < file_offset or end > declared_end or end > source_size:
|
|
errors.append(f"section {section_id}: range outside declared file data")
|
|
if length > 0:
|
|
ranges.append((start, end, section_id))
|
|
hash_record = hash_by_id.get(item["table_index"])
|
|
item["section_type"] = (
|
|
hash_record["section_type"] if hash_record is not None else None
|
|
)
|
|
item["section_hash_128_hex"] = (
|
|
hash_record["section_hash_128_hex"] if hash_record is not None else None
|
|
)
|
|
|
|
ranges.sort()
|
|
overlaps: list[dict[str, int]] = []
|
|
gaps: list[dict[str, int]] = []
|
|
previous_end = file_offset
|
|
previous_id = -1
|
|
for start, end, section_id in ranges:
|
|
if start < previous_end:
|
|
overlaps.append(
|
|
{
|
|
"left_section_id": previous_id,
|
|
"right_section_id": section_id,
|
|
"overlap_bytes": previous_end - start,
|
|
}
|
|
)
|
|
elif start > previous_end:
|
|
gaps.append(
|
|
{
|
|
"after_section_id": previous_id,
|
|
"before_section_id": section_id,
|
|
"gap_bytes": start - previous_end,
|
|
}
|
|
)
|
|
if end > previous_end:
|
|
previous_end = end
|
|
previous_id = section_id
|
|
if overlaps:
|
|
errors.append(f"overlapping section ranges: {len(overlaps)}")
|
|
if gaps:
|
|
warnings.append(f"gaps between section ranges: {len(gaps)}")
|
|
trailing_data = max(0, source_size - previous_end)
|
|
if trailing_data:
|
|
warnings.append(f"trailing data after final section: {trailing_data}")
|
|
if previous_end < declared_end:
|
|
warnings.append(
|
|
f"uncovered bytes inside declared data range: {declared_end - previous_end}"
|
|
)
|
|
|
|
normalized_segments = [
|
|
{
|
|
key: item[key]
|
|
for key in (
|
|
"section_id",
|
|
"part_number",
|
|
"data_offset",
|
|
"aligned_length",
|
|
"unaligned_length",
|
|
"hash_key_id_or_algorithm_type",
|
|
"encryption_key_id_or_algorithm_version",
|
|
"iv_hex",
|
|
)
|
|
}
|
|
for item in sorted(
|
|
metadata,
|
|
key=lambda value: (
|
|
value["section_id"],
|
|
value["part_number"],
|
|
value["data_offset"],
|
|
),
|
|
)
|
|
]
|
|
normalized_hashes = [
|
|
{
|
|
key: item[key]
|
|
for key in ("section_id", "section_type", "section_hash_128_hex")
|
|
}
|
|
for item in sorted(
|
|
hashes, key=lambda value: (value["section_id"], value["section_type"])
|
|
)
|
|
]
|
|
normalized_layout = [
|
|
{
|
|
key: item[key]
|
|
for key in (
|
|
"section_id",
|
|
"part_number",
|
|
"data_offset",
|
|
"aligned_length",
|
|
"unaligned_length",
|
|
"hash_key_id_or_algorithm_type",
|
|
"encryption_key_id_or_algorithm_version",
|
|
)
|
|
}
|
|
for item in normalized_segments
|
|
]
|
|
normalized_segment_table_sha256 = _canonical_hash(normalized_segments)
|
|
normalized_hash_blocks_sha256 = _canonical_hash(normalized_hashes)
|
|
normalized_layout_sha256 = _canonical_hash(
|
|
[
|
|
{
|
|
"unknown_u64": unknown_u64,
|
|
"version_i32": version,
|
|
"segment_count": segment_count,
|
|
"file_offset": file_offset,
|
|
"file_size": file_size,
|
|
},
|
|
*normalized_layout,
|
|
]
|
|
)
|
|
structural_fingerprint_sha256 = _canonical_hash(
|
|
[
|
|
{
|
|
"header_raw_sha256": header["raw_sha256"],
|
|
"normalized_segment_table_sha256": (normalized_segment_table_sha256),
|
|
"normalized_hash_blocks_sha256": normalized_hash_blocks_sha256,
|
|
}
|
|
]
|
|
)
|
|
return {
|
|
"source": source_label,
|
|
"source_size": source_size,
|
|
"classification": ("SIECAF_MALFORMED" if errors else "SIECAF_VALID_STRUCTURE"),
|
|
"header": header,
|
|
"tables_end": tables_end,
|
|
"table_padding_bytes": file_offset - tables_end,
|
|
"segments": sorted(metadata, key=lambda item: item["table_index"]),
|
|
"section_hashes": sorted(hashes, key=lambda item: item["table_index"]),
|
|
"normalized_segment_table_sha256": normalized_segment_table_sha256,
|
|
"normalized_hash_blocks_sha256": normalized_hash_blocks_sha256,
|
|
"normalized_layout_sha256": normalized_layout_sha256,
|
|
"structural_fingerprint_sha256": structural_fingerprint_sha256,
|
|
"duplicate_metadata_section_ids": duplicate_metadata_ids,
|
|
"duplicate_metadata_section_keys": [
|
|
{"section_id": section_id, "part_number": part_number}
|
|
for section_id, part_number in duplicate_metadata_keys
|
|
],
|
|
"repeated_metadata_section_ids": repeated_metadata_ids,
|
|
"duplicate_hash_section_ids": duplicate_hash_ids,
|
|
"overlaps": overlaps,
|
|
"gaps": gaps,
|
|
"trailing_data_bytes": trailing_data,
|
|
"errors": errors,
|
|
"warnings": warnings,
|
|
}
|
|
|
|
|
|
def compare_structures(left: dict[str, Any], right: dict[str, Any]) -> dict[str, Any]:
|
|
"""Classify two already-parsed structural fingerprints."""
|
|
if (
|
|
left.get("classification") == "SIECAF_UNSUPPORTED_VERSION"
|
|
or right.get("classification") == "SIECAF_UNSUPPORTED_VERSION"
|
|
):
|
|
classification = "SIECAF_UNSUPPORTED_VERSION"
|
|
elif (
|
|
left.get("classification") != "SIECAF_VALID_STRUCTURE"
|
|
or right.get("classification") != "SIECAF_VALID_STRUCTURE"
|
|
):
|
|
classification = "SIECAF_MALFORMED"
|
|
elif left.get("structural_fingerprint_sha256") == right.get(
|
|
"structural_fingerprint_sha256"
|
|
):
|
|
classification = "SIECAF_STRUCTURAL_EXACT"
|
|
elif left.get("normalized_layout_sha256") == right.get("normalized_layout_sha256"):
|
|
classification = "SIECAF_LAYOUT_MATCH_HASHES_DIFFER"
|
|
else:
|
|
classification = "SIECAF_LAYOUT_DIFFERENT"
|
|
return {
|
|
"classification": classification,
|
|
"left_structural_fingerprint_sha256": left.get("structural_fingerprint_sha256"),
|
|
"right_structural_fingerprint_sha256": right.get(
|
|
"structural_fingerprint_sha256"
|
|
),
|
|
"left_layout_sha256": left.get("normalized_layout_sha256"),
|
|
"right_layout_sha256": right.get("normalized_layout_sha256"),
|
|
"left_hash_blocks_sha256": left.get("normalized_hash_blocks_sha256"),
|
|
"right_hash_blocks_sha256": right.get("normalized_hash_blocks_sha256"),
|
|
}
|
|
|
|
|
|
@contextmanager
|
|
def open_source(
|
|
path: Path, zip_entry: str | None
|
|
) -> Iterator[tuple[BinaryIO, int, str]]:
|
|
if zip_entry is None:
|
|
with path.open("rb") as stream:
|
|
yield stream, path.stat().st_size, str(path)
|
|
return
|
|
with zipfile.ZipFile(path) as archive:
|
|
info = archive.getinfo(zip_entry)
|
|
with archive.open(info) as stream:
|
|
yield stream, info.file_size, f"{path}::{zip_entry}"
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("path", type=Path)
|
|
parser.add_argument("--zip-entry")
|
|
parser.add_argument("--label")
|
|
parser.add_argument("--pretty", action="store_true")
|
|
args = parser.parse_args()
|
|
try:
|
|
with open_source(args.path.resolve(), args.zip_entry) as (
|
|
stream,
|
|
size,
|
|
default_label,
|
|
):
|
|
result = inspect_siecaf(
|
|
stream, size, source_label=args.label or default_label
|
|
)
|
|
except (OSError, KeyError, zipfile.BadZipFile, SiecafError) as error:
|
|
result = {
|
|
"source": args.label or str(args.path),
|
|
"classification": "SIECAF_MALFORMED",
|
|
"errors": [str(error)],
|
|
"warnings": [],
|
|
}
|
|
print(
|
|
json.dumps(
|
|
result,
|
|
sort_keys=True,
|
|
indent=2 if args.pretty else None,
|
|
)
|
|
)
|
|
return 0 if result["classification"] == "SIECAF_VALID_STRUCTURE" else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|