from __future__ import annotations import json import re import tomllib import xml.etree.ElementTree as ET from collections import defaultdict from collections.abc import Iterable from pathlib import Path from .models import DependencySignals, Finding, Severity, TestSignals LOCKFILES = { "package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb", "uv.lock", "poetry.lock", "Pipfile.lock", "Cargo.lock", "go.sum", } MANIFEST_MAX_BYTES = 2_000_000 COVERAGE_NAMES = { "coverage.xml", "lcov.info", "cobertura.xml", ".coverage", "jacoco.xml", } COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES} CI_WORKFLOW_PREFIXES = (".github/workflows/", ".gitea/workflows/") CI_WORKFLOW_NAMES = {".gitlab-ci.yml", "jenkinsfile", "azure-pipelines.yml"} SIGNAL_TEXT_NAMES = {"package.json", "pyproject.toml"} def is_ci_workflow_path(relative: str) -> bool: normalized = relative.replace("\\", "/") return normalized.lower().startswith(CI_WORKFLOW_PREFIXES) or ( Path(normalized).name.casefold() in CI_WORKFLOW_NAMES ) def is_signal_text_path(relative: str) -> bool: """Return True when detect_test_signals needs this file's text content.""" return is_ci_workflow_path(relative) or Path(relative).name.casefold() in SIGNAL_TEXT_NAMES def _python_dependency_name(declaration: str) -> str: return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip() def _python_constraint_kind(declaration: str) -> str: """Classify the useful subset of Python version constraints without penalising bounded ranges.""" value = declaration.split(";", maxsplit=1)[0].strip().lower() if " @ " in value or "==" in value: return "pinned" if "~=" in value or "^" in value: return "bounded" has_lower = ">" in value has_upper = "<" in value if has_lower and has_upper: return "bounded" return "unbounded" def _python_dependency_finding(declaration: str, relative: str) -> Finding | None: if _python_constraint_kind(declaration) != "unbounded": return None return Finding( code="dependency.python-unbounded", category="Dependencies", severity=Severity.WARNING, message=f"Python dependency has no bounded version range: {declaration.split(';')[0]}", path=relative, deduction=3, ) def detect_test_signals( relative_paths: list[str], source_files: int, test_files: int, text_by_path: dict[str, str], readable_paths: Iterable[str] | None = None, ) -> TestSignals: """Derive test/CI signals. ``text_by_path`` only needs the texts accepted by :func:`is_signal_text_path`; ``readable_paths`` lists every successfully read text file (defaults to the ``text_by_path`` keys) so coverage evidence does not require retaining text. """ frameworks: set[str] = set() coverage: list[str] = [] ci: list[str] = [] ci_validation: list[str] = [] ci_build: list[str] = [] ci_deploy_only: list[str] = [] coverage_evidence: list[str] = [] for path in relative_paths: normalized = path.replace("\\", "/") name = Path(normalized).name name_lower = name.casefold() lower = normalized.lower() if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}": coverage.append(normalized) if is_ci_workflow_path(normalized): ci.append(normalized) if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"): frameworks.add("pytest/unittest") if (".test." in name_lower or ".spec." in name_lower) and name_lower.endswith( (".ts", ".tsx", ".js", ".jsx") ): frameworks.add("JavaScript test runner") if name_lower.endswith("_test.go"): frameworks.add("go test") if name_lower.endswith(("tests.cs", "test.cs")): frameworks.add(".NET test") ci_paths = set(ci) for path in text_by_path if readable_paths is None else readable_paths: if Path(path).name.casefold() in COVERAGE_NAMES_CASEFOLD: coverage_evidence.append(path.replace("\\", "/")) for path, text in text_by_path.items(): name = Path(path).name normalized = path.replace("\\", "/") if normalized in ci_paths: lowered_text = text.casefold() validates = any( token in lowered_text for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile") ) builds = any( token in lowered_text for token in ("npm run build", "pnpm build", "docker build", "vite build", "cargo build", "go build", "dotnet build") ) deploys = any(token in lowered_text for token in ("deploy", "release", "publish", "kubectl apply")) if validates: ci_validation.append(normalized) if builds: ci_build.append(normalized) if deploys and not validates: ci_deploy_only.append(normalized) if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")): coverage_evidence.append(normalized) if name.casefold() == "package.json": try: data = json.loads(text) except (TypeError, json.JSONDecodeError): continue if not isinstance(data, dict): continue deps: dict[object, object] = {} for section_name in ( "dependencies", "devDependencies", "peerDependencies", "optionalDependencies", ): section = data.get(section_name, {}) if isinstance(section, dict): deps.update(section) if "vitest" in deps: frameworks.add("Vitest") if "jest" in deps: frameworks.add("Jest") if "@playwright/test" in deps: frameworks.add("Playwright") if "cypress" in deps: frameworks.add("Cypress") elif name.casefold() == "pyproject.toml": lowered = text.lower() if "pytest" in lowered: frameworks.add("pytest") ratio = round((test_files / source_files) * 100, 2) if source_files else None return TestSignals( frameworks=sorted(frameworks), coverage_artifacts=sorted(set(coverage))[:30], ci_workflows=sorted(set(ci))[:30], ci_validation_workflows=sorted(set(ci_validation))[:30], ci_build_workflows=sorted(set(ci_build))[:30], ci_deploy_only_workflows=sorted(set(ci_deploy_only))[:30], coverage_evidence=sorted(set(coverage_evidence))[:30], test_source_ratio=ratio, ) def analyze_dependencies( root: Path, files_by_name: dict[str, list[Path]], ) -> tuple[DependencySignals, list[Finding]]: findings: list[Finding] = [] manifests: list[str] = [] lockfiles: list[str] = [] dependency_origins: dict[str, set[str]] = defaultdict(set) declared = 0 unbounded = 0 unpinned = 0 for lock_name in sorted(LOCKFILES): for path in files_by_name.get(lock_name, []): lockfiles.append(str(path.relative_to(root)).replace("\\", "/")) for package_json in files_by_name.get("package.json", []): relative = str(package_json.relative_to(root)).replace("\\", "/") manifests.append(relative) try: if package_json.stat().st_size > MANIFEST_MAX_BYTES: raise ValueError("package.json is too large to parse") data = json.loads(package_json.read_text(encoding="utf-8")) if not isinstance(data, dict): raise TypeError("package.json root must be an object") except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError): findings.append( Finding( code="dependency.invalid-package-json", category="Dependencies", severity=Severity.WARNING, message="package.json could not be parsed safely", path=relative, deduction=8, ) ) continue deps: dict[str, object] = {} invalid_sections = False for section_name in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"): section = data.get(section_name, {}) if section is None: continue if not isinstance(section, dict): invalid_sections = True continue deps.update(section) if invalid_sections: findings.append( Finding( code="dependency.invalid-package-json", category="Dependencies", severity=Severity.WARNING, message="One or more package dependency sections are not objects", path=relative, deduction=8, ) ) declared += len(deps) for name, version in deps.items(): dependency_name = str(name) dependency_origins[dependency_name.casefold()].add(relative) raw = str(version).strip().lower() if raw in {"*", "latest", "x"} and not raw.startswith("workspace:"): unbounded += 1 findings.append( Finding( code="dependency.unbounded", category="Dependencies", severity=Severity.WARNING, message=f"Dependency {dependency_name} uses an unbounded version", path=relative, deduction=5, ) ) parent = package_json.parent package_roots = [parent, *[ancestor for ancestor in parent.parents if ancestor == root or root in ancestor.parents]] if not any( (package_root / lock).exists() for package_root in package_roots for lock in ("package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb") ): findings.append(Finding(code="dependency.no-lockfile", category="Dependencies", severity=Severity.WARNING, message="JavaScript package has no recognized lockfile", path=relative, deduction=8)) requirement_paths = [ path for name, values in files_by_name.items() if name.casefold().startswith("requirements") and name.casefold().endswith(".txt") for path in values ] included_requirements: list[str] = [] processed_requirements: set[Path] = set() pending_requirements = list(requirement_paths) while pending_requirements: requirements = pending_requirements.pop(0) try: requirements = requirements.resolve(strict=True) except OSError: continue if requirements in processed_requirements or requirements != root and root not in requirements.parents: continue processed_requirements.add(requirements) relative = str(requirements.relative_to(root)).replace("\\", "/") manifests.append(relative) try: lines = requirements.read_text(encoding="utf-8").splitlines() except (OSError, UnicodeDecodeError): findings.append( Finding( code="dependency.unreadable-requirements", category="Dependencies", severity=Severity.WARNING, message="Python requirements file could not be read", path=relative, deduction=8, ) ) continue for line in lines: stripped = line.strip() if not stripped or stripped.startswith("#"): continue if stripped.startswith(("-r ", "--requirement ")): include = stripped.split(maxsplit=1)[1].strip() candidate = (requirements.parent / include).resolve() if candidate == root or root in candidate.parents: pending_requirements.append(candidate) included_requirements.append(str(candidate.relative_to(root)).replace("\\", "/")) continue if stripped.startswith("-"): continue declared += 1 name = _python_dependency_name(stripped) if name: dependency_origins[name.casefold()].add(relative) finding = _python_dependency_finding(stripped, relative) if finding: unbounded += 1 findings.append(finding) for pyproject in files_by_name.get("pyproject.toml", []): relative = str(pyproject.relative_to(root)).replace("\\", "/") manifests.append(relative) try: if pyproject.stat().st_size > MANIFEST_MAX_BYTES: raise ValueError("pyproject.toml is too large to parse") data = tomllib.loads(pyproject.read_text(encoding="utf-8")) if not isinstance(data, dict): raise TypeError("pyproject.toml root must be a table") except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError, TypeError): findings.append( Finding( code="dependency.invalid-pyproject", category="Dependencies", severity=Severity.WARNING, message="pyproject.toml could not be parsed safely", path=relative, deduction=8, ) ) continue project = data.get("project", {}) if not isinstance(project, dict): findings.append( Finding( code="dependency.invalid-pyproject", category="Dependencies", severity=Severity.WARNING, message="pyproject.toml project section is not a table", path=relative, deduction=8, ) ) project = {} project_deps = project.get("dependencies", []) if isinstance(project_deps, list): declared += len(project_deps) for dependency in project_deps: declaration = str(dependency) name = _python_dependency_name(declaration) if name: dependency_origins[name.casefold()].add(relative) finding = _python_dependency_finding(declaration, relative) if finding: unbounded += 1 findings.append(finding) elif project_deps is not None: findings.append( Finding( code="dependency.invalid-pyproject", category="Dependencies", severity=Severity.WARNING, message="pyproject.toml project.dependencies is not a list", path=relative, deduction=8, ) ) optional_deps = project.get("optional-dependencies", {}) if isinstance(optional_deps, dict): for dependencies in optional_deps.values(): if not isinstance(dependencies, list): continue declared += len(dependencies) for dependency in dependencies: declaration = str(dependency) name = _python_dependency_name(declaration) if name: dependency_origins[name.casefold()].add(relative) finding = _python_dependency_finding(declaration, relative) if finding: unbounded += 1 findings.append(finding) elif optional_deps is not None: findings.append( Finding( code="dependency.invalid-pyproject", category="Dependencies", severity=Severity.WARNING, message="pyproject.toml project.optional-dependencies is not a table", path=relative, deduction=8, ) ) for go_mod in files_by_name.get("go.mod", []): relative = str(go_mod.relative_to(root)).replace("\\", "/") manifests.append(relative) try: text = go_mod.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): findings.append( Finding( code="dependency.unreadable-go-mod", category="Dependencies", severity=Severity.WARNING, message="go.mod could not be read", path=relative, deduction=8, ) ) continue matches = list(re.finditer(r"(?m)^\s*([\w./-]+)\s+v\d[^\s]*", text)) if "require" in text and not matches: findings.append( Finding( code="dependency.invalid-go-mod", category="Dependencies", severity=Severity.WARNING, message="go.mod contains no parseable versioned dependencies", path=relative, deduction=8, ) ) for match in matches: declared += 1 dependency_origins[match.group(1).casefold()].add(relative) for cargo in files_by_name.get("Cargo.toml", []): relative = str(cargo.relative_to(root)).replace("\\", "/") manifests.append(relative) try: data = tomllib.loads(cargo.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError): findings.append( Finding( code="dependency.invalid-cargo-toml", category="Dependencies", severity=Severity.WARNING, message="Cargo.toml could not be parsed safely", path=relative, deduction=8, ) ) continue cargo_deps = data.get("dependencies", {}) if isinstance(cargo_deps, dict): declared += len(cargo_deps) for name in cargo_deps: dependency_origins[str(name).casefold()].add(relative) for pom in files_by_name.get("pom.xml", []): relative = str(pom.relative_to(root)).replace("\\", "/") manifests.append(relative) try: xml_root = ET.parse(pom).getroot() except (OSError, UnicodeError, ET.ParseError): findings.append( Finding( code="dependency.invalid-pom", category="Dependencies", severity=Severity.WARNING, message="pom.xml could not be parsed safely", path=relative, deduction=8, ) ) continue dependencies = [element for element in xml_root.iter() if element.tag.split("}")[-1] == "dependency"] declared += len(dependencies) for dependency in dependencies: artifact = next((child.text for child in dependency if child.tag.split("}")[-1] == "artifactId"), None) if artifact: dependency_origins[artifact.casefold()].add(relative) duplicates = sorted(name for name, origins in dependency_origins.items() if len(origins) > 1) for name in duplicates[:25]: origins = sorted(dependency_origins[name]) findings.append(Finding(code="dependency.duplicate-manifest", category="Dependencies", severity=Severity.INFO, message=f"Dependency {name} is declared across multiple manifests", path=origins[0], deduction=1)) return ( DependencySignals( manifests=sorted(set(manifests)), lockfiles=sorted(set(lockfiles)), declared_dependencies=declared, unbounded_dependencies=unbounded, unpinned_dependencies=unpinned, duplicate_dependencies=duplicates[:50], included_requirements=sorted(set(included_requirements))[:50], ), findings, )