diff --git a/.github/scripts/minimum_requirements.py b/.github/scripts/minimum_requirements.py new file mode 100644 index 000000000..4801e2937 --- /dev/null +++ b/.github/scripts/minimum_requirements.py @@ -0,0 +1,81 @@ +"""Print pins for the minimum version of each runtime dependency. + +Reads [project].dependencies from pyproject.toml and prints one `name==minimum` +line per dependency that applies to the given Python version. CI installs the +package with these as constraints, so the test suite runs against the oldest +versions that pyproject.toml admits. Optional extras are not pinned. + +Every applicable dependency must declare a minimum (>=, ==, or ~=). A +dependency without one is an error: GitHub's dependency graph omits it, and +its oldest release is untested. + +Usage: python minimum_requirements.py [pyproject.toml] [--python 3.10] +""" + +import argparse +import re +import sys + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name +from packaging.version import Version + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib + + +def python_version(value: str) -> str: + if not re.fullmatch(r"3\.\d+", value): + raise argparse.ArgumentTypeError(f"expected a version like 3.10, got {value!r}") + return value + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("pyproject", nargs="?", default="pyproject.toml") + parser.add_argument("--python", type=python_version, default="3.10", help="Python version to evaluate markers for") + args = parser.parse_args() + + with open(args.pyproject, "rb") as f: + dependencies = tomllib.load(f)["project"]["dependencies"] + + # A dependency may span several lines, e.g. a minimum plus an upper bound + # under a marker. Combine the applicable lines per package. + environment = {"python_version": args.python, "python_full_version": f"{args.python}.0"} + specifiers: dict[str, SpecifierSet] = {} + for line in dependencies: + req = Requirement(line) + if req.marker is not None and not req.marker.evaluate(environment): + continue + name = canonicalize_name(req.name) + specifiers[name] = specifiers.get(name, SpecifierSet()) & req.specifier + + pins, errors = {}, [] + for name, specifier in sorted(specifiers.items()): + bounds = [s.version for s in specifier if s.operator in (">=", "==", "~=")] + if not bounds: + errors.append(f"{name} declares no minimum version") + continue + if any("*" in bound for bound in bounds): + errors.append(f"{name}: wildcard versions are not supported ({specifier})") + continue + minimum = max(bounds, key=Version) + if minimum not in specifier: + errors.append(f"{name}: minimum {minimum} is excluded by {specifier}") + continue + pins[name] = minimum + + if errors: + for error in errors: + print(f"error: {error}", file=sys.stderr) + return 1 + for name, version in pins.items(): + print(f"{name}=={version}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a3f2bc5c8..f5404b036 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,6 +11,7 @@ on: - "tests/**" - "pyproject.toml" - "pixi.lock" + - ".github/scripts/minimum_requirements.py" - ".github/workflows/test.yaml" pull_request: branches: @@ -22,6 +23,7 @@ on: - "tests/**" - "pyproject.toml" - "pixi.lock" + - ".github/scripts/minimum_requirements.py" - ".github/workflows/test.yaml" jobs: @@ -104,3 +106,39 @@ jobs: - name: Run unit tests run: pytest tests/unit -v + + # Minimum versions: run the full suite on the oldest supported Python with + # every runtime dependency ([project].dependencies; extras are not pinned) at + # the minimum pyproject.toml declares, so those minimums stay true. + # minimum_requirements.py also fails the job when a runtime dependency + # declares no minimum. Uses pip, not pixi, because pixi resolves the newest + # versions from the lock file. + test-min-versions: + runs-on: ubuntu-latest + timeout-minutes: 45 + name: test (minimum dependency versions, py3.10) + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 # hatch-vcs derives the version from git history/tags + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.10" + + - name: Install graphviz + run: sudo apt-get update && sudo apt-get install -y graphviz + + - name: Pin runtime dependencies at their minimums + run: | + python -m pip install tomli==2.5.0 packaging==26.3 + python .github/scripts/minimum_requirements.py --python 3.10 > "$RUNNER_TEMP/minimum.txt" + cat "$RUNNER_TEMP/minimum.txt" + + - name: Install package with test extras + run: python -m pip install -e ".[test]" -c "$RUNNER_TEMP/minimum.txt" + + - name: Run tests + run: python -m pytest tests/ diff --git a/pixi.lock b/pixi.lock index e287ed2ae..51ea3feda 100644 --- a/pixi.lock +++ b/pixi.lock @@ -4495,21 +4495,21 @@ packages: requires_python: '>=3.8' - pypi: ./ name: datajoint - version: 2.3.5.dev28+ge3d2a553e.d20261024 - sha256: 9ac2b52fcb44ab2d1c54ec75a4c390fc2a8d53b0480a2490d897c2dc22f593a3 + version: 2.3.5.dev45+gcc4ca9e3d.d20261106 + sha256: 204220782b38892346a27748aa110b40929b1a0075f07b5b72fabd073c84a924 requires_dist: - - deepdiff - deepdiff<9 ; python_full_version >= '3.15' + - deepdiff>=6.6.0 - fsspec>=2023.1.0 - - networkx - - numpy - - packaging - - pandas + - networkx>=3.0 + - numpy>=1.22.4 + - packaging>=22.0 + - pandas>=2.2.2 - pydantic-settings>=2.0.0 - - pydot - - pymysql>=0.7.2 - - pyparsing - - tqdm + - pydot>=3.0.0 + - pymysql>=1.1.1 + - pyparsing>=3.1.1 + - tqdm>=4.66.3 - pyarrow>=14.0.0 ; extra == 'arrow' - adlfs>=2023.1.0 ; extra == 'azure' - codespell ; extra == 'dev' diff --git a/pyproject.toml b/pyproject.toml index 47c1019e7..75789801c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,20 +6,31 @@ build-backend = "hatchling.build" name = "datajoint" dynamic = ["version"] dependencies = [ - "numpy", - "pymysql>=0.7.2", - "deepdiff", + "numpy>=1.22.4", + # pymysql 1.0 adds the ssl_disabled connection option; 1.1.1 fixes + # CVE-2024-36039. + "pymysql>=1.1.1", + # deepdiff 6.4 adds DeepHash(ignore_iterable_order=...); 6.6 supports numpy 2. + # CVE-2025-58367 and CVE-2026-33155 (fixed in 8.6.2) affect Delta and + # pickle_load; datajoint uses only DeepHash. + "deepdiff>=6.6.0", # deepdiff 9 requires cachebox<6, which ships no Python 3.15 wheels and cannot # be built for 3.15 (PyO3 0.27). Drop once deepdiff accepts cachebox 6. "deepdiff<9; python_version >= '3.15'", - "pyparsing", - "pandas", - "tqdm", - "networkx", - "pydot", + # pyparsing 3.1.0 drops the quotes from quoted defaults (= ""), which + # declares the attribute without a default. + "pyparsing>=3.1.1", + # pandas 2.2.2 is the first release built against numpy 2; older pandas + # fails on import when numpy 2 is installed. + "pandas>=2.2.2", + # CVE-2024-34062 + "tqdm>=4.66.3", + "networkx>=3.0", + # pydot 2.x emits DOT that graphviz rejects for datajoint diagrams. + "pydot>=3.0.0", "fsspec>=2023.1.0", "pydantic-settings>=2.0.0", - "packaging", + "packaging>=22.0", ] requires-python = ">=3.10,<3.16"