From 834995b3227fca058a5f62d46d354b4a2b555658 Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:03:34 +0100 Subject: [PATCH 1/2] feat: align deprecated bump-upstream with the org bump engine --- README.md | 30 +- doc/package-spec.md | 24 +- package.json | 5 +- src/bump/discovery.ts | 110 +++++++ src/bump/dockerfile.ts | 89 ++++++ src/bump/edits.ts | 177 +++++++++++ src/bump/metadata.ts | 72 +++++ src/bump/plan.ts | 110 +++++++ src/bump/registry.ts | 188 +++++++++++ src/bump/resolve.ts | 145 +++++++++ src/bump/versions.ts | 36 +++ .../githubActions/bumpUpstream/deprecation.ts | 11 + .../github/fetchGithubUpstreamVersion.ts | 26 +- .../bumpUpstream/github/getBumpPrBody.ts | 7 +- .../bumpUpstream/github/isValidRelease.ts | 32 +- .../bumpUpstream/github/pickLatestRelease.ts | 40 ++- .../githubActions/bumpUpstream/index.ts | 192 +++--------- .../settings/getInitialSettings.ts | 98 ++---- .../githubActions/bumpUpstream/types.ts | 6 +- src/providers/github/Github.ts | 27 +- src/spec/checkPackageSpec.ts | 55 +++- src/spec/rules.ts | 6 +- test/bump/compareEngines.ts | 155 ++++++++++ test/bump/deprecation.test.ts | 36 +++ test/bump/discovery.test.ts | 134 ++++++++ test/bump/edits.test.ts | 292 ++++++++++++++++++ test/bump/registry.test.ts | 198 ++++++++++++ test/bump/resolve.test.ts | 208 +++++++++++++ test/bump/versions.test.ts | 143 +++++++++ test/commands/gaBumpUpstream/format.test.ts | 10 +- .../gaBumpUpstream/pickLatestRelease.test.ts | 56 +++- test/spec/checkPackageSpec.test.ts | 40 +++ yarn.lock | 5 + 33 files changed, 2469 insertions(+), 294 deletions(-) create mode 100644 src/bump/discovery.ts create mode 100644 src/bump/dockerfile.ts create mode 100644 src/bump/edits.ts create mode 100644 src/bump/metadata.ts create mode 100644 src/bump/plan.ts create mode 100644 src/bump/registry.ts create mode 100644 src/bump/resolve.ts create mode 100644 src/bump/versions.ts create mode 100644 src/commands/githubActions/bumpUpstream/deprecation.ts create mode 100644 test/bump/compareEngines.ts create mode 100644 test/bump/deprecation.test.ts create mode 100644 test/bump/discovery.test.ts create mode 100644 test/bump/edits.test.ts create mode 100644 test/bump/registry.test.ts create mode 100644 test/bump/resolve.test.ts create mode 100644 test/bump/versions.test.ts diff --git a/README.md b/README.md index d9d72cdb..05cbae41 100644 --- a/README.md +++ b/README.md @@ -76,38 +76,26 @@ npx @dappnode/dappnodesdk ## Github Actions -### `bump-upstream` action +### `bump-upstream` action (deprecated) -This Github Action automatically creates pull requests bumping the version to the latest released upstream version on DAppNode Packages. It then tests building the new bumped upstream version and automatically posts an IPFS hash if the build was successful so that the new version can be tested further with other Github Actions and by human testers. +`dappnodesdk github-action bump-upstream` is deprecated and will be removed in a future release. DAppNode org packages should call the reusable workflow `dappnode/workflows/.github/workflows/bump-upstream.yml@master`. Other packages should use the `dappnode/workflows/actions/bump-upstream@master` action, which edits the files, followed by their own PR step. -_Note: This action also requires a Pinata account with an API Access Token and a Github Personal Access Token stored in the repo's, individual's, or organization's secrets in order to work properly._ - -#### Sample Usage - -`.github/workflows/auto_check.yml` +Replace `.github/workflows/auto_check.yml` with: ```yaml name: Bump upstream version - on: schedule: - cron: "00 */4 * * *" - push: - branches: - - "master" - + workflow_dispatch: jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: npx @dappnode/dappnodesdk github-action bump-upstream - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PINATA_API_KEY: ${{ secrets.PINATA_API_KEY }} - PINATA_SECRET_API_KEY: ${{ secrets.PINATA_SECRET_API_KEY }} + bump-upstream: + uses: dappnode/workflows/.github/workflows/bump-upstream.yml@master + secrets: inherit ``` +The deprecated command continues to create bump PRs and run package builds until its removal. It preserves Compose comments and digest pins, reads optional upstream `provider` and `tagPattern` metadata, and automatically detects `package_variants/`. `--use_variants` is accepted but ignored; `--variants_dir` supports legacy directory layouts. Use `--upstream_version` to request an explicit tag for a single upstream. + You must specify in the package manifest what upstream Github repo the package is tracking. Also indicate what build variable in the docker-compose file should be updated. `dappnode_package.json` diff --git a/doc/package-spec.md b/doc/package-spec.md index e1f5cd43..a8345ab4 100644 --- a/doc/package-spec.md +++ b/doc/package-spec.md @@ -29,7 +29,7 @@ Repository counts and migration observations belong to dated audit reports; see **SPEC-M04: The upstream `arg` is a compose build arg.** Every upstream entry names an `arg` that some compose service declares in `build.args`. Packages using the legacy fields often rely on the implicit default `UPSTREAM_VERSION`. -**SPEC-M05: The upstream `version` equals every matching build arg value.** Every compose service declaring the upstream `arg` must use the same string as `upstream[].version`. Matching one service does not excuse a conflicting value in another. A value `@sha256:` also matches when the digest contains exactly 64 hexadecimal characters. Findings identify the conflicting service and the compose fragment that supplies its value. +**SPEC-M05: The upstream `version` equals every matching build arg value.** Every compose service declaring the upstream `arg` must use the same string as `upstream[].version`. Matching one service does not excuse a conflicting value in another. A value `@sha256:` also matches when the digest contains exactly 64 lowercase hexadecimal characters. Findings identify the conflicting service and the compose fragment that supplies its value. **SPEC-M06: A variant `upstream[]` keeps the root's positions.** Root and variant manifests are merged by array position. A variant may list fewer or more entries than the root, but an entry at an existing position must refer to the same `repo`. @@ -43,6 +43,8 @@ Repository counts and migration observations belong to dated audit reports; see **SPEC-M11: Global environment names match the runtime catalogue.** Entries in `globalEnvs[].envs` use short names from the `GLOBAL_ENVS` catalogue bundled with `@dappnode/types`, for example `PUBLIC_IP`, `HOSTNAME` or `ENVS_ACTIVE`. DAPPMANAGER adds `_DAPPNODE_GLOBAL_` at install, so `_DAPPNODE_GLOBAL_PUBLIC_IP` in the manifest would be prefixed twice. Catalogue keys are not aliases: the key `ACTIVE` maps to `ENVS_ACTIVE`, which is the valid manifest value. Catalogue updates must be released through shared types and consumed by the SDK before packages adopt newly supported names. +**SPEC-M12: Upstream bump metadata is valid.** Each `upstream[]` entry may set `provider` to `docker` (the default), `github`, or `docker_tags`, case-insensitively. `docker-tags` and `dockerhub` are aliases for `docker_tags`; legacy manifests use `upstreamProvider`. An optional `tagPattern` filters upstream tags and extracts capture group 1, or the whole match when no capture is present. Patterns must compile with JavaScript `RegExp` and use the common JavaScript/Rust regex subset: no lookaround or backreferences. Unknown providers and invalid patterns are reported as warnings. + ### Manifest semantics The source manifest describes package metadata and installation instructions. A template root and its variant manifests are source fragments; the merged result is the installable manifest and is validated against the full shared schema. Root fragments need not repeat each variant’s required identity fields. @@ -118,7 +120,7 @@ These are not checked yet, either because they need consumer changes first or be - **Upstream identity and inheritance:** define duplicate-entry behavior, a stable identity for overrides, and any migration away from positional array merging before changing either consumer. - **Compose `version:`** is obsolete. Rather than warn, the SDK should strip it from releases by default once versionless composes are verified with DAPPMANAGER (`--strip_compose_version` exists today as an opt-in). - **Dockerfiles:** no `FROM …:latest` or untagged base images, and no remote `ADD` without `--checksum`. The staking `ADD ${STAKER_SCRIPTS_URL}/*.sh` pattern needs a shared replacement first. -- **Declarative upstream metadata:** optional `upstream[].provider`, `tagPattern` and `imageTag` fields would let tropibot drop its built-in provider table and tag-to-image-tag inference. This needs a manifest schema change. +- **Declarative upstream metadata:** `upstream[].provider` and `tagPattern` are now checked by SPEC-M12. Publishing the shared schema/types support and defining an optional image override would allow more package-specific bump configuration to move into manifests. - **ENS names of new variants:** `-..dappnode.eth` for new variants, with an allowlist of existing on-chain names (`hoodi-geth`, `nethermind-xdai`, …). - **Explicit `architectures`**, replacing reliance on the implicit `linux/amd64`. - **Setup wizard targets:** every wizard target refers to a service and environment variable that exist in the compose (service/environment existence should be checked after merging). @@ -138,15 +140,15 @@ These came up while defining the rules and should be fixed regardless of the rol tropibot and the SDK share most package handling. Once a rule is an error, tropibot can drop the matching tolerance: -| Rule | tropibot code that becomes removable | +| Rule | TropiBot tolerance that becomes removable | |---|---| | SPEC-M01, SPEC-C01 | Hard-coded name assumptions in discovery, pin cleanup, on-chain sync and dashboard artifacts become guarantees. | -| SPEC-M03 | The legacy `upstreamRepo/Version/Arg` path in bump discovery and edits (`bump/package/discovery.rs`, `edits.rs`); dashboards keep it only for history. | -| SPEC-M04, SPEC-C05 | Upstream arg detection heuristics and the `UPSTREAM_VERSION` default (`discovery.rs`, `compose.rs`), list-style arg parsing and string `build:` handling. | -| SPEC-M05 | Part of the tag/arg format reconciliation in `resolve.rs` and `version_format.rs`; external tag formats (`n8n@x`, `v` prefixes) remain. | -| SPEC-M06 | The workaround for variants that re-declare `upstream` (`package.rs`). | -| SPEC-T01 | Phantom root packages in dashboard manifest merging (`source/manifests.rs`) and root-only pin cleanup. | -| SPEC-T02 | The `variants/` vs `package_variants/` dual layout (`bump/package/types.rs`, `discovery.rs`). | +| SPEC-M03 | Legacy upstream-field discovery and editing; dashboards keep the fields only for history. | +| SPEC-M04, SPEC-C05 | Upstream arg detection heuristics, the `UPSTREAM_VERSION` default, list-style arg parsing and string `build:` handling. | +| SPEC-M05 | Some version/build-arg reconciliation; external tag formats (`n8n@x`, `v` prefixes) remain. | +| SPEC-M06 | The workaround for variants that re-declare `upstream`. | +| SPEC-T01 | Phantom root packages in dashboard manifest merging and root-only pin cleanup. | +| SPEC-T02 | The `variants/` vs `package_variants/` dual layout. | | SPEC-M02, SPEC-T03 | Divergent name and variant regexes across modules collapse into one. | Issues in tropibot independent of the spec: @@ -154,8 +156,8 @@ Issues in tropibot independent of the spec: - Pin cleanup on branch close or delete uses only the root manifest `name`, so it errors for templates whose root has no name and misses variant packages. The SDK now cleans root and variant names (`cleanPinsFromDeletedBranches`); tropibot should do the same or leave cleanup to the SDK. - Pinata `pinList` is not paginated (default page size 10), so cleanup sees only the first page. - tropibot unpins when a PR closes even if its branch still exists; the SDK only cleans deleted branches. -- `docs/PACKAGE_CREATION_GUIDE.md` recommends a `variants//` layout with per-variant Dockerfiles, which contradicts SPEC-T02 and every real template. -- Two upstream bumpers exist: tropibot and the SDK's `github-action bump-upstream`. One should become canonical. +- Older package-creation guidance recommends a `variants//` layout with per-variant Dockerfiles, which contradicts SPEC-T02. +- The SDK’s `github-action bump-upstream` is deprecated and will be removed in a future release. DAppNode org packages should call `dappnode/workflows/.github/workflows/bump-upstream.yml@master`. Other packages should use `dappnode/workflows/actions/bump-upstream@master` for file edits, followed by their own PR step. See the migration example in [README.md](../README.md#bump-upstream-action-deprecated). Contract between the SDK and tropibot that the spec work must keep: diff --git a/package.json b/package.json index 8ffa489c..d0bd4f0d 100644 --- a/package.json +++ b/package.json @@ -60,8 +60,9 @@ "source-map-support": "^0.5.19", "tar-fs": "^3.1.3", "timestring": "^5.0.1", - "yargs": "^17.7.1", - "wrap-ansi": "^9.0.2" + "wrap-ansi": "^9.0.2", + "yaml": "^2.8.1", + "yargs": "^17.7.1" }, "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/src/bump/discovery.ts b/src/bump/discovery.ts new file mode 100644 index 00000000..7b549502 --- /dev/null +++ b/src/bump/discovery.ts @@ -0,0 +1,110 @@ +import fs from "node:fs"; +import path from "node:path"; +import { Manifest } from "@dappnode/types"; +import { + BumpUpstreamItem, + compileTagPattern, + parseProvider, + Provider +} from "./metadata.js"; + +export interface Upstream { + repo: string; + version: string; + arg: string; + provider: Provider; + tagPattern?: string; +} + +export interface BuildService { + build?: + | string + | { + context?: string; + dockerfile?: string; + args?: Record | string[]; + }; +} +export interface BumpCompose { + services: Record; +} + +export function buildArgs(service: BuildService): [string, unknown][] { + const args = + typeof service.build === "object" ? service.build.args : undefined; + return Array.isArray(args) + ? args.map(item => { + const split = item.indexOf("="); + return split < 0 + ? [item.trim(), null] + : [item.slice(0, split).trim(), item.slice(split + 1)]; + }) + : Object.entries(args || {}); +} + +export function discoverUpstreams( + manifest: Partial & { upstreamProvider?: string }, + compose: BumpCompose +): Upstream[] { + const entries: (Partial & { repo: string })[] = + manifest.upstream ?? + (manifest.upstreamRepo + ? [ + { + repo: manifest.upstreamRepo, + version: manifest.upstreamVersion, + arg: manifest.upstreamArg, + provider: manifest.upstreamProvider + } + ] + : []); + const named = [ + ...new Set( + Object.values(compose.services) + .flatMap(service => buildArgs(service).map(([arg]) => arg)) + .filter(arg => /VERSION|TAG/.test(arg)) + ) + ]; + const result = entries.map(entry => { + if (!entry.repo || !entry.version) + throw Error("Upstream repo and version are required"); + if (entries.length > 1 && !entry.arg) + throw Error("Every entry in a multi-upstream manifest must name an arg"); + if (!entry.arg && named.length > 1) + throw Error(`Cannot uniquely detect upstream arg: ${named.join(", ")}`); + compileTagPattern(entry.tagPattern); + return { + ...entry, + version: entry.version, + arg: entry.arg || named[0] || "UPSTREAM_VERSION", + provider: parseProvider(entry.provider) + }; + }); + for (const key of ["repo", "arg"] as const) + if (new Set(result.map(entry => entry[key])).size !== result.length) + throw Error( + `Upstream ${key === "repo" ? "repositories" : "args"} must be unique` + ); + return result; +} + +export function packageManifestDirs( + dir: string, + variantsDir = "package_variants" +): string[] { + for (const name of [...new Set([variantsDir, "package_variants"])]) { + const base = path.resolve(dir, name); + if (!fs.existsSync(base)) continue; + const dirs = fs + .readdirSync(base, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => path.join(base, entry.name)); + if (dirs.length) { + for (const variant of dirs) + if (!fs.existsSync(path.join(variant, "dappnode_package.json"))) + throw Error(`Variant has no manifest: ${variant}`); + return dirs; + } + } + return [dir]; +} diff --git a/src/bump/dockerfile.ts b/src/bump/dockerfile.ts new file mode 100644 index 00000000..d46b7e45 --- /dev/null +++ b/src/bump/dockerfile.ts @@ -0,0 +1,89 @@ +import fs from "node:fs"; +import path from "node:path"; +import { buildArgs, BumpCompose } from "./discovery.js"; + +export interface ImageSource { + image: string; + template: string; +} + +export function renderDockerTag(source: ImageSource, version: string): string { + return source.template.replace( + /\{version(?:#([^}]*))?\}/g, + (_, prefix: string | undefined) => + prefix && version.startsWith(prefix) + ? version.slice(prefix.length) + : version + ); +} + +export function extractImageSource( + content: string, + arg: string +): ImageSource | null { + const defaults = new Map(); + const lines = content.replace(/\\\r?\n/g, "").split(/\r?\n/); + for (const line of lines) { + const match = /^\s*ARG\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)\s*$/i.exec(line); + if (match) defaults.set(match[1], match[2].replace(/^["']|["']$/g, "")); + } + const expand = (expression: string, seen = new Set()): string => + expression.replace( + /\$\{([A-Za-z_][A-Za-z0-9_]*)(?:#([^}]*))?\}|\$([A-Za-z_][A-Za-z0-9_]*)/g, + ( + original, + braced: string | undefined, + prefix: string | undefined, + bare: string | undefined + ) => { + const name = braced || bare!; + if (name === arg) + return prefix === undefined ? "{version}" : `{version#${prefix}}`; + if (!defaults.has(name) || seen.has(name)) return original; + const value = expand(defaults.get(name)!, new Set([...seen, name])); + return prefix && value.startsWith(prefix) + ? value.slice(prefix.length) + : value; + } + ); + for (const line of lines) { + const from = /^\s*FROM\s+(?:--platform=\S+\s+)?(\S+)/i.exec(line); + if (!from) continue; + const token = expand(from[1]); + if (!token.includes("{version")) continue; + const colon = token.lastIndexOf(":"); + if (colon <= token.lastIndexOf("/")) + throw Error(`Cannot find image tag for ${arg}`); + const image = token.slice(0, colon); + const template = token.slice(colon + 1); + if ( + image.includes("$") || + image.includes("{version") || + template.includes("$") + ) + throw Error(`Unresolved Dockerfile argument for ${arg}`); + return { image, template }; + } + return null; +} + +export function findImageSource( + dir: string, + compose: BumpCompose, + arg: string +): ImageSource | null { + for (const service of Object.values(compose.services)) { + if ( + typeof service.build !== "object" || + !buildArgs(service).some(([key]) => key === arg) + ) + continue; + const file = path.resolve( + dir, + service.build.context || ".", + service.build.dockerfile || "Dockerfile" + ); + return extractImageSource(fs.readFileSync(file, "utf8"), arg); + } + return null; +} diff --git a/src/bump/edits.ts b/src/bump/edits.ts new file mode 100644 index 00000000..60905a77 --- /dev/null +++ b/src/bump/edits.ts @@ -0,0 +1,177 @@ +import fs from "node:fs"; +import { isDeepStrictEqual } from "node:util"; +import yaml from "js-yaml"; +import { + isAlias, + isMap, + isScalar, + isSeq, + parseDocument, + Scalar, + visit +} from "yaml"; +import { BumpCompose } from "./discovery.js"; + +export interface ValueChange { + path: (string | number)[]; + value: string; +} +interface TextEdit { + start: number; + end: number; + value: string; +} + +function document(text: string) { + const doc = parseDocument(text, { uniqueKeys: true }); + if (doc.errors.length) + throw Error( + `Invalid document: ${doc.errors.map(error => error.message).join("; ")}` + ); + visit(doc, { + Node(key, node) { + if (isAlias(node)) + throw Error("Aliases are ambiguous for upstream edits"); + if (isScalar(node) && node.value === "<<" && key === "key") + throw Error("YAML merges are ambiguous for upstream edits"); + } + }); + return doc; +} + +function scalarEdit( + text: string, + node: unknown, + value: string, + json = false +): TextEdit | null { + if (!isScalar(node) || !node.range) + throw Error("Expected a scalar at the edit location"); + if (node.value === value) return null; + let replacement: string; + if (json || node.type === Scalar.QUOTE_DOUBLE) + replacement = JSON.stringify(value); + else if (node.type === Scalar.QUOTE_SINGLE) + replacement = `'${value.replace(/'/g, "''")}'`; + else if (node.type === Scalar.PLAIN && yaml.load(value) === value) + replacement = value; + else if (node.type === Scalar.PLAIN) replacement = JSON.stringify(value); + else throw Error("Block scalars cannot be safely edited as upstream pins"); + const [start, end] = node.range; + if (!text.slice(start, end)) throw Error("Empty scalar edit location"); + return { start, end, value: replacement }; +} + +function replace(text: string, edits: (TextEdit | null)[]): string { + let previous = text.length; + for (const edit of edits + .filter((edit): edit is TextEdit => edit !== null) + .sort((a, b) => b.start - a.start)) { + if (edit.end > previous) throw Error("Overlapping upstream edits"); + text = text.slice(0, edit.start) + edit.value + text.slice(edit.end); + previous = edit.start; + } + return text; +} + +/** Reparse independently, allowing only the explicitly requested semantic changes. */ +export function validateEdits( + before: string, + after: string, + changes: ValueChange[], + json = false +): void { + const parse = (text: string): unknown => + json ? JSON.parse(text) : yaml.load(text); + const expected = parse(before); + for (const change of changes) { + let parent = expected as Record; + for (const key of change.path.slice(0, -1)) { + if (!parent || typeof parent !== "object" || !(key in parent)) + throw Error("Missing edit path"); + parent = parent[key] as Record; + } + const key = change.path[change.path.length - 1]; + if (!parent || typeof parent !== "object" || !(key in parent)) + throw Error("Missing edit value"); + parent[key] = change.value; + } + if (!isDeepStrictEqual(expected, parse(after))) + throw Error( + "Edit changed values outside the intended upstream/package updates" + ); +} + +export function editManifest(text: string, changes: ValueChange[]): string { + JSON.parse(text); + const doc = document(text); + const result = replace( + text, + changes.map(change => + scalarEdit(text, doc.getIn(change.path, true), change.value, true) + ) + ); + validateEdits(text, result, changes, true); + return result; +} + +export function editCompose( + text: string, + updates: Map +): string { + const doc = document(text); + const compose = yaml.load(text) as BumpCompose; + const edits: (TextEdit | null)[] = []; + const changes: ValueChange[] = []; + const found = new Set(); + for (const service of Object.keys(compose.services || {})) { + const base = ["services", service, "build", "args"]; + const args = doc.getIn(base, true); + if (isMap(args)) { + for (const [arg, value] of updates) { + if (!args.has(arg)) continue; + found.add(arg); + const node = args.get(arg, true); + edits.push(scalarEdit(text, node, value)); + if (!isScalar(node) || node.value !== value) + changes.push({ path: [...base, arg], value }); + } + } else if (isSeq(args)) { + args.items.forEach((node, index) => { + if (!isScalar(node) || typeof node.value !== "string") + throw Error("List build args must be strings"); + const split = node.value.indexOf("="); + const arg = ( + split < 0 ? node.value : node.value.slice(0, split) + ).trim(); + const value = updates.get(arg); + if (value === undefined) return; + found.add(arg); + const replacement = `${arg}=${value}`; + edits.push(scalarEdit(text, node, replacement)); + if (node.value !== replacement) + changes.push({ path: [...base, index], value: replacement }); + }); + } + } + for (const arg of updates.keys()) + if (!found.has(arg)) + throw Error(`Required Compose build argument '${arg}' is absent`); + const result = replace(text, edits); + validateEdits(text, result, changes); + return result; +} + +export interface PlannedFile { + path: string; + before: string; + after: string; +} + +export function writePlan(plan: PlannedFile[]): void { + for (const file of plan) + if (fs.readFileSync(file.path, "utf8") !== file.before) + throw Error(`File changed during bump planning: ${file.path}`); + for (const file of plan) + if (file.before !== file.after) fs.writeFileSync(file.path, file.after); +} diff --git a/src/bump/metadata.ts b/src/bump/metadata.ts new file mode 100644 index 00000000..4616db28 --- /dev/null +++ b/src/bump/metadata.ts @@ -0,0 +1,72 @@ +import { UpstreamItem } from "@dappnode/types"; + +/** Optional metadata supported before it is published in shared types. */ +export type BumpUpstreamItem = UpstreamItem & { + provider?: string; + tagPattern?: string; +}; +export type Provider = "docker" | "github" | "docker_tags"; + +export function parseProvider(value: unknown): Provider { + if (value === undefined) return "docker"; + if (typeof value === "string") { + switch (value.toLowerCase()) { + case "docker": + return "docker"; + case "github": + return "github"; + case "docker_tags": + case "docker-tags": + case "dockerhub": + return "docker_tags"; + } + } + throw Error(`Unknown upstream provider: ${String(value)}`); +} + +/** Restrict patterns to constructs shared by JavaScript and Rust regex. */ +export function compileTagPattern(value: unknown): RegExp | undefined { + if (value === undefined) return undefined; + if (typeof value !== "string") throw Error("tagPattern must be a string"); + let inClass = false; + for (let i = 0; i < value.length; i++) { + const char = value[i]; + if (char === "\\") { + const escaped = value[++i]; + if (/[1-9]/.test(escaped || "") || escaped === "k") + throw Error("tagPattern must not use backreferences"); + if ( + escaped && + /[a-zA-Z0-9]/.test(escaped) && + !/[dDsSwWbBnrtfvxu]/.test(escaped) + ) + throw Error( + "tagPattern uses an escape outside the common regex subset" + ); + if (inClass && escaped === "b") + throw Error( + "tagPattern uses a class escape outside the common regex subset" + ); + continue; + } + if (char === "[") inClass = true; + if (char === "]") inClass = false; + if ( + !inClass && + char === "(" && + value[i + 1] === "?" && + value[i + 2] !== ":" + ) + throw Error( + "tagPattern must use the common regex subset (no lookaround or named groups)" + ); + } + return new RegExp(value); +} + +export function extractVersion(tag: string, pattern?: string): string | null { + const regex = compileTagPattern(pattern); + if (!regex) return tag; + const match = regex.exec(tag); + return match ? match[1] ?? match[0] : null; +} diff --git a/src/bump/plan.ts b/src/bump/plan.ts new file mode 100644 index 00000000..e72d46f9 --- /dev/null +++ b/src/bump/plan.ts @@ -0,0 +1,110 @@ +import fs from "node:fs"; +import path from "node:path"; +import yaml from "js-yaml"; +import { Manifest } from "@dappnode/types"; +import { + BumpCompose, + discoverUpstreams, + packageManifestDirs +} from "./discovery.js"; +import { + editCompose, + editManifest, + PlannedFile, + ValueChange +} from "./edits.js"; +import { + ResolutionDependencies, + ResolvedUpstream, + resolveUpstream +} from "./resolve.js"; + +export async function resolvePackage( + dir: string, + dependencies: ResolutionDependencies, + explicitVersion?: string +): Promise { + const manifest = JSON.parse( + fs.readFileSync(path.join(dir, "dappnode_package.json"), "utf8") + ) as Manifest; + const compose = yaml.load( + fs.readFileSync(path.join(dir, "docker-compose.yml"), "utf8") + ) as BumpCompose; + const upstreams = discoverUpstreams(manifest, compose); + if (explicitVersion !== undefined && upstreams.length !== 1) + throw Error("An explicit version requires exactly one upstream"); + const results: ResolvedUpstream[] = []; + for (const upstream of upstreams) { + const result = await resolveUpstream( + dir, + compose, + upstream, + dependencies, + explicitVersion + ); + if (result) results.push(result); + } + return results; +} + +export async function planPackageEdits({ + dir, + upstreams, + nextPackageVersion, + variantsDir +}: { + dir: string; + upstreams: ResolvedUpstream[]; + nextPackageVersion: (dir: string) => Promise; + variantsDir?: string; +}): Promise { + const changed = upstreams.filter(item => item.targetVersion !== item.version); + if (!changed.length) return []; + const manifestFile = path.join(dir, "dappnode_package.json"); + const manifestText = fs.readFileSync(manifestFile, "utf8"); + const manifest = JSON.parse(manifestText) as Manifest; + const changes: ValueChange[] = changed.map(item => { + const index = manifest.upstream?.findIndex( + upstream => upstream.repo === item.repo + ); + if (manifest.upstream && (index === undefined || index < 0)) + throw Error(`Upstream '${item.repo}' is absent from root manifest`); + return { + path: manifest.upstream + ? ["upstream", index!, "version"] + : ["upstreamVersion"], + value: item.targetVersion + }; + }); + const plan: PlannedFile[] = []; + for (const packageDir of packageManifestDirs(dir, variantsDir)) { + const file = path.join(packageDir, "dappnode_package.json"); + const value = await nextPackageVersion(packageDir); + if (path.resolve(file) === path.resolve(manifestFile)) + changes.push({ path: ["version"], value }); + else { + const before = fs.readFileSync(file, "utf8"); + plan.push({ + path: file, + before, + after: editManifest(before, [{ path: ["version"], value }]) + }); + } + } + plan.push({ + path: manifestFile, + before: manifestText, + after: editManifest(manifestText, changes) + }); + const composeFile = path.join(dir, "docker-compose.yml"); + const before = fs.readFileSync(composeFile, "utf8"); + plan.push({ + path: composeFile, + before, + after: editCompose( + before, + new Map(changed.map(item => [item.arg, item.composeValue])) + ) + }); + return plan; +} diff --git a/src/bump/registry.ts b/src/bump/registry.ts new file mode 100644 index 00000000..dde3b004 --- /dev/null +++ b/src/bump/registry.ts @@ -0,0 +1,188 @@ +import { createHash } from "node:crypto"; +import { highestSemverTag } from "./versions.js"; + +export const manifestAccept = + "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json"; +export const digestPattern = /^sha256:[a-f0-9]{64}$/; + +export function registryImage(image: string): { + host: string; + name: string; + hub: boolean; +} { + const parts = image.split("/"); + const explicit = + parts.length > 1 && (/[.:]/.test(parts[0]) || parts[0] === "localhost"); + const host = explicit ? parts.shift()! : "docker.io"; + const hub = ["docker.io", "index.docker.io", "registry-1.docker.io"].includes( + host + ); + const name = parts.join("/"); + return { + host: hub ? "registry-1.docker.io" : host, + name: hub && !name.includes("/") ? `library/${name}` : name, + hub + }; +} + +function checkUrl(value: string): URL { + const url = new URL(value); + const loopback = + url.hostname === "localhost" || + url.hostname === "[::1]" || + /^127\.\d+\.\d+\.\d+$/.test(url.hostname); + if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback)) + throw Error(`Registry URL must use HTTPS: ${url.origin}`); + return url; +} + +export class RegistryClient { + constructor(private readonly request: typeof fetch = fetch) {} + + private async send( + url: string, + method: string, + name: string, + accept = "application/json" + ): Promise { + checkUrl(url); + const headers: Record = { Accept: accept }; + const send = () => + this.request(url, { + method, + headers, + signal: AbortSignal.timeout(30000) + }); + let response = await send(); + if (response.status !== 401) return response; + const challenge = response.headers.get("www-authenticate") || ""; + if (!/^Bearer\s/i.test(challenge)) + throw Error(`Registry authentication failed (${response.status})`); + const fields = new Map( + [...challenge.matchAll(/([a-z_]+)=(?:"([^"]*)"|([^,\s]+))/gi)].map( + match => [match[1].toLowerCase(), match[2] ?? match[3]] + ) + ); + const realm = fields.get("realm"); + if (!realm) throw Error("Registry bearer challenge has no realm"); + const tokenUrl = checkUrl(realm); + if (fields.has("service")) + tokenUrl.searchParams.set("service", fields.get("service")!); + tokenUrl.searchParams.set("scope", `repository:${name}:pull`); + const tokenResponse = await this.request(tokenUrl, { + signal: AbortSignal.timeout(30000) + }); + if (tokenResponse.status !== 200) + throw Error(`Registry token request failed (${tokenResponse.status})`); + const body = (await tokenResponse.json()) as { + token?: string; + access_token?: string; + }; + const token = body.token || body.access_token; + if (!token) throw Error("Registry token response has no token"); + headers.Authorization = `Bearer ${token}`; + response = await send(); + return response; + } + + private manifestUrl( + image: string, + tag: string + ): { url: string; name: string } { + const { host, name } = registryImage(image); + const scheme = + host === "localhost" || + host.startsWith("localhost:") || + host.startsWith("127.") || + host.startsWith("[::1]") + ? "http" + : "https"; + return { + url: `${scheme}://${host}/v2/${name}/manifests/${encodeURIComponent( + tag + )}`, + name + }; + } + + async tagExists(image: string, tag: string): Promise { + const { url, name } = this.manifestUrl(image, tag); + const response = await this.send(url, "HEAD", name, manifestAccept); + if (response.status === 200) return true; + if (response.status === 404) return false; + throw Error( + `Registry returned ${response.status} checking ${image}:${tag}` + ); + } + + async manifestDigest(image: string, tag: string): Promise { + const { url, name } = this.manifestUrl(image, tag); + let response = await this.send(url, "HEAD", name, manifestAccept); + if (response.status !== 200) + throw Error( + `Registry returned ${response.status} pinning ${image}:${tag}` + ); + let digest = response.headers.get("docker-content-digest")?.trim(); + if (digest === undefined) { + response = await this.send(url, "GET", name, manifestAccept); + if (response.status !== 200) + throw Error( + `Registry returned ${response.status} pinning ${image}:${tag}` + ); + digest = `sha256:${createHash("sha256") + .update(Buffer.from(await response.arrayBuffer())) + .digest("hex")}`; + } + if (!digestPattern.test(digest)) + throw Error( + `Registry returned malformed digest for ${image}:${tag}: ${digest}` + ); + return digest; + } + + async listTags(image: string): Promise { + const { host, name, hub } = registryImage(image); + const scheme = + host === "localhost" || + host.startsWith("localhost:") || + host.startsWith("127.") || + host.startsWith("[::1]") + ? "http" + : "https"; + let url: string | null = hub + ? `https://registry.hub.docker.com/v2/repositories/${name}/tags?page_size=100&ordering=last_updated` + : `${scheme}://${host}/v2/${name}/tags/list`; + const tags: string[] = []; + const seen = new Set(); + for (let page = 0; url && page < 50; page++) { + if (seen.has(url)) throw Error(`Registry pagination loop for ${image}`); + seen.add(url); + const response = await this.send(url, "GET", name); + if (response.status !== 200) + throw Error(`Registry returned ${response.status} listing ${image}`); + if (hub) { + const body = (await response.json()) as { + results: { name: string }[]; + next?: string | null; + }; + tags.push(...body.results.map(tag => tag.name)); + url = body.next ? new URL(body.next, url).href : null; + } else { + const body = (await response.json()) as { tags?: string[] | null }; + tags.push(...(body.tags || [])); + const link = response.headers.get("link"); + const next: string | undefined = link + ?.split(",") + .map(entry => /<([^>]+)>\s*;\s*rel\s*=\s*"?next"?/i.exec(entry)?.[1]) + .find(Boolean); + url = next ? new URL(next, url).href : null; + } + } + if (url) throw Error(`Tag listing exceeded 50 pages for ${image}`); + return tags; + } + + async latestTag(image: string): Promise { + return highestSemverTag(await this.listTags(image)); + } +} diff --git a/src/bump/resolve.ts b/src/bump/resolve.ts new file mode 100644 index 00000000..1472645b --- /dev/null +++ b/src/bump/resolve.ts @@ -0,0 +1,145 @@ +import { BumpCompose, buildArgs, Upstream } from "./discovery.js"; +import { findImageSource, ImageSource, renderDockerTag } from "./dockerfile.js"; +import { extractVersion } from "./metadata.js"; +import { digestPattern, RegistryClient } from "./registry.js"; +import { isOlderVersion, versionCandidates } from "./versions.js"; + +export function splitPin(value: string): { version: string; digest?: string } { + const at = value.lastIndexOf("@"); + if (at < 0 || !value.slice(at + 1).startsWith("sha256:")) + return { version: value }; + const version = value.slice(0, at); + const digest = value.slice(at + 1); + if (!version || !digestPattern.test(digest)) + throw Error( + `Invalid digest pin: ${value}; expected 64 lowercase hex characters` + ); + return { version, digest }; +} + +export function currentPin( + compose: BumpCompose, + upstream: Upstream +): { version: string; digest?: string } { + const entries = Object.entries(compose.services).flatMap(([service, value]) => + buildArgs(value) + .filter(([key]) => key === upstream.arg) + .map(([, value]) => ({ service, value })) + ); + if (!entries.length) + throw Error(`Required Compose build argument '${upstream.arg}' is absent`); + const pins = entries.map(({ service, value }) => { + if (typeof value !== "string" && typeof value !== "number") + throw Error(`Inherited or invalid arg ${upstream.arg} in ${service}`); + const pin = splitPin(String(value)); + if (pin.version !== upstream.version) + throw Error( + `Inconsistent Compose pin for ${upstream.arg} in ${service}: ${String( + value + )}; manifest has ${upstream.version}` + ); + return pin; + }); + if (pins.some(pin => pin.digest !== pins[0].digest)) + throw Error(`Inconsistent digest pins across services for ${upstream.arg}`); + return pins[0]; +} + +export interface ResolvedUpstream extends Upstream { + targetVersion: string; + composeValue: string; +} +export interface ResolutionDependencies { + latestRelease: (repo: string, tagPattern?: string) => Promise; + registry: Pick; + imageSource?: ( + dir: string, + compose: BumpCompose, + arg: string + ) => ImageSource | null; +} + +export async function resolveUpstream( + dir: string, + compose: BumpCompose, + upstream: Upstream, + dependencies: ResolutionDependencies, + explicitVersion?: string +): Promise { + const pin = currentPin(compose, upstream); + let source: ImageSource | null | undefined; + const image = () => + source === undefined + ? (source = (dependencies.imageSource || findImageSource)( + dir, + compose, + upstream.arg + )) + : source; + let tag: string | null; + if (explicitVersion !== undefined) + tag = + extractVersion(explicitVersion, upstream.tagPattern) ?? explicitVersion; + else if (upstream.provider === "docker_tags") { + const found = image(); + if (!found) throw Error(`No FROM image uses ${upstream.arg}`); + const chosen = await dependencies.registry.latestTag(found.image); + tag = chosen === null ? null : extractVersion(chosen, upstream.tagPattern); + if (chosen !== null && tag === null) + throw Error(`Latest image tag '${chosen}' does not match tagPattern`); + } else + tag = await dependencies.latestRelease(upstream.repo, upstream.tagPattern); + if (tag === null) return null; + const unchanged = (): ResolvedUpstream => ({ + ...upstream, + targetVersion: upstream.version, + composeValue: upstream.version + (pin.digest ? `@${pin.digest}` : "") + }); + if (versionCandidates(tag).includes(upstream.version)) return unchanged(); + if (explicitVersion === undefined && isOlderVersion(tag, upstream.version)) + return unchanged(); + let targetVersion = tag; + if ( + upstream.provider === "docker" || + (upstream.provider === "docker_tags" && explicitVersion !== undefined) + ) { + const found = image(); + if (found) { + let candidate: string | undefined; + for (const version of versionCandidates(tag)) { + if ( + await dependencies.registry.tagExists( + found.image, + renderDockerTag(found, version) + ) + ) { + candidate = version; + break; + } + } + if (!candidate) + throw Error( + `No Docker tag candidate exists on ${found.image} for ${tag}` + ); + targetVersion = candidate; + } + } + if ( + targetVersion === upstream.version || + (explicitVersion === undefined && + isOlderVersion(targetVersion, upstream.version)) + ) + return unchanged(); + let composeValue = targetVersion; + if (pin.digest) { + const found = image(); + if (!found) + throw Error(`Pinned argument ${upstream.arg} needs a FROM image`); + const digest = await dependencies.registry.manifestDigest( + found.image, + renderDockerTag(found, targetVersion) + ); + composeValue += `@${digest}`; + } + return { ...upstream, targetVersion, composeValue }; +} diff --git a/src/bump/versions.ts b/src/bump/versions.ts new file mode 100644 index 00000000..b9d2fd88 --- /dev/null +++ b/src/bump/versions.ts @@ -0,0 +1,36 @@ +import semver from "semver"; + +export function versionCandidates(tag: string): string[] { + const bare = /^\d/.test(tag) + ? undefined + : tag.match(/\d+\.\d+\.\d+(?:[-+].*)?/)?.[0]; + return bare && bare !== tag ? [tag, bare] : [tag]; +} + +export function comparableVersion(value: string): string | null { + return ( + versionCandidates(value) + .map(candidate => semver.valid(candidate)) + .find(Boolean) ?? null + ); +} + +export function isOlderVersion(latest: string, current: string): boolean { + const a = comparableVersion(latest); + const b = comparableVersion(current); + return Boolean(a && b && semver.lt(a, b)); +} + +export function highestSemverTag(tags: string[]): string | null { + const valid = tags.filter( + tag => /^v?\d+\.\d+\.\d+(?:[-+].*)?$/.test(tag) && semver.valid(tag) + ); + const stable = valid.filter(tag => semver.prerelease(tag) === null); + return ( + (stable.length ? stable : valid).sort( + (a, b) => + semver.rcompare(a, b) || + Number(a.startsWith("v")) - Number(b.startsWith("v")) + )[0] ?? null + ); +} diff --git a/src/commands/githubActions/bumpUpstream/deprecation.ts b/src/commands/githubActions/bumpUpstream/deprecation.ts new file mode 100644 index 00000000..3b2b91cd --- /dev/null +++ b/src/commands/githubActions/bumpUpstream/deprecation.ts @@ -0,0 +1,11 @@ +export const bumpDeprecation = + "dappnodesdk github-action bump-upstream is deprecated and will be removed in a future release. DAppNode org packages: call the reusable workflow dappnode/workflows/.github/workflows/bump-upstream.yml@master. Other packages: use the dappnode/workflows/actions/bump-upstream@master action, which edits the files, followed by your own PR step."; + +export function printBumpDeprecation( + log: (message: string) => void = console.log +): void { + log(bumpDeprecation); + log( + `::warning title=dappnodesdk bump-upstream is deprecated::${bumpDeprecation}` + ); +} diff --git a/src/commands/githubActions/bumpUpstream/github/fetchGithubUpstreamVersion.ts b/src/commands/githubActions/bumpUpstream/github/fetchGithubUpstreamVersion.ts index dee7324c..42fdca72 100644 --- a/src/commands/githubActions/bumpUpstream/github/fetchGithubUpstreamVersion.ts +++ b/src/commands/githubActions/bumpUpstream/github/fetchGithubUpstreamVersion.ts @@ -1,11 +1,12 @@ import { Github } from "../../../../providers/github/Github.js"; -import { pickLatestRelease } from "./pickLatestRelease.js"; +import { selectLatestRelease } from "./pickLatestRelease.js"; export async function fetchGithubUpstreamVersion( - repo: string + repo: string, + tagPattern?: string ): Promise { try { - const newVersion = await fetchGithubLatestTag(repo); + const newVersion = await fetchGithubLatestTag(repo, tagPattern); if (!newVersion) { console.log( `No stable release found (only drafts, pre-releases or release candidates) - ${repo}` @@ -21,12 +22,21 @@ export async function fetchGithubUpstreamVersion( } } -async function fetchGithubLatestTag(repo: string): Promise { +async function fetchGithubLatestTag( + repo: string, + tagPattern?: string +): Promise { const [owner, repoName] = repo.split("/"); const githubRepo = new Github({ owner, repo: repoName }); - const releases = await githubRepo.listReleases(); - if (releases.length === 0) throw Error(`No release found for ${repo}`); - - return pickLatestRelease(releases); + return selectLatestRelease( + await githubRepo.getLatestRelease(), + async () => { + // Release bodies make pages of 100 several MB; 30 recent releases suffice. + const releases = await githubRepo.listReleases(30); + if (releases.length === 0) throw Error(`No release found for ${repo}`); + return releases; + }, + tagPattern + ); } diff --git a/src/commands/githubActions/bumpUpstream/github/getBumpPrBody.ts b/src/commands/githubActions/bumpUpstream/github/getBumpPrBody.ts index 79f6faf3..88b64693 100644 --- a/src/commands/githubActions/bumpUpstream/github/getBumpPrBody.ts +++ b/src/commands/githubActions/bumpUpstream/github/getBumpPrBody.ts @@ -5,15 +5,16 @@ export function getBumpPrBody(upstreamSettings: UpstreamSettings[]): string { "Bumps upstream version", upstreamSettings .flatMap( - ({ repo, githubVersion, manifestVersion }) => + ({ repo, githubVersion, manifestVersion, composeValue }) => `- [${repo}](${getGitHubUrl({ repo })}) from ${manifestVersion} to [${githubVersion}](${getGitHubUrl({ repo, tag: githubVersion - })})` + })}) (Compose: \`${composeValue || githubVersion}\`)` ) - .join("\n") + .join("\n"), + '' ].join("\n\n"); } diff --git a/src/commands/githubActions/bumpUpstream/github/isValidRelease.ts b/src/commands/githubActions/bumpUpstream/github/isValidRelease.ts index 7d5d84b4..58c77f1a 100644 --- a/src/commands/githubActions/bumpUpstream/github/isValidRelease.ts +++ b/src/commands/githubActions/bumpUpstream/github/isValidRelease.ts @@ -1,18 +1,38 @@ import semver from "semver"; +import { versionCandidates } from "../../../../bump/versions.js"; + +export const releaseTagPattern = /^[a-zA-Z0-9][a-zA-Z0-9._+/@-]*$/; + +/** + * Semver pre-release parts starting with one of these words mark a pre-release. + * Other suffixes are stable flavours, e.g. gnosischain/go-ethereum `v1.17.10-gc`. + */ +const prereleaseWords = [ + "alpha", + "beta", + "rc", + "pre", + "preview", + "dev", + "canary", + "nightly", + "snapshot", + "unstable" +]; export function isValidRelease(version: string): boolean { // Commands receive tags as arguments, not shell source; this only rejects // characters that can never form a branch name. `@` is used by real // upstreams (e.g. n8n@2.46.6). - if (!/^[a-zA-Z0-9][a-zA-Z0-9._+/@-]*$/.test(version)) return false; + if (!releaseTagPattern.test(version)) return false; // Nightly builds are not considered valid releases (not taken into account by semver) if (version.toLowerCase().includes("nightly")) return false; - if (semver.valid(version)) { - const preReleases = semver.prerelease(version); - - // A version is considered a valid release if it has no pre-release components. - return preReleases === null || preReleases.length === 0; + // Strip a non-numeric prefix such as `v` or `n8n@` before parsing. + const core = semver.valid(versionCandidates(version).at(-1)!); + if (core) { + const pre = (semver.prerelease(core) || []).join(".").toLowerCase(); + return !prereleaseWords.some(word => pre.startsWith(word)); } console.warn( diff --git a/src/commands/githubActions/bumpUpstream/github/pickLatestRelease.ts b/src/commands/githubActions/bumpUpstream/github/pickLatestRelease.ts index 42ab0053..8e22bde4 100644 --- a/src/commands/githubActions/bumpUpstream/github/pickLatestRelease.ts +++ b/src/commands/githubActions/bumpUpstream/github/pickLatestRelease.ts @@ -1,4 +1,5 @@ -import { isValidRelease } from "./isValidRelease.js"; +import { isValidRelease, releaseTagPattern } from "./isValidRelease.js"; +import { extractVersion } from "../../../../bump/metadata.js"; interface Release { tag_name: string; @@ -7,15 +8,42 @@ interface Release { } /** - * Returns the tag of the newest release that is not a draft, not flagged as a - * pre-release and passes isValidRelease, or null when there is none. + * Returns the version of the first qualifying release, or null when there is + * none. A release qualifies when it is not a draft, not flagged as a + * pre-release, matches `tagPattern` and passes isValidRelease. The version is + * capture group 1 of `tagPattern` (or the whole match), else the tag. * `releases` comes newest first, as GitHub lists them. A release candidate * published after a stable release must not hide that stable release. */ -export function pickLatestRelease(releases: Release[]): string | null { +export function pickLatestRelease( + releases: Release[], + tagPattern?: string +): string | null { for (const { tag_name, draft, prerelease } of releases) { - if (draft || prerelease || !isValidRelease(tag_name)) continue; - return tag_name; + if (draft || prerelease || tag_name.toLowerCase().includes("nightly")) + continue; + const version = extractVersion(tag_name, tagPattern); + if ( + !version || + !releaseTagPattern.test(tag_name) || + !isValidRelease(version) + ) + continue; + return version; } return null; } + +/** + * GitHub's Latest release when it qualifies, else the first qualifying recent + * release. Latest respects the maintainer's choice, e.g. upstreams that + * publish backports after a newer release line. + */ +export async function selectLatestRelease( + latest: Release | null, + recentReleases: () => Promise, + tagPattern?: string +): Promise { + const fromLatest = latest && pickLatestRelease([latest], tagPattern); + return fromLatest || pickLatestRelease(await recentReleases(), tagPattern); +} diff --git a/src/commands/githubActions/bumpUpstream/index.ts b/src/commands/githubActions/bumpUpstream/index.ts index b0cc6ed0..1c8ea40e 100644 --- a/src/commands/githubActions/bumpUpstream/index.ts +++ b/src/commands/githubActions/bumpUpstream/index.ts @@ -4,16 +4,15 @@ import { defaultDir, defaultVariantsDirName } from "../../../params.js"; import { shell } from "../../../utils/shell.js"; import { getGitHead } from "../../../utils/git.js"; import { buildAndComment } from "../build/index.js"; -import { - writeManifest, - writeCompose, - readManifest -} from "../../../files/index.js"; +import { readManifest } from "../../../files/index.js"; +import { parseProvider } from "../../../bump/metadata.js"; +import { packageManifestDirs } from "../../../bump/discovery.js"; +import { planPackageEdits } from "../../../bump/plan.js"; +import { writePlan } from "../../../bump/edits.js"; +import { printBumpDeprecation } from "./deprecation.js"; import { getNextVersionFromApm } from "../../../utils/versions/getNextVersionFromApm.js"; -import { Compose, Manifest } from "@dappnode/types"; import { GitSettings, GithubSettings, UpstreamSettings } from "./types.js"; import { printSettings, getInitialSettings } from "./settings/index.js"; -import { ManifestFormat } from "../../../files/manifest/types.js"; import { branchNeedsPr, closeOldPrs, @@ -23,7 +22,6 @@ import { recoverBumpBranch } from "./github/index.js"; import path from "path"; -import { getAllVariantsInPath } from "../../../files/variants/getAllPackageVariants.js"; interface CliCommandOptions extends CliGlobalOptions { eth_provider: string; @@ -31,6 +29,7 @@ interface CliCommandOptions extends CliGlobalOptions { use_variants?: boolean; variants_dir: string; skip_build?: boolean; + upstream_version?: string; } // This action should be run periodically @@ -41,7 +40,7 @@ export const gaBumpUpstream: CommandModule< > = { command: "bump-upstream", describe: - "Check if upstream repo has released a new version and open a PR with version bump", + "DEPRECATED: bump upstream and open a PR; migrate to dappnode/workflows bump-upstream", builder: { eth_provider: { alias: "p", @@ -59,14 +58,19 @@ export const gaBumpUpstream: CommandModule< }, use_variants: { alias: "use-variants", - description: `It will use the dappnode_package.json and docker-compose.yml files in the root of the project together with the specific ones defined for each package variant to build all of them`, + description: "Ignored: package variants are detected automatically", type: "boolean" }, variants_dir: { - description: `Path to the directory where the package variants are located (only for packages that support it and combined with either "--all-variants" or "--variants"). By default, it is ${defaultVariantsDirName}`, + description: `Directory containing package variant manifests, detected automatically (default: ${defaultVariantsDirName})`, type: "string", default: defaultVariantsDirName }, + upstream_version: { + description: + "Explicit upstream release tag or version (single upstream only)", + type: "string" + }, skip_build: { description: `Only create the bump PR without building the package`, type: "boolean" @@ -81,15 +85,25 @@ async function gaBumpUpstreamHandler({ use_fallback: useFallback, use_variants: useVariants, variants_dir: variantsDir, - skip_build: skipBuild + skip_build: skipBuild, + upstream_version: explicitVersion }: CliCommandOptions): Promise { - const { - upstreamSettings, - manifestData: { manifest, format: manifestFormat }, - compose, - gitSettings, - ethProvider - } = await getInitialSettings({ dir, userEthProvider, useFallback }); + printBumpDeprecation(); + if (useVariants !== undefined) + console.log( + "--use_variants is ignored; variants are detected automatically" + ); + const manifestDirs = packageManifestDirs(dir, variantsDir); + const allVariants = manifestDirs.some( + packageDir => path.resolve(packageDir) !== path.resolve(dir) + ); + const { upstreamSettings, gitSettings, ethProvider } = + await getInitialSettings({ + dir, + userEthProvider, + useFallback, + explicitVersion + }); if (!upstreamSettings) { console.log("There are no upstream repos/versions defined in the manifest"); return; @@ -129,21 +143,22 @@ async function gaBumpUpstreamHandler({ }) }); } else { - updateComposeUpstreamVersions(dir, compose, upstreamSettings); - - updateManifestUpstreamVersion({ - manifest, - manifestFormat, - upstreamSettings, - dir - }); - - await updateManifestPkgVersion({ + const plan = await planPackageEdits({ dir, - ethProvider, - allVariants: useVariants, - variantsDir + variantsDir, + upstreams: upstreamSettings.map(item => ({ + repo: item.repo, + arg: item.arg, + provider: parseProvider(item.provider), + tagPattern: item.tagPattern, + version: item.manifestVersion, + targetVersion: item.githubVersion, + composeValue: item.composeValue || item.githubVersion + })), + nextPackageVersion: packageDir => + getNewManifestVersion({ dir: packageDir, ethProvider }) }); + writePlan(plan); await prepareAndCommitChanges({ dir, @@ -169,118 +184,13 @@ async function gaBumpUpstreamHandler({ dir, commitSha: gitHead.commit, branch: branchName, - all_variants: useVariants, - variants_dir_name: variantsDir + all_variants: allVariants, + variants_dir_name: allVariants + ? path.relative(dir, path.dirname(manifestDirs[0])) + : variantsDir }); } -/** - * Updates Docker Compose service build arguments with new versions based on `upstreamRepoVersions`. - * Creates a deep copy of `compose`, modifies build arguments as needed, and writes the updated - * compose to `dir` if changes occur. Returns an object detailing updates or `null` if no update is needed. - * - * @param {string} dir - Directory for the Compose file. - * @param {Compose} compose - Original Docker Compose configuration. - * @param {UpstreamSettings[]} upstreamSettings - New versions for the Compose services. - */ -function updateComposeUpstreamVersions( - dir: string, - compose: Compose, - upstreamSettings: UpstreamSettings[] -): void { - const newCompose: Compose = JSON.parse(JSON.stringify(compose)); // Deep copy - - for (const upstreamItem of upstreamSettings) { - // Checks if the service includes a build argument with the same name as the - // upstream item (e.g. "GETH_VERSION" is a build argument for the Geth service) - for (const [, service] of Object.entries(newCompose.services)) - if ( - typeof service.build !== "string" && - service.build?.args && - upstreamItem.arg in service.build.args - ) - service.build.args[upstreamItem.arg] = upstreamItem.githubVersion; - } - - writeCompose(newCompose, { dir }); -} - -/** - * Updates the manifest with new version tags based on the provided `composeVersionsToUpdate` - * and optionally fetches a new version for the manifest. The updated manifest is returned. - * @param {Object} options - The options for updating the manifest. - * @param {Manifest} options.manifest - The manifest object to update. - * @param {UpstreamSettings[]} options.upstreamSettings - The new versions for the manifest. - * @param {string} options.dir - Directory path where the manifest will be saved. - * @param {string} options.ethProvider - Ethereum provider URL. - * @returns {Promise} The updated manifest object. - */ -function updateManifestUpstreamVersion({ - manifest, - manifestFormat, - upstreamSettings, - dir -}: { - manifest: Manifest; - manifestFormat: ManifestFormat; - upstreamSettings: UpstreamSettings[]; - dir: string; -}): void { - if (manifest.upstream) { - for (const upstreamItem of manifest.upstream) { - const versionUpdate = upstreamSettings.find( - ({ repo }) => repo === upstreamItem.repo - )?.githubVersion; - - if (versionUpdate) upstreamItem.version = versionUpdate; - } - } else { - // There should be only one upstream repo in the legacy format - manifest.upstreamVersion = upstreamSettings[0].githubVersion; - } - - try { - writeManifest(manifest, manifestFormat, { dir }); - } catch (e) { - throw new Error(`Error writing manifest: ${e.message}`, { cause: e }); - } -} - -async function updateManifestPkgVersion({ - dir, - ethProvider, - allVariants, - variantsDir -}: { - dir: string; - ethProvider: string; - allVariants?: boolean; - variantsDir: string; -}): Promise { - const variantsPath = path.resolve(dir, variantsDir); - const manifestDirs = allVariants - ? getAllVariantsInPath(variantsPath).map(variant => - path.join(variantsPath, variant) - ) - : [dir]; - - for (const dir of manifestDirs) { - try { - const { manifest, format } = readManifest([{ dir }]); - manifest.version = await getNewManifestVersion({ dir, ethProvider }); - - console.log( - `New manifest version for ${manifest.name}: ${manifest.version}` - ); - - writeManifest(manifest, format, { dir }); - } catch (e) { - // Not throwing an error here because updating the manifest version is not critical - console.error(`Could not fetch new manifest version: ${e}`); - } - } -} - async function getNewManifestVersion({ ethProvider, dir diff --git a/src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts b/src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts index e1dd5487..6e2cdc0a 100644 --- a/src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts +++ b/src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts @@ -1,23 +1,43 @@ -import { Manifest, UpstreamItem } from "@dappnode/types"; import { readManifest, readCompose } from "../../../../files/index.js"; -import { arrIsUnique } from "../../../../utils/array.js"; import { getFirstAvailableEthProvider } from "../../../../utils/tryEthProviders.js"; -import { InitialSetupData, GitSettings, UpstreamSettings } from "../types.js"; +import { InitialSetupData, GitSettings } from "../types.js"; +import { resolvePackage } from "../../../../bump/plan.js"; +import { RegistryClient } from "../../../../bump/registry.js"; import { fetchGithubUpstreamVersion } from "../github/fetchGithubUpstreamVersion.js"; export async function getInitialSettings({ dir, userEthProvider, - useFallback + useFallback, + explicitVersion }: { dir: string; userEthProvider: string; useFallback: boolean; + explicitVersion?: string; }): Promise { const { manifest, format } = readManifest([{ dir }]); const compose = readCompose([{ dir }]); - const upstreamSettings = await parseUpstreamSettings(manifest); + const resolved = await resolvePackage( + dir, + { + latestRelease: fetchGithubUpstreamVersion, + registry: new RegistryClient() + }, + explicitVersion + ); + const upstreamSettings = resolved.length + ? resolved.map(item => ({ + repo: item.repo, + arg: item.arg, + provider: item.provider, + tagPattern: item.tagPattern, + manifestVersion: item.version, + githubVersion: item.targetVersion, + composeValue: item.composeValue + })) + : null; const gitSettings = getGitSettings(); @@ -39,66 +59,6 @@ export async function getInitialSettings({ }; } -/** - * Supports both legacy 'upstreamRepo' and 'upstreamArg' fields and current 'upstream' - * field (array of objects with 'repo', 'arg' and 'version' fields) - */ -async function parseUpstreamSettings( - manifest: Manifest -): Promise { - const upstreamSettings = manifest.upstream - ? await parseUpstreamSettingsNewFormat(manifest.upstream) - : await parseUpstreamSettingsLegacyFormat(manifest); - - if (!upstreamSettings || upstreamSettings.length < 1) return null; - - validateUpstreamSettings(upstreamSettings); - - return upstreamSettings; -} - -async function parseUpstreamSettingsNewFormat( - upstream: UpstreamItem[] -): Promise { - const upstreamPromises = upstream.map(async ({ repo, arg, version }) => { - const githubVersion = await fetchGithubUpstreamVersion(repo); - - if (githubVersion) - return { repo, arg, manifestVersion: version, githubVersion }; - }); - - const upstreamResults = await Promise.all(upstreamPromises); - return upstreamResults.filter( - item => item !== undefined - ) as UpstreamSettings[]; -} - -/** - * Legacy support for 'upstreamRepo' and 'upstreamArg' fields - * Currently, 'upstream' field is used instead, which is an array of objects with 'repo', 'arg' and 'version' fields - */ -async function parseUpstreamSettingsLegacyFormat( - manifest: Manifest -): Promise { - // 'upstreamRepo' and 'upstreamArg' being defined as arrays has been deprecated - - if (!manifest.upstreamRepo || manifest.upstreamRepo.trim() === "") - return null; - - const githubVersion = await fetchGithubUpstreamVersion(manifest.upstreamRepo); - - if (!githubVersion) return null; - - return [ - { - repo: manifest.upstreamRepo, - manifestVersion: manifest.upstreamVersion || "UPSTREAM_VERSION", - arg: manifest.upstreamArg || "UPSTREAM_VERSION", - githubVersion - } - ]; -} - function getEthProviders( useFallback: boolean, userEthProvider: string @@ -124,11 +84,3 @@ function getGitSettings(): GitSettings { userEmail }; } - -function validateUpstreamSettings(upstreamSettings: UpstreamSettings[]): void { - if (!arrIsUnique(upstreamSettings.map(s => s.repo))) - throw new Error("Upstream repositories must be unique"); - - if (!arrIsUnique(upstreamSettings.map(s => s.arg))) - throw new Error("Upstream args must be unique"); -} diff --git a/src/commands/githubActions/bumpUpstream/types.ts b/src/commands/githubActions/bumpUpstream/types.ts index f9b26ac6..62796da0 100644 --- a/src/commands/githubActions/bumpUpstream/types.ts +++ b/src/commands/githubActions/bumpUpstream/types.ts @@ -1,10 +1,12 @@ -import { UpstreamItem, Manifest, Compose } from "@dappnode/types"; +import { Manifest, Compose } from "@dappnode/types"; +import { BumpUpstreamItem } from "../../../bump/metadata.js"; import { ManifestFormat } from "../../../files/manifest/types.js"; import { Github } from "../../../providers/github/Github.js"; -export type UpstreamSettings = Omit & { +export type UpstreamSettings = Omit & { manifestVersion: string; githubVersion: string; + composeValue?: string; }; export interface GitSettings { diff --git a/src/providers/github/Github.ts b/src/providers/github/Github.ts index 3325c597..05b72b32 100644 --- a/src/providers/github/Github.ts +++ b/src/providers/github/Github.ts @@ -131,7 +131,32 @@ export class Github { } } - async listReleases() { + /** GitHub's Latest release, or null when the repository has none. */ + async getLatestRelease() { + try { + const { data } = await this.octokit.rest.repos.getLatestRelease({ + owner: this.owner, + repo: this.repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + } + }); + return data; + } catch (e) { + if (e instanceof RequestError && e.status === 404) return null; + throw e; + } + } + + async listReleases(firstPageSize?: number) { + if (firstPageSize !== undefined) { + const { data } = await this.octokit.rest.repos.listReleases({ + owner: this.owner, + repo: this.repo, + per_page: firstPageSize + }); + return data; + } return await this.octokit.paginate(this.octokit.rest.repos.listReleases, { owner: this.owner, repo: this.repo, diff --git a/src/spec/checkPackageSpec.ts b/src/spec/checkPackageSpec.ts index 65d0fcb9..8cb9fec2 100644 --- a/src/spec/checkPackageSpec.ts +++ b/src/spec/checkPackageSpec.ts @@ -1,3 +1,8 @@ +import { + BumpUpstreamItem, + compileTagPattern, + parseProvider +} from "../bump/metadata.js"; import fs from "fs"; import path from "path"; import yaml from "js-yaml"; @@ -17,8 +22,8 @@ const manifestFileName = "dappnode_package.json"; const packageNamePattern = /^[a-z0-9]+(-[a-z0-9]+)*\.(dnp|public)\.dappnode\.eth$/; const variantNamePattern = /^[a-z0-9]+(-[a-z0-9]+)*$/; -/** Supported by DAPPMANAGER but not yet in the published manifest schema. */ -const knownUnreleasedManifestFields = ["upnpDisable"]; +/** Supported by consumers but not yet in the published manifest schema. */ +const knownUnreleasedManifestFields = ["upnpDisable", "upstreamProvider"]; /** Single release files that are copied (not merged) from the root or the variant. */ const singleCopiedReleaseFiles = [ "avatar", @@ -343,17 +348,23 @@ export function checkPackageSpec({ const sourceManifest = mergedSourceManifest(pkg) || manifest; const buildArgs = collectBuildArgs(compose); const usesUpstreamArray = Array.isArray(sourceManifest.upstream); - const upstreams = usesUpstreamArray - ? sourceManifest.upstream || [] - : sourceManifest.upstreamRepo - ? [ - { - repo: sourceManifest.upstreamRepo, - version: sourceManifest.upstreamVersion, - arg: sourceManifest.upstreamArg - } - ] - : []; + const upstreams: (Partial & { repo: string })[] = + usesUpstreamArray + ? sourceManifest.upstream || [] + : sourceManifest.upstreamRepo + ? [ + { + repo: sourceManifest.upstreamRepo, + version: sourceManifest.upstreamVersion, + arg: sourceManifest.upstreamArg, + provider: ( + sourceManifest as Partial & { + upstreamProvider?: string; + } + ).upstreamProvider + } + ] + : []; const upstreamDir = manifestOrigin(m => usesUpstreamArray ? "upstream" in m @@ -361,6 +372,20 @@ export function checkPackageSpec({ ); for (const item of upstreams) { if (!item?.repo) continue; + for (const validate of [ + () => parseProvider(item.provider), + () => compileTagPattern(item.tagPattern) + ]) { + try { + validate(); + } catch (error) { + add( + "SPEC-M12", + `Upstream ${item.repo}${where(upstreamDir)}: ${error.message}`, + manifestFileIn(upstreamDir) + ); + } + } // The legacy fields default the arg to UPSTREAM_VERSION; check the version against it too. const upstream = { ...item, arg: item.arg || "UPSTREAM_VERSION" }; if (!item.arg) { @@ -394,7 +419,7 @@ export function checkPackageSpec({ return ( value === String(upstream.version) || (value.startsWith(prefix) && - /^[a-f0-9]{64}$/i.test(value.slice(prefix.length))) + /^[a-f0-9]{64}$/.test(value.slice(prefix.length))) ); }; if (upstream.version !== undefined) @@ -407,7 +432,7 @@ export function checkPackageSpec({ const variant = where(upstreamDir) || where(argDir); add( "SPEC-M05", - `Upstream ${upstream.repo}${variant} has version "${upstream.version}" but services.${service}.build.args.${upstream.arg} is "${value}"; every service using this arg must match (optionally with @sha256:<64 hex digits>).`, + `Upstream ${upstream.repo}${variant} has version "${upstream.version}" but services.${service}.build.args.${upstream.arg} is "${value}"; every service using this arg must match (optionally with @sha256:<64 lowercase hex digits>).`, composeFileIn(argDir) ); } diff --git a/src/spec/rules.ts b/src/spec/rules.ts index 80a623d8..97f62d9e 100644 --- a/src/spec/rules.ts +++ b/src/spec/rules.ts @@ -25,7 +25,7 @@ export const specRules = { }, "SPEC-M05": { title: "Upstream version equals the build arg value", - why: "Every service using an upstream arg must match its declared version, optionally pinned by a complete SHA-256 digest." + why: "Every service using an upstream arg must match its declared version, optionally pinned by a complete lowercase SHA-256 digest." }, "SPEC-M06": { title: "Variant upstream[] matches the root order", @@ -51,6 +51,10 @@ export const specRules = { title: "Global environment names match the runtime catalogue", why: "DAPPMANAGER prefixes short global-env names at install; unknown or already-prefixed names cannot be injected." }, + "SPEC-M12": { + title: "Upstream bump metadata is valid", + why: "Providers must be recognised and tag patterns must compile in the common JavaScript/Rust regex subset." + }, // Templates (multi-variant packages) "SPEC-T01": { title: "Template root manifest has no name or version", diff --git a/test/bump/compareEngines.ts b/test/bump/compareEngines.ts new file mode 100644 index 00000000..c74c4967 --- /dev/null +++ b/test/bump/compareEngines.ts @@ -0,0 +1,155 @@ +/** Manual network check: node --import=tsx test/bump/compareEngines.ts --workspace [--version ] */ +import "dotenv/config"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { parseArgs } from "node:util"; +import assert from "node:assert/strict"; +import semver from "semver"; +import { packageManifestDirs } from "../../src/bump/discovery.js"; +import { editManifest, writePlan } from "../../src/bump/edits.js"; +import { planPackageEdits, resolvePackage } from "../../src/bump/plan.js"; +import { RegistryClient } from "../../src/bump/registry.js"; +import { selectLatestRelease } from "../../src/commands/githubActions/bumpUpstream/github/pickLatestRelease.js"; + +const { values } = parseArgs({ + options: { + workspace: { type: "string" }, + version: { type: "string" } + } +}); +if (!values.workspace) throw Error("--workspace is required"); +const original = path.resolve(values.workspace); +const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-bump-compare-")); +const sdk = path.join(scratch, "sdk"); +const reference = path.join(scratch, "reference"); +for (const destination of [sdk, reference]) + fs.cpSync(original, destination, { recursive: true }); +const dependencies = { + registry: new RegistryClient(), + latestRelease: async (repo: string, pattern?: string) => { + const headers: Record = { + Accept: "application/vnd.github+json" + }; + if (process.env.GITHUB_TOKEN) + headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`; + const get = async (url: string) => { + const response = await fetch( + `https://api.github.com/repos/${repo}/${url}`, + { + headers + } + ); + if (response.status === 404) return null; + if (response.status !== 200) + throw Error(`GitHub release lookup returned ${response.status}`); + return response.json(); + }; + return selectLatestRelease( + await get("releases/latest"), + async () => (await get("releases?per_page=30")) || [], + pattern + ); + } +}; +const resolved = await resolvePackage(sdk, dependencies, values.version); +const plan = await planPackageEdits({ + dir: sdk, + upstreams: resolved, + // A deterministic stand-in for the APM lookup; only this extra package bump + // is reversed below before comparing upstream edits byte for byte. + nextPackageVersion: async dir => { + const manifest = JSON.parse( + fs.readFileSync(path.join(dir, "dappnode_package.json"), "utf8") + ); + const next = semver.inc(manifest.version, "patch"); + if (!next) throw Error("Invalid package version"); + return next; + } +}); +writePlan(plan); +const dockerArgs = [ + "run", + "--rm", + "-v", + `${reference}:/workspace`, + "-w", + "/workspace", + "-e", + "GITHUB_TOKEN", + "docker.io/dappnode/tropibot:latest", + "bump-runner", + "update", + "--format", + "json" +]; +if (values.version) dockerArgs.push("--version", values.version); +const referenceOutput = execFileSync("docker", dockerArgs, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: 180000 +}); +fs.writeFileSync(path.join(scratch, "reference.json"), referenceOutput); +const packageManifests = packageManifestDirs(sdk).map(dir => + path.relative(sdk, path.join(dir, "dappnode_package.json")) +); +const files = [ + ...new Set([ + "dappnode_package.json", + "docker-compose.yml", + ...packageManifests + ]) +]; +for (const file of files) { + let actual = fs.readFileSync(path.join(sdk, file), "utf8"); + const expected = fs.readFileSync(path.join(reference, file), "utf8"); + if (packageManifests.includes(file) && plan.length) { + const originalVersion = JSON.parse( + fs.readFileSync(path.join(original, file), "utf8") + ).version; + assert.equal( + JSON.parse(actual).version, + semver.inc(originalVersion, "patch") + ); + actual = editManifest(actual, [ + { path: ["version"], value: originalVersion } + ]); + } + assert.equal(actual, expected, `Byte mismatch: ${file}; inspect ${scratch}`); +} +const mtimes = files.map(file => fs.statSync(path.join(sdk, file)).mtimeMs); +const rerun = await resolvePackage(sdk, dependencies, values.version); +const rerunPlan = await planPackageEdits({ + dir: sdk, + upstreams: rerun, + nextPackageVersion: async () => { + throw Error("Idempotent rerun must not bump package"); + } +}); +assert.deepEqual(rerunPlan, []); +assert.deepEqual( + files.map(file => fs.statSync(path.join(sdk, file)).mtimeMs), + mtimes +); +console.log( + JSON.stringify( + { + workspace: original, + request: values.version || "latest", + results: resolved.map(item => ({ + repo: item.repo, + previous: item.version, + version: item.targetVersion, + compose: item.composeValue + })), + compared: files, + packageVersionBumps: plan.length ? packageManifests.length : 0, + byteIdentical: true, + idempotent: true, + scratch + }, + null, + 2 + ) +); diff --git a/test/bump/deprecation.test.ts b/test/bump/deprecation.test.ts new file mode 100644 index 00000000..157fc20c --- /dev/null +++ b/test/bump/deprecation.test.ts @@ -0,0 +1,36 @@ +import { expect } from "chai"; +import { printBumpDeprecation } from "../../src/commands/githubActions/bumpUpstream/deprecation.js"; +import { getBumpPrBody } from "../../src/commands/githubActions/bumpUpstream/github/getBumpPrBody.js"; + +describe("bump migration and PR metadata", () => { + it("logs the deprecation and an Actions warning with both migration paths", () => { + const lines: string[] = []; + printBumpDeprecation(line => lines.push(line)); + expect(lines).to.have.length(2); + expect(lines[1]).to.equal( + `::warning title=dappnodesdk bump-upstream is deprecated::${lines[0]}` + ); + expect(lines[0]) + .to.include( + "dappnode/workflows/.github/workflows/bump-upstream.yml@master" + ) + .and.to.include("dappnode/workflows/actions/bump-upstream@master"); + }); + + it("includes Explorer metadata and the exact Compose pin", () => { + const value = `v2.0.0@sha256:${"a".repeat(64)}`; + const body = getBumpPrBody([ + { + repo: "acme/app", + arg: "VERSION", + manifestVersion: "v1.0.0", + githubVersion: "v2.0.0", + composeValue: value + } + ]); + expect(body).to.include(`Compose: \`${value}\``); + expect(body.split("\n")).to.include( + '' + ); + }); +}); diff --git a/test/bump/discovery.test.ts b/test/bump/discovery.test.ts new file mode 100644 index 00000000..0e1058cb --- /dev/null +++ b/test/bump/discovery.test.ts @@ -0,0 +1,134 @@ +import { expect } from "chai"; +import fs from "node:fs"; +import path from "node:path"; +import { + discoverUpstreams, + packageManifestDirs, + BumpCompose +} from "../../src/bump/discovery.js"; +import { + compileTagPattern, + parseProvider, + extractVersion +} from "../../src/bump/metadata.js"; +import { temporaryDirectory } from "../helpers/fixtures.js"; + +const compose: BumpCompose = { + services: { app: { build: { args: { APP_TAG: "1.0.0" } } } } +}; + +describe("bump discovery and metadata", () => { + it("discovers array and legacy fields, inferring a single arg", () => { + expect( + discoverUpstreams( + { upstream: [{ repo: "acme/app", version: "1.0.0" }] }, + compose + ) + ).to.deep.equal([ + { repo: "acme/app", version: "1.0.0", arg: "APP_TAG", provider: "docker" } + ]); + expect( + discoverUpstreams( + { + upstreamRepo: "acme/app", + upstreamVersion: "1.0.0", + upstreamProvider: "GITHUB" + }, + compose + )[0] + ).to.include({ arg: "APP_TAG", provider: "github" }); + expect( + discoverUpstreams( + { upstreamRepo: "acme/app", upstreamVersion: "1.0.0" }, + { services: {} } + )[0].arg + ).to.equal("UPSTREAM_VERSION"); + expect(discoverUpstreams({}, compose)).to.deep.equal([]); + }); + + it("requires explicit multi-upstream args, and unique repos and args", () => { + const upstream = [ + { repo: "acme/a", version: "1", arg: "A" }, + { repo: "acme/b", version: "2", arg: "B" } + ]; + expect(discoverUpstreams({ upstream }, compose)).to.have.length(2); + expect(() => + discoverUpstreams( + { upstream: [upstream[0], { ...upstream[1], arg: undefined }] }, + compose + ) + ).to.throw("multi-upstream"); + expect(() => + discoverUpstreams( + { upstream: [upstream[0], { ...upstream[1], repo: "acme/a" }] }, + compose + ) + ).to.throw("repositories must be unique"); + expect(() => + discoverUpstreams( + { upstream: [upstream[0], { ...upstream[1], arg: "A" }] }, + compose + ) + ).to.throw("args must be unique"); + expect(() => + discoverUpstreams( + { upstreamRepo: "acme/app", upstreamVersion: "1" }, + { services: { app: { build: { args: ["A_VERSION=1", "B_TAG=2"] } } } } + ) + ).to.throw("uniquely detect"); + }); + + it("normalizes providers and rejects invalid regex constructs", () => { + for (const name of [ + "docker_tags", + "DOCKER_TAGS", + "docker-tags", + "dockerhub" + ]) + expect(parseProvider(name)).to.equal("docker_tags"); + expect(parseProvider(undefined)).to.equal("docker"); + expect(() => parseProvider("npm")).to.throw("Unknown upstream provider"); + for (const pattern of [ + "[", + "(?=a)", + "(?<=a)b", + "(a)\\1", + "(?a)\\k" + ]) + expect(() => compileTagPattern(pattern), pattern).to.throw(); + expect(extractVersion("op-node/v1.2.3", "^op-node/(v.*)$")).to.equal( + "v1.2.3" + ); + expect(extractVersion("v1.2.3", "v.*")).to.equal("v1.2.3"); + expect(extractVersion("other/v1", "^node/(.*)$")).to.equal(null); + expect(compileTagPattern(String.raw`^a\\1$`)).to.be.instanceOf(RegExp); + }); + + it("detects variant manifests automatically and supports legacy directories", () => { + const dir = temporaryDirectory(); + try { + expect(packageManifestDirs(dir)).to.deep.equal([dir]); + for (const variant of ["mainnet", "testnet"]) { + const base = path.join(dir, "package_variants", variant); + fs.mkdirSync(base, { recursive: true }); + fs.writeFileSync(path.join(base, "dappnode_package.json"), "{}"); + } + expect( + packageManifestDirs(dir).map(file => path.basename(file)) + ).to.deep.equal(["mainnet", "testnet"]); + fs.renameSync( + path.join(dir, "package_variants"), + path.join(dir, "legacy") + ); + expect(packageManifestDirs(dir, "legacy")).to.have.length(2); + fs.unlinkSync( + path.join(dir, "legacy", "mainnet", "dappnode_package.json") + ); + expect(() => packageManifestDirs(dir, "legacy")).to.throw( + "Variant has no manifest" + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/bump/edits.test.ts b/test/bump/edits.test.ts new file mode 100644 index 00000000..d7825a55 --- /dev/null +++ b/test/bump/edits.test.ts @@ -0,0 +1,292 @@ +import { expect } from "chai"; +import fs from "node:fs"; +import path from "node:path"; +import { + editCompose, + editManifest, + validateEdits, + writePlan +} from "../../src/bump/edits.js"; +import { planPackageEdits, resolvePackage } from "../../src/bump/plan.js"; +import { + ResolvedUpstream, + ResolutionDependencies +} from "../../src/bump/resolve.js"; +import { temporaryDirectory } from "../helpers/fixtures.js"; + +const upstream: ResolvedUpstream = { + repo: "acme/app", + arg: "VERSION", + provider: "github", + version: "v1.0.0", + targetVersion: "v2.0.0", + composeValue: "v2.0.0" +}; +const compose = + "# header\nservices:\n app:\n build:\n args: {VERSION: \"v1.0.0\", STATIC: keep} # comment\n worker:\n build:\n args:\n - 'VERSION=v1.0.0' # worker\n - OTHER=keep\n"; + +describe("bump targeted edits and planning", () => { + it("preserves mapping/list args, quotes, comments, flow styles, CRLF and unrelated bytes", () => { + for (const before of [compose, compose.replace(/\n/g, "\r\n")]) { + const after = editCompose(before, new Map([["VERSION", "v2.0.0"]])); + expect(after).to.equal(before.replace(/v1\.0\.0/g, "v2.0.0")); + expect(editCompose(after, new Map([["VERSION", "v2.0.0"]]))).to.equal( + after + ); + } + const flow = "services: {app: {build: {args: [VERSION=v1.0.0, X=keep]}}}\n"; + expect(editCompose(flow, new Map([["VERSION", "v2.0.0"]]))).to.equal( + flow.replace("v1.0.0", "v2.0.0") + ); + }); + + it("preserves plain scalars and quotes newly ambiguous strings", () => { + const plain = + "services:\n app:\n build:\n args:\n VERSION: v1.0.0\n"; + expect(editCompose(plain, new Map([["VERSION", "true"]]))).to.equal( + plain.replace("v1.0.0", '"true"') + ); + expect(editCompose(plain, new Map([["VERSION", "v2.0.0"]]))).to.equal( + plain.replace("v1.0.0", "v2.0.0") + ); + }); + + it("preserves JSON key order, indentation, spacing and trailing newline", () => { + for (const newline of ["", "\n", "\r\n"]) { + const before = + '{\n\t"upstream": [{"repo":"acme/app", "version": "v1.0.0"}],\n\t"version":"0.1.0",\n\t"name":"app"\n}' + + newline; + expect( + editManifest(before, [ + { path: ["upstream", 0, "version"], value: "v2.0.0" }, + { path: ["version"], value: "0.1.1" } + ]) + ).to.equal(before.replace("v1.0.0", "v2.0.0").replace("0.1.0", "0.1.1")); + } + const legacy = '{"upstreamVersion":"v1.0.0","version":"0.1.0"}\n'; + expect( + editManifest(legacy, [{ path: ["upstreamVersion"], value: "v2.0.0" }]) + ).to.equal(legacy.replace("v1.0.0", "v2.0.0")); + }); + + it("rejects unintended edits, incomplete updates, duplicate keys and aliases", () => { + const changes = [ + { path: ["services", "app", "build", "args", "VERSION"], value: "v2.0.0" } + ]; + expect(() => + validateEdits( + compose, + compose.replace("STATIC: keep", "STATIC: bad"), + changes + ) + ).to.throw("outside the intended"); + expect(() => validateEdits(compose, compose, changes)).to.throw( + "outside the intended" + ); + expect(() => + editCompose( + "services: {app: &build {build: {args: {VERSION: v1}}}, worker: *build}", + new Map([["VERSION", "v2"]]) + ) + ).to.throw("Aliases"); + expect(() => + editCompose( + "services: {app: {build: {args: {VERSION: v1, VERSION: v2}}}}", + new Map([["VERSION", "v3"]]) + ) + ).to.throw(); + }); + + it("plans root upstream edits and bumps every variant, with idempotent re-runs", async () => { + const dir = temporaryDirectory(); + try { + fs.writeFileSync( + path.join(dir, "dappnode_package.json"), + JSON.stringify( + { + upstream: [ + { + repo: "acme/app", + version: "v1.0.0", + arg: "VERSION", + provider: "github" + } + ] + }, + null, + 2 + ) + "\n" + ); + fs.writeFileSync(path.join(dir, "docker-compose.yml"), compose); + for (const variant of ["mainnet", "testnet"]) { + const base = path.join(dir, "package_variants", variant); + fs.mkdirSync(base, { recursive: true }); + fs.writeFileSync( + path.join(base, "dappnode_package.json"), + '{"name":"app","version":"0.1.0"}\n' + ); + } + const plan = await planPackageEdits({ + dir, + upstreams: [upstream], + nextPackageVersion: async () => "0.1.1" + }); + expect(plan).to.have.length(4); + expect( + fs.readFileSync(path.join(dir, "docker-compose.yml"), "utf8") + ).to.equal(compose); + writePlan(plan); + expect( + JSON.parse( + fs.readFileSync(path.join(dir, "dappnode_package.json"), "utf8") + ) + ).not.to.have.property("version"); + for (const variant of ["mainnet", "testnet"]) + expect( + fs.readFileSync( + path.join( + dir, + "package_variants", + variant, + "dappnode_package.json" + ), + "utf8" + ) + ).to.equal('{"name":"app","version":"0.1.1"}\n'); + const dependencies: ResolutionDependencies = { + latestRelease: async () => "v2.0.0", + registry: { + tagExists: async () => { + throw Error("registry must not be called"); + }, + manifestDigest: async () => { + throw Error("registry must not be called"); + }, + latestTag: async () => null + } + }; + const resolved = await resolvePackage(dir, dependencies); + const beforeStats = plan.map(file => fs.statSync(file.path).mtimeMs); + const again = await planPackageEdits({ + dir, + upstreams: resolved, + nextPackageVersion: async () => { + throw Error("must not bump twice"); + } + }); + writePlan(again); + expect(again).to.deep.equal([]); + expect(plan.map(file => fs.statSync(file.path).mtimeMs)).to.deep.equal( + beforeStats + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("leaves all files unchanged if a later package-version lookup fails", async () => { + const dir = temporaryDirectory(); + try { + const before = JSON.stringify({ + version: "0.1.0", + upstreamRepo: "acme/app", + upstreamVersion: "v1.0.0" + }); + const file = path.join(dir, "dappnode_package.json"); + fs.writeFileSync(file, before); + fs.writeFileSync(path.join(dir, "docker-compose.yml"), compose); + for (const variant of ["first", "second"]) { + const base = path.join(dir, "package_variants", variant); + fs.mkdirSync(base, { recursive: true }); + fs.writeFileSync( + path.join(base, "dappnode_package.json"), + '{"version":"0.1.0"}\n' + ); + } + let lookups = 0; + let failure: Error | undefined; + try { + await planPackageEdits({ + dir, + upstreams: [upstream], + nextPackageVersion: async () => { + if (++lookups === 1) return "0.1.1"; + throw Error("APM unavailable"); + } + }); + } catch (error) { + failure = error; + } + expect(failure?.message).to.equal("APM unavailable"); + expect(lookups).to.equal(2); + for (const variant of ["first", "second"]) + expect( + fs.readFileSync( + path.join( + dir, + "package_variants", + variant, + "dappnode_package.json" + ), + "utf8" + ) + ).to.equal('{"version":"0.1.0"}\n'); + expect(fs.readFileSync(file, "utf8")).to.equal(before); + expect( + fs.readFileSync(path.join(dir, "docker-compose.yml"), "utf8") + ).to.equal(compose); + expect(() => + writePlan([{ path: file, before: "stale", after: "bad" }]) + ).to.throw("File changed"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("leaves the checkout untouched if digest resolution fails", async () => { + const dir = temporaryDirectory(); + const digest = `sha256:${"a".repeat(64)}`; + const beforeManifest = JSON.stringify({ + name: "app.dnp.dappnode.eth", + version: "0.1.0", + upstream: [{ repo: "acme/app", version: "v1.0.0", arg: "VERSION" }] + }); + const beforeCompose = compose.replace(/v1\.0\.0/g, `v1.0.0@${digest}`); + try { + fs.writeFileSync(path.join(dir, "dappnode_package.json"), beforeManifest); + fs.writeFileSync(path.join(dir, "docker-compose.yml"), beforeCompose); + let failure: Error | undefined; + try { + const upstreams = await resolvePackage(dir, { + latestRelease: async () => "v2.0.0", + imageSource: () => ({ image: "org/app", template: "{version}" }), + registry: { + tagExists: async () => true, + latestTag: async () => null, + manifestDigest: async () => { + throw Error("Registry pinning unavailable"); + } + } + }); + writePlan( + await planPackageEdits({ + dir, + upstreams, + nextPackageVersion: async () => "0.1.1" + }) + ); + } catch (error) { + failure = error; + } + expect(failure?.message).to.equal("Registry pinning unavailable"); + expect( + fs.readFileSync(path.join(dir, "dappnode_package.json"), "utf8") + ).to.equal(beforeManifest); + expect( + fs.readFileSync(path.join(dir, "docker-compose.yml"), "utf8") + ).to.equal(beforeCompose); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/bump/registry.test.ts b/test/bump/registry.test.ts new file mode 100644 index 00000000..8ff0d75a --- /dev/null +++ b/test/bump/registry.test.ts @@ -0,0 +1,198 @@ +import { expect } from "chai"; +import { + createServer, + Server, + IncomingMessage, + ServerResponse +} from "node:http"; +import { AddressInfo } from "node:net"; +import { createHash } from "node:crypto"; +import { + manifestAccept, + RegistryClient, + registryImage +} from "../../src/bump/registry.js"; + +const digest = `sha256:${"b".repeat(64)}`; + +describe("bump anonymous registry client", () => { + let server: Server; + let host: string; + let handler: (request: IncomingMessage, response: ServerResponse) => void; + const registry = new RegistryClient(); + beforeEach(async () => { + server = createServer((request, response) => handler(request, response)); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + host = `127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + afterEach(async () => { + if (!server.listening) return; + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ); + }); + async function rejects( + task: Promise, + message: string + ): Promise { + try { + await task; + } catch (error) { + expect(error.message).to.include(message); + return; + } + throw Error(`Expected rejection: ${message}`); + } + + it("normalizes Docker Hub and explicit registry names", () => { + for (const image of [ + "alpine", + "docker.io/alpine", + "index.docker.io/library/alpine" + ]) + expect(registryImage(image)).to.deep.equal({ + host: "registry-1.docker.io", + name: "library/alpine", + hub: true + }); + expect(registryImage("org/app").name).to.equal("org/app"); + expect(registryImage("ghcr.io/org/app")).to.deep.equal({ + host: "ghcr.io", + name: "org/app", + hub: false + }); + }); + + it("answers a bearer challenge, using token or access_token and pull scope", async () => { + for (const tokenKey of ["token", "access_token"]) { + const requests: string[] = []; + handler = (request, response) => { + requests.push(`${request.method} ${request.url}`); + if (request.url?.startsWith("/token")) { + const url = new URL(request.url, `http://${host}`); + expect(url.searchParams.get("service")).to.equal("registry"); + expect(url.searchParams.get("scope")).to.equal( + "repository:org/app:pull" + ); + response.end(JSON.stringify({ [tokenKey]: "anonymous" })); + } else if (request.headers.authorization !== "Bearer anonymous") { + response + .writeHead(401, { + "WWW-Authenticate": `Bearer realm="http://${host}/token",service="registry",scope="ignored"` + }) + .end(); + } else { + expect(request.method).to.equal("HEAD"); + expect(request.headers.accept).to.equal(manifestAccept); + response.writeHead(200, { "Docker-Content-Digest": digest }).end(); + } + }; + expect( + await registry.manifestDigest(`${host}/org/app`, "v1.0.0") + ).to.equal(digest); + expect(requests).to.have.length(3); + } + }); + + it("hashes GET bytes when the HEAD digest is absent", async () => { + const bytes = Buffer.from( + '{"mediaType":"application/vnd.oci.image.index.v1+json"}\n' + ); + const methods: string[] = []; + handler = (request, response) => { + methods.push(request.method!); + response.end(request.method === "HEAD" ? undefined : bytes); + }; + expect(await registry.manifestDigest(`${host}/org/app`, "1.0.0")).to.equal( + `sha256:${createHash("sha256").update(bytes).digest("hex")}` + ); + expect(methods).to.deep.equal(["HEAD", "GET"]); + }); + + it("distinguishes missing tags from errors and rejects invalid digest headers", async () => { + handler = (request, response) => + response.writeHead(request.url?.endsWith("missing") ? 404 : 500).end(); + expect(await registry.tagExists(`${host}/org/app`, "missing")).to.equal( + false + ); + await rejects(registry.tagExists(`${host}/org/app`, "error"), "500"); + await rejects(registry.manifestDigest(`${host}/org/app`, "missing"), "404"); + handler = (request, response) => + response + .writeHead(200, { "Docker-Content-Digest": `sha256:${"B".repeat(64)}` }) + .end(); + await rejects( + registry.manifestDigest(`${host}/org/app`, "v1"), + "malformed digest" + ); + }); + + it("follows OCI Link pagination and picks the highest stable tag", async () => { + const paths: string[] = []; + handler = (request, response) => { + paths.push(request.url!); + response.setHeader("Content-Type", "application/json"); + if (request.url === "/v2/org/app/tags/list") { + response.setHeader( + "Link", + '; rel="next"' + ); + response.end(JSON.stringify({ tags: ["v2.0.0", "3.0.0-rc.1"] })); + } else response.end(JSON.stringify({ tags: ["2.0.0", "1.0.0"] })); + }; + expect(await registry.latestTag(`${host}/org/app`)).to.equal("2.0.0"); + expect(paths).to.have.length(2); + }); + + it("follows Docker Hub next pages and selects semver instead of update order", async () => { + const urls: string[] = []; + const client = new RegistryClient(async input => { + const url = String(input); + urls.push(url); + return Response.json( + urls.length === 1 + ? { + results: [{ name: "v1.0.0" }, { name: "9.0.0-rc.1" }], + next: "https://registry.hub.docker.com/next" + } + : { results: [{ name: "2.0.0" }, { name: "v2.0.0" }], next: null } + ); + }); + expect(await client.latestTag("alpine")).to.equal("2.0.0"); + expect(urls[0]).to.equal( + "https://registry.hub.docker.com/v2/repositories/library/alpine/tags?page_size=100&ordering=last_updated" + ); + }); + + it("caps pagination and retries bearer authentication only once", async () => { + let pages = 0; + handler = (request, response) => { + pages++; + response.setHeader( + "Link", + `; rel=next` + ); + response.end('{"tags":[]}'); + }; + await rejects(registry.listTags(`${host}/org/app`), "exceeded 50 pages"); + expect(pages).to.equal(50); + let challenges = 0; + handler = (request, response) => { + if (request.url?.startsWith("/token")) response.end('{"token":"denied"}'); + else { + challenges++; + response + .writeHead(401, { + "WWW-Authenticate": `Bearer realm="http://${host}/token"` + }) + .end(); + } + }; + await rejects(registry.tagExists(`${host}/org/app`, "v1"), "401"); + expect(challenges).to.equal(2); + }); +}); diff --git a/test/bump/resolve.test.ts b/test/bump/resolve.test.ts new file mode 100644 index 00000000..d1e2bc56 --- /dev/null +++ b/test/bump/resolve.test.ts @@ -0,0 +1,208 @@ +import { expect } from "chai"; +import { BumpCompose, Upstream } from "../../src/bump/discovery.js"; +import { + currentPin, + ResolutionDependencies, + resolveUpstream, + splitPin +} from "../../src/bump/resolve.js"; + +const digest = `sha256:${"a".repeat(64)}`; +const upstream: Upstream = { + repo: "acme/app", + arg: "VERSION", + version: "1.0.0", + provider: "docker" +}; +const compose = (value = "1.0.0"): BumpCompose => ({ + services: { app: { build: { args: { VERSION: value } } } } +}); +async function rejects(task: Promise, message: string): Promise { + try { + await task; + } catch (error) { + expect(error.message).to.include(message); + return; + } + throw Error(`Expected rejection: ${message}`); +} +function dependencies( + tag: string | null +): ResolutionDependencies & { calls: string[] } { + const calls: string[] = []; + return { + calls, + latestRelease: async () => tag, + imageSource: () => ({ image: "org/app", template: "multiarch-{version}" }), + registry: { + tagExists: async (_, tag) => { + calls.push(tag); + return !tag.includes("-v"); + }, + manifestDigest: async (_, tag) => { + calls.push(`digest:${tag}`); + return digest; + }, + latestTag: async () => tag + } + }; +} + +describe("bump resolution and pinning", () => { + it("splits only the last @sha256 suffix and enforces lowercase digests", () => { + expect(splitPin(`n8n@2.10.3@${digest}`)).to.deep.equal({ + version: "n8n@2.10.3", + digest + }); + expect(splitPin("n8n@2.10.3")).to.deep.equal({ version: "n8n@2.10.3" }); + for (const value of [ + `1@sha256:${"A".repeat(64)}`, + "1@sha256:short", + `@${digest}` + ]) + expect(() => splitPin(value)).to.throw("Invalid digest pin"); + }); + + it("maps candidate tags and writes the candidate rather than its rendered template", async () => { + const deps = dependencies("v2.0.0"); + expect(await resolveUpstream(".", compose(), upstream, deps)).to.include({ + targetVersion: "2.0.0", + composeValue: "2.0.0" + }); + expect(deps.calls).to.deep.equal(["multiarch-v2.0.0", "multiarch-2.0.0"]); + }); + + it("preserves plain and pinned styles and resolves the new image digest", async () => { + const deps = dependencies("v2.0.0"); + expect( + await resolveUpstream(".", compose(`1.0.0@${digest}`), upstream, deps) + ).to.include({ targetVersion: "2.0.0", composeValue: `2.0.0@${digest}` }); + expect(deps.calls.at(-1)).to.equal("digest:multiarch-2.0.0"); + const plain = dependencies("v2.0.0"); + await resolveUpstream(".", compose(), upstream, plain); + expect(plain.calls.some(call => call.startsWith("digest:"))).to.equal( + false + ); + }); + + it("short-circuits already-current and older releases before registry or Dockerfile access", async () => { + for (const tag of ["1.0.0", "v1.0.0", "v0.9.0"]) { + const deps = dependencies(tag); + deps.imageSource = () => { + throw Error("must not read image"); + }; + expect(await resolveUpstream(".", compose(), upstream, deps)).to.include({ + targetVersion: "1.0.0", + composeValue: "1.0.0" + }); + expect(deps.calls).to.deep.equal([]); + } + }); + + it("uses github versions as-is and supports args outside FROM without registry calls", async () => { + const deps = dependencies("v2.0.0"); + expect( + await resolveUpstream( + ".", + compose(), + { ...upstream, provider: "github" }, + deps + ) + ).to.include({ targetVersion: "v2.0.0" }); + deps.imageSource = () => null; + expect(await resolveUpstream(".", compose(), upstream, deps)).to.include({ + targetVersion: "v2.0.0" + }); + expect(deps.calls).to.deep.equal([]); + await rejects( + resolveUpstream(".", compose(`1.0.0@${digest}`), upstream, deps), + "needs a FROM image" + ); + }); + + it("extracts docker_tags versions and does not downgrade them", async () => { + const deps = dependencies("v2.0.0"); + expect( + await resolveUpstream( + ".", + compose(), + { ...upstream, provider: "docker_tags", tagPattern: "^v(.*)$" }, + deps + ) + ).to.include({ targetVersion: "2.0.0" }); + expect(deps.calls).to.deep.equal([]); + expect( + await resolveUpstream( + ".", + compose("3.0.0"), + { ...upstream, version: "3.0.0", provider: "docker_tags" }, + deps + ) + ).to.include({ targetVersion: "3.0.0" }); + await rejects( + resolveUpstream( + ".", + compose(), + { ...upstream, provider: "docker_tags", tagPattern: "^other/(.*)$" }, + deps + ), + "does not match tagPattern" + ); + }); + + it("allows explicit downgrades and skips release lookup", async () => { + const deps = dependencies(null); + deps.latestRelease = async () => { + throw Error("must not query releases"); + }; + expect( + await resolveUpstream(".", compose(), upstream, deps, "v0.9.0") + ).to.include({ targetVersion: "0.9.0" }); + }); + + it("rejects inconsistent versions, pinning styles, digests and missing args", () => { + const mixed = (a: string, b: string): BumpCompose => ({ + services: { + a: { build: { args: { VERSION: a } } }, + b: { build: { args: [`VERSION=${b}`] } } + } + }); + expect(() => currentPin(mixed("1.0.0", "0.9.0"), upstream)).to.throw( + "Inconsistent Compose pin" + ); + expect(() => + currentPin(mixed("1.0.0", `1.0.0@${digest}`), upstream) + ).to.throw("Inconsistent digest pins"); + expect(() => + currentPin( + mixed(`1.0.0@${digest}`, `1.0.0@sha256:${"b".repeat(64)}`), + upstream + ) + ).to.throw("Inconsistent digest pins"); + expect(() => currentPin({ services: {} }, upstream)).to.throw("is absent"); + }); + + it("fails on absent candidates and registry errors", async () => { + const deps = dependencies("v2.0.0"); + deps.registry.tagExists = async () => false; + await rejects( + resolveUpstream(".", compose(), upstream, deps), + "No Docker tag candidate exists" + ); + deps.registry.tagExists = async () => { + throw Error("registry 500"); + }; + await rejects( + resolveUpstream(".", compose(), upstream, deps), + "registry 500" + ); + deps.registry.tagExists = async () => true; + deps.registry.manifestDigest = async () => { + throw Error("pinning failed"); + }; + await rejects( + resolveUpstream(".", compose(`1.0.0@${digest}`), upstream, deps), + "pinning failed" + ); + }); +}); diff --git a/test/bump/versions.test.ts b/test/bump/versions.test.ts new file mode 100644 index 00000000..627596ab --- /dev/null +++ b/test/bump/versions.test.ts @@ -0,0 +1,143 @@ +import { expect } from "chai"; +import fs from "node:fs"; +import path from "node:path"; +import { + extractImageSource, + findImageSource, + renderDockerTag +} from "../../src/bump/dockerfile.js"; +import { + highestSemverTag, + isOlderVersion, + versionCandidates +} from "../../src/bump/versions.js"; +import { pickLatestRelease } from "../../src/commands/githubActions/bumpUpstream/github/pickLatestRelease.js"; +import { temporaryDirectory } from "../helpers/fixtures.js"; + +const release = (tag_name: string, draft = false, prerelease = false) => ({ + tag_name, + draft, + prerelease +}); + +describe("bump version selection and Dockerfile templates", () => { + it("selects the first eligible release with optional tag extraction", () => { + const releases = [ + release("v9.0.0", true), + release("v8.0.0", false, true), + release("v7.0.0-NiGhTlY"), + release("n8n@6.0.0-rc.1"), + release("op-deployer/v0.9.0-allowlist"), + release("op-node/v1.2.3"), + release("op-node/v1.1.0") + ]; + expect(pickLatestRelease(releases, "^op-node/(v.*)$")).to.equal("v1.2.3"); + // Without a pattern another tag family wins: `-allowlist` is not a pre-release word. + expect(pickLatestRelease(releases)).to.equal( + "op-deployer/v0.9.0-allowlist" + ); + const warn = console.warn; + const warnings: string[] = []; + try { + console.warn = message => warnings.push(message); + expect(pickLatestRelease([release("release_2026")])).to.equal( + "release_2026" + ); + expect( + pickLatestRelease([release("bad tag"), release("stable")]) + ).to.equal("stable"); + } finally { + console.warn = warn; + } + expect(warnings).to.have.length(2); + expect(pickLatestRelease([release("v2.0.0-rc.1")])).to.equal(null); + expect(pickLatestRelease([release("nightly/v1.0.0")], "(v.*)$")).to.equal( + null + ); + }); + + it("tries the original tag before a bare semver suffix", () => { + expect(versionCandidates("v1.2.3")).to.deep.equal(["v1.2.3", "1.2.3"]); + expect(versionCandidates("n8n@2.10.3")).to.deep.equal([ + "n8n@2.10.3", + "2.10.3" + ]); + expect(versionCandidates("1.2.3")).to.deep.equal(["1.2.3"]); + expect(versionCandidates("stable")).to.deep.equal(["stable"]); + expect(isOlderVersion("v1.2.3", "n8n@2.10.3")).to.equal(true); + expect(isOlderVersion("stable", "v2.0.0")).to.equal(false); + }); + + it("prefers stable image tags, highest semver, and bare tags on ties", () => { + expect( + highestSemverTag(["latest", "3.0.0-rc.1", "v2.0.0", "1.9.0", "2.0.0"]) + ).to.equal("2.0.0"); + expect(highestSemverTag(["v1.0.0-rc.1", "1.0.0-rc.2"])).to.equal( + "1.0.0-rc.2" + ); + expect(highestSemverTag(["latest", "1.0", "n8n@1.2.3"])).to.equal(null); + expect(highestSemverTag(["1.0.0+build", "v0.9.0"])).to.equal("1.0.0+build"); + }); + + it("expands direct, indirect and static Dockerfile arguments", () => { + const source = extractImageSource( + "ARG BASE_IMAGE=statusim/nimbus-eth2\nARG ARCH=multiarch\nARG BASE_TAG=${ARCH}-${VERSION}\nFROM --platform=$TARGETPLATFORM ${BASE_IMAGE}:${BASE_TAG} AS base\n", + "VERSION" + ); + expect(source).to.deep.equal({ + image: "statusim/nimbus-eth2", + template: "multiarch-{version}" + }); + expect(renderDockerTag(source!, "v26.10.0")).to.equal("multiarch-v26.10.0"); + expect( + extractImageSource("FROM host:5000/org/app:$VERSION", "VERSION") + ).to.deep.equal({ image: "host:5000/org/app", template: "{version}" }); + expect( + extractImageSource("FROM app:${VERSION#v}", "VERSION") + ).to.deep.equal({ image: "app", template: "{version#v}" }); + expect( + renderDockerTag({ image: "app", template: "{version#v}" }, "v1.2.3") + ).to.equal("1.2.3"); + expect( + extractImageSource( + "FROM node:22\nRUN npm install app@${VERSION}", + "VERSION" + ) + ).to.equal(null); + expect(extractImageSource("FROM app:$VERSION_EXTRA", "VERSION")).to.equal( + null + ); + }); + + it("finds the service context and Dockerfile with mapping or list args", () => { + const dir = temporaryDirectory(); + try { + fs.mkdirSync(path.join(dir, "build")); + fs.writeFileSync( + path.join(dir, "build", "custom.dockerfile"), + "FROM org/app:$VERSION\n" + ); + for (const args of [{ VERSION: "1" }, ["VERSION=1"]]) + expect( + findImageSource( + dir, + { + services: { + unrelated: { build: "." }, + app: { + build: { + context: "build", + dockerfile: "custom.dockerfile", + args + } + } + } + }, + "VERSION" + ) + ).to.deep.equal({ image: "org/app", template: "{version}" }); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/commands/gaBumpUpstream/format.test.ts b/test/commands/gaBumpUpstream/format.test.ts index deeb2071..5bc0dafb 100644 --- a/test/commands/gaBumpUpstream/format.test.ts +++ b/test/commands/gaBumpUpstream/format.test.ts @@ -23,7 +23,9 @@ describe("command / gaBumpUpstream / format", () => { const upstreamVersion = getBumpPrBody(versionsToUpdate); expect(upstreamVersion).to.equal(`Bumps upstream version -- [ipfs/go-ipfs](https://github.com/ipfs/go-ipfs) from v0.6.0 to [v0.7.0](https://github.com/ipfs/go-ipfs/releases/tag/v0.7.0)`); +- [ipfs/go-ipfs](https://github.com/ipfs/go-ipfs) from v0.6.0 to [v0.7.0](https://github.com/ipfs/go-ipfs/releases/tag/v0.7.0) (Compose: \`v0.7.0\`) + +`); }); }); @@ -47,8 +49,10 @@ describe("command / gaBumpUpstream / format", () => { const upstreamVersion = getBumpPrBody(versionsToUpdate); expect(upstreamVersion).to.equal(`Bumps upstream version -- [sigp/lighthouse](https://github.com/sigp/lighthouse) from v0.1.2 to [v0.1.4](https://github.com/sigp/lighthouse/releases/tag/v0.1.4) -- [prysmaticlabs/prysm](https://github.com/prysmaticlabs/prysm) from v0.1.0-beta.28 to [v0.1.0-beta.29](https://github.com/prysmaticlabs/prysm/releases/tag/v0.1.0-beta.29)`); +- [sigp/lighthouse](https://github.com/sigp/lighthouse) from v0.1.2 to [v0.1.4](https://github.com/sigp/lighthouse/releases/tag/v0.1.4) (Compose: \`v0.1.4\`) +- [prysmaticlabs/prysm](https://github.com/prysmaticlabs/prysm) from v0.1.0-beta.28 to [v0.1.0-beta.29](https://github.com/prysmaticlabs/prysm/releases/tag/v0.1.0-beta.29) (Compose: \`v0.1.0-beta.29\`) + +`); }); }); diff --git a/test/commands/gaBumpUpstream/pickLatestRelease.test.ts b/test/commands/gaBumpUpstream/pickLatestRelease.test.ts index 47e2998e..dd4b491e 100644 --- a/test/commands/gaBumpUpstream/pickLatestRelease.test.ts +++ b/test/commands/gaBumpUpstream/pickLatestRelease.test.ts @@ -1,5 +1,8 @@ import { expect } from "chai"; -import { pickLatestRelease } from "../../../src/commands/githubActions/bumpUpstream/github/index.js"; +import { + pickLatestRelease, + selectLatestRelease +} from "../../../src/commands/githubActions/bumpUpstream/github/index.js"; const release = ( tag_name: string, @@ -46,4 +49,55 @@ describe("command / gaBumpUpstream / pickLatestRelease", () => { ]) ).to.equal(null); }); + + it("treats suffixes that are not pre-release words as stable", () => { + expect( + pickLatestRelease([release("v1.17.11-beta.1"), release("v1.17.10-gc")]) + ).to.equal("v1.17.10-gc"); + expect(pickLatestRelease([release("n8n@2.46.6")])).to.equal("n8n@2.46.6"); + }); + + it("filters tag families with tagPattern", () => { + expect( + pickLatestRelease( + [release("op-deployer/v0.9.0-allowlist"), release("op-node/v1.20.1")], + "^op-node/v(.+)$" + ) + ).to.equal("1.20.1"); + }); + + describe("selectLatestRelease", () => { + const noList = async () => { + throw Error("must not list releases"); + }; + + it("prefers GitHub's Latest release when it qualifies", async () => { + expect(await selectLatestRelease(release("v1.9.5"), noList)).to.equal( + "v1.9.5" + ); + }); + + it("falls back to recent releases when Latest does not qualify", async () => { + const recent = async () => [ + release("v5.1.0-rc.1"), + release("v5.0.1", { draft: true }), + release("v5.0.0") + ]; + expect( + await selectLatestRelease(release("v5.1.0-rc.1"), recent) + ).to.equal("v5.0.0"); + expect(await selectLatestRelease(null, recent)).to.equal("v5.0.0"); + }); + + it("applies tagPattern to Latest", async () => { + const recent = async () => [release("op-node/v1.20.1")]; + expect( + await selectLatestRelease( + release("op-deployer/v0.9.0"), + recent, + "^op-node/v(.+)$" + ) + ).to.equal("1.20.1"); + }); + }); }); diff --git a/test/spec/checkPackageSpec.test.ts b/test/spec/checkPackageSpec.test.ts index 66c6d0f7..4a12cc77 100644 --- a/test/spec/checkPackageSpec.test.ts +++ b/test/spec/checkPackageSpec.test.ts @@ -298,6 +298,7 @@ describe("package spec checks", function () { [`1.2.3@sha256:${"a".repeat(64)}`, true], ["1.2.2", false], ["1.2.3@sha256:abc", false], + [`1.2.3@sha256:${"A".repeat(64)}`, false], [`1.2.3@sha256:${"g".repeat(64)}`, false], ["", false] ] as const) { @@ -321,6 +322,45 @@ describe("package spec checks", function () { } }); + it("reports invalid upstream providers and patterns as SPEC-M12 warnings", () => { + write("docker-compose.yml", { + services: { app: { build: { args: { VERSION: "1.2.3" } } } } + }); + for (const metadata of [ + { provider: "npm" }, + { tagPattern: "[" }, + { tagPattern: "(?=v)(.*)" }, + { tagPattern: "(v)\\1" } + ]) { + write("dappnode_package.json", { + name: "demo.dnp.dappnode.eth", + version: "0.1.0", + upstream: [ + { repo: "acme/app", version: "1.2.3", arg: "VERSION", ...metadata } + ] + }); + const findings = check(dir).filter(f => f.rule === "SPEC-M12"); + expect(findings, JSON.stringify(metadata)).to.have.length(1); + expect(findings[0].file).to.equal( + path.join(dir, "dappnode_package.json") + ); + } + write("dappnode_package.json", { + name: "demo.dnp.dappnode.eth", + version: "0.1.0", + upstream: [ + { + repo: "acme/app", + version: "1.2.3", + arg: "VERSION", + provider: "DockerHub", + tagPattern: "^v(.*)$" + } + ] + }); + expect(check(dir).filter(f => f.rule === "SPEC-M12")).to.deep.equal([]); + }); + it("attributes conflicting upstream values to the variant compose", () => { write("dappnode_package.json", { upstream: [{ repo: "acme/app", version: "1.2.3", arg: "APP_VERSION" }] diff --git a/yarn.lock b/yarn.lock index 8e1d3a54..6aef048a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2621,6 +2621,11 @@ y18n@^5.0.5: resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yaml@^2.8.1: + version "2.9.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.9.1.tgz#c16233fb31944e705cfefaff38795587f57588ce" + integrity sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw== + yargs-parser@^21.1.1: version "21.1.1" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" From a41223ec118964ea05d9c0ff686a92ccf766c1ad Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:21:38 +0100 Subject: [PATCH 2/2] fix code style --- doc/package-spec-audit-2026-11.md | 33 -- src/bump/discovery.ts | 206 ++++++---- src/bump/dockerfile.ts | 151 +++++--- src/bump/edits.ts | 324 ++++++++++------ src/bump/metadata.ts | 118 +++--- src/bump/plan.ts | 150 +++++--- src/bump/registry.ts | 354 +++++++++++------- src/bump/resolve.ts | 284 ++++++++------ src/bump/versions.ts | 66 ++-- .../githubActions/bumpUpstream/index.ts | 27 +- .../settings/getInitialSettings.ts | 33 +- .../githubActions/bumpUpstream/types.ts | 2 + src/spec/checkPackageSpec.ts | 78 ++-- test/bump/compareEngines.ts | 4 +- test/bump/deprecation.test.ts | 36 -- test/bump/discovery.test.ts | 226 ++++++----- test/bump/dockerfile.test.ts | 94 +++++ test/bump/edits.test.ts | 2 +- test/bump/metadata.test.ts | 66 ++++ test/bump/registry.test.ts | 20 +- test/bump/resolve.test.ts | 22 +- test/bump/versions.test.ts | 161 ++------ .../gaBumpUpstream/deprecation.test.ts | 19 + test/commands/gaBumpUpstream/format.test.ts | 16 + .../gaBumpUpstream/pickLatestRelease.test.ts | 37 +- 25 files changed, 1517 insertions(+), 1012 deletions(-) delete mode 100644 doc/package-spec-audit-2026-11.md delete mode 100644 test/bump/deprecation.test.ts create mode 100644 test/bump/dockerfile.test.ts create mode 100644 test/bump/metadata.test.ts create mode 100644 test/commands/gaBumpUpstream/deprecation.test.ts diff --git a/doc/package-spec-audit-2026-11.md b/doc/package-spec-audit-2026-11.md deleted file mode 100644 index 25b8a8bf..00000000 --- a/doc/package-spec-audit-2026-11.md +++ /dev/null @@ -1,33 +0,0 @@ -# Package-spec audit — November 2026 baseline - -This is the historical baseline used when the package spec was introduced. It predates subsequent package migrations and is not a pending-work list or a current scan. The normative rules are in [package-spec.md](package-spec.md). New scans should record their date, repository revisions and scan scope separately. - -Counts are from all 113 non-archived `DAppNodePackage*` and `DNP_*` repositories in November 2026; "active" means a push in the last 12 months (87 repositories). 58 active repositories currently have at least one finding, mostly SPEC-M03; 37 repositories are already clean. - -| Rule | Baseline observation | -|---|---| -| SPEC-M01 | 0 active repositories. | -| SPEC-M02 | 2 inactive (Zosma-Nethermind, Zosma-OE). | -| SPEC-M03 | 44 active repositories still use them, including DNP_BIND, DNP_IPFS, Hopr, NEAR, Hermes-agent and Honcho. This is the largest migration and removes a second code path from the SDK bumper, tropibot's discovery and edit code, and the dashboards. | -| SPEC-M04 | 9 active (SafeStake-testnet, SplitPro, avalanche, graph, monero, rocketpool, rocketpool-testnet, supabase, web3signer-kiln). | -| SPEC-M05 | 4 active (GnosisVPNExitNode, Hopr-testnet, Tornado-Cash-Relayer, aztec-testnet). | -| SPEC-M06 | 0 repositories. | -| SPEC-M07 | 1 active (owncloud: `minimumDappnodeVersion`), 5 inactive with legacy `avatar`/`image`. | -| SPEC-M08 | 8 active point elsewhere: renamed repositories (DMS, anchor-generic), sibling packages (Erigon-Sepolia-Archival, lukso-erigon, NEAR-testnet), a fork (avalanche) and upstream projects (SplitPro, supabase). | -| SPEC-M09 | 2 active (op-erigon, op-geth). | -| SPEC-T01 | 5 active (juno-generic, jupyter-generic, nethermind-generic, ollama-generic, pathfinder-generic) and madara-generic. | -| SPEC-T02 | 0 repositories. | -| SPEC-T03 | 0 repositories. | -| SPEC-T04 | 1 active (erigon-generic: a stale root `prometheus-targets.json`). | -| SPEC-T05 | 0 repositories. | -| SPEC-C01 | 0 repositories use the flag. | -| SPEC-C02 | 6 active (Hermes-agent, Nextcloud, graph, jupyter-generic, ollama-generic, openclaw). | -| SPEC-C03 | 2 active (graph: `postgres`; owncloud: `webhippie/mariadb:latest`, `webhippie/redis:latest`). | -| SPEC-C04 | 2 active (alephium, supabase) and madara-generic. | -| SPEC-C05 | 3 active (DNP_NOTIFICATIONS, pi-hole, portainer). | -| SPEC-F01 | 5 active: NEAR `setup-wizard-ui.yml`, Tornado-Cash-Relayer `avatar-default.jpg`, package-harness `avatar-harness.svg`, pathfinder-generic `getting-started.yml` (must be `.md`) and reth-gnosis `notifications.yml` (must be `.yaml`). | -| SPEC-F02 | 2 active (Lido-CSM-generic avatar 123 KB, lukso-erigon avatar 245 KB). | -| SPEC-F03 | 1 active (DMS notifications). | -| SPEC-B01 | no workflow uses it. | - -Other observations from the same review: 109 repositories declared Compose version, 25 omitted architectures, 6 inactive repositories used unpinned Dockerfile bases, and 20 active repositories used remote ADD without a checksum (19 through the staking-scripts pattern). Tropibot workflows covered 46 repositories and the SDK bumper covered 5. diff --git a/src/bump/discovery.ts b/src/bump/discovery.ts index 7b549502..a65e8a44 100644 --- a/src/bump/discovery.ts +++ b/src/bump/discovery.ts @@ -1,21 +1,36 @@ -import fs from "node:fs"; -import path from "node:path"; +import fs from "fs"; +import path from "path"; import { Manifest } from "@dappnode/types"; +import { + defaultManifestFileName, + defaultVariantsDirName, + UPSTREAM_VERSION_VARNAME +} from "../params.js"; +import { arrIsUnique } from "../utils/array.js"; +import { getAllVariantsInPath } from "../files/variants/getAllPackageVariants.js"; import { BumpUpstreamItem, compileTagPattern, parseProvider, - Provider + UpstreamProvider } from "./metadata.js"; export interface Upstream { repo: string; version: string; arg: string; - provider: Provider; + provider: UpstreamProvider; tagPattern?: string; } +/** + * The part of a compose file that upstream bumps read. Unlike the shared + * `Compose` type, build args may use list syntax (`- ARG=value`). + */ +export interface BumpCompose { + services: Record; +} + export interface BuildService { build?: | string @@ -25,86 +40,127 @@ export interface BuildService { args?: Record | string[]; }; } -export interface BumpCompose { - services: Record; -} -export function buildArgs(service: BuildService): [string, unknown][] { - const args = - typeof service.build === "object" ? service.build.args : undefined; - return Array.isArray(args) - ? args.map(item => { - const split = item.indexOf("="); - return split < 0 - ? [item.trim(), null] - : [item.slice(0, split).trim(), item.slice(split + 1)]; - }) - : Object.entries(args || {}); +export type BumpManifest = Partial & { upstreamProvider?: string }; + +type UpstreamEntry = Partial & { repo: string }; + +/** + * Returns the build args of a compose service as [name, value] pairs, for both + * mapping and list syntax. A list arg without `=` has a null value. + */ +export function getBuildArgs(service: BuildService): [string, unknown][] { + if (typeof service.build !== "object" || !service.build.args) return []; + + const args = service.build.args; + if (!Array.isArray(args)) return Object.entries(args); + + return args.map(arg => { + const separator = arg.indexOf("="); + if (separator < 0) return [arg.trim(), null]; + return [arg.slice(0, separator).trim(), arg.slice(separator + 1)]; + }); } +/** + * Lists the upstreams declared in the manifest, from `upstream[]` or from the + * legacy `upstreamRepo`, `upstreamVersion` and `upstreamArg` fields. + */ export function discoverUpstreams( - manifest: Partial & { upstreamProvider?: string }, + manifest: BumpManifest, compose: BumpCompose ): Upstream[] { - const entries: (Partial & { repo: string })[] = - manifest.upstream ?? - (manifest.upstreamRepo - ? [ - { - repo: manifest.upstreamRepo, - version: manifest.upstreamVersion, - arg: manifest.upstreamArg, - provider: manifest.upstreamProvider - } - ] - : []); - const named = [ - ...new Set( - Object.values(compose.services) - .flatMap(service => buildArgs(service).map(([arg]) => arg)) - .filter(arg => /VERSION|TAG/.test(arg)) - ) - ]; - const result = entries.map(entry => { - if (!entry.repo || !entry.version) - throw Error("Upstream repo and version are required"); - if (entries.length > 1 && !entry.arg) - throw Error("Every entry in a multi-upstream manifest must name an arg"); - if (!entry.arg && named.length > 1) - throw Error(`Cannot uniquely detect upstream arg: ${named.join(", ")}`); - compileTagPattern(entry.tagPattern); - return { - ...entry, - version: entry.version, - arg: entry.arg || named[0] || "UPSTREAM_VERSION", - provider: parseProvider(entry.provider) - }; - }); - for (const key of ["repo", "arg"] as const) - if (new Set(result.map(entry => entry[key])).size !== result.length) - throw Error( - `Upstream ${key === "repo" ? "repositories" : "args"} must be unique` - ); - return result; + const entries = getUpstreamEntries(manifest); + const versionArgs = getVersionArgs(compose); + const upstreams = entries.map(entry => + parseUpstream(entry, entries.length > 1, versionArgs) + ); + + if (!arrIsUnique(upstreams.map(({ repo }) => repo))) + throw Error("Upstream repositories must be unique"); + if (!arrIsUnique(upstreams.map(({ arg }) => arg))) + throw Error("Upstream args must be unique"); + + return upstreams; } -export function packageManifestDirs( +/** + * Returns the directories whose manifest version a bump updates: each variant + * in `variantsDirName`, or the package root when the package has no variants. + */ +export function getPackageManifestDirs( dir: string, - variantsDir = "package_variants" + variantsDirName = defaultVariantsDirName ): string[] { - for (const name of [...new Set([variantsDir, "package_variants"])]) { - const base = path.resolve(dir, name); - if (!fs.existsSync(base)) continue; - const dirs = fs - .readdirSync(base, { withFileTypes: true }) - .filter(entry => entry.isDirectory()) - .map(entry => path.join(base, entry.name)); - if (dirs.length) { - for (const variant of dirs) - if (!fs.existsSync(path.join(variant, "dappnode_package.json"))) - throw Error(`Variant has no manifest: ${variant}`); - return dirs; - } + const variantsPath = path.resolve(dir, variantsDirName); + if (!fs.existsSync(variantsPath)) return [dir]; + + const variantDirs = getAllVariantsInPath(variantsPath).map(variant => + path.join(variantsPath, variant) + ); + if (variantDirs.length === 0) return [dir]; + + for (const variantDir of variantDirs) { + if (!fs.existsSync(path.join(variantDir, defaultManifestFileName))) + throw Error(`Variant has no manifest: ${variantDir}`); } - return [dir]; + + return variantDirs; +} + +function getUpstreamEntries(manifest: BumpManifest): UpstreamEntry[] { + if (manifest.upstream) return manifest.upstream; + if (!manifest.upstreamRepo) return []; + + return [ + { + repo: manifest.upstreamRepo, + version: manifest.upstreamVersion, + arg: manifest.upstreamArg, + provider: manifest.upstreamProvider + } + ]; +} + +/** + * Compose build args that look like versions. They identify the arg of a + * single upstream that does not name one. + */ +function getVersionArgs(compose: BumpCompose): string[] { + const argNames = Object.values(compose.services).flatMap(service => + getBuildArgs(service).map(([name]) => name) + ); + return [...new Set(argNames)].filter(name => /VERSION|TAG/.test(name)); +} + +function parseUpstream( + entry: UpstreamEntry, + isMultiUpstream: boolean, + versionArgs: string[] +): Upstream { + if (!entry.repo || !entry.version) + throw Error("Upstream repo and version are required"); + + compileTagPattern(entry.tagPattern); + + return { + ...entry, + version: entry.version, + arg: entry.arg || inferUpstreamArg(isMultiUpstream, versionArgs), + provider: parseProvider(entry.provider) + }; +} + +function inferUpstreamArg( + isMultiUpstream: boolean, + versionArgs: string[] +): string { + if (isMultiUpstream) + throw Error("Every entry in a multi-upstream manifest must name an arg"); + if (versionArgs.length > 1) + throw Error( + `Cannot uniquely detect upstream arg: ${versionArgs.join(", ")}` + ); + + return versionArgs[0] || UPSTREAM_VERSION_VARNAME; } diff --git a/src/bump/dockerfile.ts b/src/bump/dockerfile.ts index d46b7e45..02929259 100644 --- a/src/bump/dockerfile.ts +++ b/src/bump/dockerfile.ts @@ -1,89 +1,132 @@ -import fs from "node:fs"; -import path from "node:path"; -import { buildArgs, BumpCompose } from "./discovery.js"; +import fs from "fs"; +import path from "path"; +import { BumpCompose, getBuildArgs } from "./discovery.js"; +/** + * The image a Dockerfile `FROM` line builds on. In `template`, the image tag, + * `{version}` stands for the upstream arg and `{version#prefix}` for + * `${ARG#prefix}`. + */ export interface ImageSource { image: string; template: string; } +/** + * Renders the image tag that `source` uses for an upstream version. + */ export function renderDockerTag(source: ImageSource, version: string): string { return source.template.replace( /\{version(?:#([^}]*))?\}/g, - (_, prefix: string | undefined) => - prefix && version.startsWith(prefix) - ? version.slice(prefix.length) - : version + (_match, prefix?: string) => removePrefix(version, prefix) ); } +/** + * Returns the image of the first `FROM` line whose tag uses `arg`, or null when + * no `FROM` uses it. Other args are replaced by their static `ARG NAME=value` + * defaults. + */ export function extractImageSource( - content: string, + dockerfile: string, arg: string ): ImageSource | null { - const defaults = new Map(); - const lines = content.replace(/\\\r?\n/g, "").split(/\r?\n/); - for (const line of lines) { - const match = /^\s*ARG\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)\s*$/i.exec(line); - if (match) defaults.set(match[1], match[2].replace(/^["']|["']$/g, "")); - } - const expand = (expression: string, seen = new Set()): string => - expression.replace( - /\$\{([A-Za-z_][A-Za-z0-9_]*)(?:#([^}]*))?\}|\$([A-Za-z_][A-Za-z0-9_]*)/g, - ( - original, - braced: string | undefined, - prefix: string | undefined, - bare: string | undefined - ) => { - const name = braced || bare!; - if (name === arg) - return prefix === undefined ? "{version}" : `{version#${prefix}}`; - if (!defaults.has(name) || seen.has(name)) return original; - const value = expand(defaults.get(name)!, new Set([...seen, name])); - return prefix && value.startsWith(prefix) - ? value.slice(prefix.length) - : value; - } - ); + // Join line continuations so each instruction is on one line + const lines = dockerfile.replace(/\\\r?\n/g, "").split(/\r?\n/); + const argDefaults = getArgDefaults(lines); + for (const line of lines) { const from = /^\s*FROM\s+(?:--platform=\S+\s+)?(\S+)/i.exec(line); if (!from) continue; - const token = expand(from[1]); - if (!token.includes("{version")) continue; - const colon = token.lastIndexOf(":"); - if (colon <= token.lastIndexOf("/")) - throw Error(`Cannot find image tag for ${arg}`); - const image = token.slice(0, colon); - const template = token.slice(colon + 1); - if ( - image.includes("$") || - image.includes("{version") || - template.includes("$") - ) - throw Error(`Unresolved Dockerfile argument for ${arg}`); - return { image, template }; + + const imageRef = expandArgs(from[1], arg, argDefaults); + if (imageRef.includes("{version")) return splitImageRef(imageRef, arg); } + return null; } +/** + * Reads the Dockerfile of the compose service that declares `arg` and returns + * the image whose tag uses it. + */ export function findImageSource( dir: string, compose: BumpCompose, arg: string ): ImageSource | null { for (const service of Object.values(compose.services)) { - if ( - typeof service.build !== "object" || - !buildArgs(service).some(([key]) => key === arg) - ) - continue; - const file = path.resolve( + if (typeof service.build !== "object") continue; + if (!getBuildArgs(service).some(([name]) => name === arg)) continue; + + const dockerfilePath = path.resolve( dir, service.build.context || ".", service.build.dockerfile || "Dockerfile" ); - return extractImageSource(fs.readFileSync(file, "utf8"), arg); + return extractImageSource(fs.readFileSync(dockerfilePath, "utf8"), arg); } + return null; } + +function getArgDefaults(lines: string[]): Map { + const argDefaults = new Map(); + for (const line of lines) { + const match = /^\s*ARG\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)\s*$/i.exec(line); + if (match) argDefaults.set(match[1], match[2].replace(/^["']|["']$/g, "")); + } + return argDefaults; +} + +/** + * Replaces `$NAME`, `${NAME}` and `${NAME#prefix}` in a Dockerfile expression. + * The upstream arg becomes the `{version}` placeholder, other args their + * default. Args without a default are kept as they are. + */ +function expandArgs( + expression: string, + arg: string, + argDefaults: Map, + expanding = new Set() +): string { + return expression.replace( + /\$\{([A-Za-z_][A-Za-z0-9_]*)(?:#([^}]*))?\}|\$([A-Za-z_][A-Za-z0-9_]*)/g, + (original, bracedName?: string, prefix?: string, bareName?: string) => { + const name = bracedName || bareName || ""; + if (name === arg) + return prefix === undefined ? "{version}" : `{version#${prefix}}`; + + const defaultValue = argDefaults.get(name); + if (defaultValue === undefined || expanding.has(name)) return original; + + const value = expandArgs( + defaultValue, + arg, + argDefaults, + new Set([...expanding, name]) + ); + return removePrefix(value, prefix); + } + ); +} + +function splitImageRef(imageRef: string, arg: string): ImageSource { + const tagSeparator = imageRef.lastIndexOf(":"); + if (tagSeparator <= imageRef.lastIndexOf("/")) + throw Error(`Cannot find image tag for ${arg}`); + + const image = imageRef.slice(0, tagSeparator); + const template = imageRef.slice(tagSeparator + 1); + if (image.includes("$") || image.includes("{version")) + throw Error(`Unresolved Dockerfile argument for ${arg}`); + if (template.includes("$")) + throw Error(`Unresolved Dockerfile argument for ${arg}`); + + return { image, template }; +} + +function removePrefix(value: string, prefix?: string): string { + if (prefix && value.startsWith(prefix)) return value.slice(prefix.length); + return value; +} diff --git a/src/bump/edits.ts b/src/bump/edits.ts index 60905a77..b7a5b5fa 100644 --- a/src/bump/edits.ts +++ b/src/bump/edits.ts @@ -1,45 +1,206 @@ -import fs from "node:fs"; -import { isDeepStrictEqual } from "node:util"; +import fs from "fs"; +import { isDeepStrictEqual } from "util"; import yaml from "js-yaml"; import { + Document, isAlias, isMap, isScalar, isSeq, parseDocument, Scalar, - visit + visit, + YAMLMap, + YAMLSeq } from "yaml"; import { BumpCompose } from "./discovery.js"; +/** A value to set at a path of a parsed manifest or compose file. */ export interface ValueChange { path: (string | number)[]; value: string; } + +/** A file to write, with the content it was planned from. */ +export interface PlannedFile { + path: string; + before: string; + after: string; +} + +/** Replaces the text between `start` and `end` with `value`. */ interface TextEdit { start: number; end: number; value: string; } -function document(text: string) { +interface ComposeEdits { + edits: TextEdit[]; + changes: ValueChange[]; + foundArgs: Set; +} + +/** + * Sets values in manifest JSON, keeping its formatting and key order. + */ +export function editManifest(text: string, changes: ValueChange[]): string { + // Only JSON manifests are edited + JSON.parse(text); + + const doc = parseEditableDocument(text); + const edits: TextEdit[] = []; + for (const change of changes) { + const node = doc.getIn(change.path, true); + const edit = getScalarEdit(text, node, change.value, true); + if (edit) edits.push(edit); + } + + const edited = applyTextEdits(text, edits); + validateEdits(text, edited, changes, true); + return edited; +} + +/** + * Sets compose build args, keeping comments, quotes and formatting. `updates` + * maps arg names to their new value; at least one service must declare each. + */ +export function editCompose( + text: string, + updates: Map +): string { + const doc = parseEditableDocument(text); + const compose = yaml.load(text) as BumpCompose; + const result: ComposeEdits = { edits: [], changes: [], foundArgs: new Set() }; + + for (const service of Object.keys(compose.services || {})) { + const argsPath = ["services", service, "build", "args"]; + const args = doc.getIn(argsPath, true); + if (isMap(args)) editMappingArgs(text, args, argsPath, updates, result); + else if (isSeq(args)) editListArgs(text, args, argsPath, updates, result); + } + + for (const arg of updates.keys()) { + if (!result.foundArgs.has(arg)) + throw Error(`Required Compose build argument '${arg}' is absent`); + } + + const edited = applyTextEdits(text, result.edits); + validateEdits(text, edited, result.changes); + return edited; +} + +/** + * Parses both texts and checks that the edited one differs from the original + * only by `changes`, so an edit can never change any other value. + */ +export function validateEdits( + before: string, + after: string, + changes: ValueChange[], + json = false +): void { + const parse = (text: string): unknown => + json ? JSON.parse(text) : yaml.load(text); + + const expected = parse(before); + for (const change of changes) setValueAtPath(expected, change); + + if (!isDeepStrictEqual(expected, parse(after))) + throw Error( + "Edit changed values outside the intended upstream/package updates" + ); +} + +/** + * Writes the planned files, after checking that none changed since planning. + */ +export function writePlan(plan: PlannedFile[]): void { + for (const file of plan) { + if (fs.readFileSync(file.path, "utf8") !== file.before) + throw Error(`File changed during bump planning: ${file.path}`); + } + + for (const file of plan) { + if (file.after !== file.before) fs.writeFileSync(file.path, file.after); + } +} + +/** + * Parses YAML or JSON, keeping node positions. Aliases and merge keys are + * rejected, since editing one node could then change several values. + */ +function parseEditableDocument(text: string): Document.Parsed { const doc = parseDocument(text, { uniqueKeys: true }); - if (doc.errors.length) + if (doc.errors.length > 0) throw Error( `Invalid document: ${doc.errors.map(error => error.message).join("; ")}` ); + visit(doc, { Node(key, node) { if (isAlias(node)) throw Error("Aliases are ambiguous for upstream edits"); - if (isScalar(node) && node.value === "<<" && key === "key") + if (key === "key" && isScalar(node) && node.value === "<<") throw Error("YAML merges are ambiguous for upstream edits"); } }); + return doc; } -function scalarEdit( +function editMappingArgs( + text: string, + args: YAMLMap, + argsPath: string[], + updates: Map, + result: ComposeEdits +): void { + for (const [arg, value] of updates) { + if (!args.has(arg)) continue; + result.foundArgs.add(arg); + + const edit = getScalarEdit(text, args.get(arg, true), value); + if (edit) { + result.edits.push(edit); + result.changes.push({ path: [...argsPath, arg], value }); + } + } +} + +function editListArgs( + text: string, + args: YAMLSeq, + argsPath: string[], + updates: Map, + result: ComposeEdits +): void { + args.items.forEach((node, index) => { + if (!isScalar(node) || typeof node.value !== "string") + throw Error("List build args must be strings"); + + const separator = node.value.indexOf("="); + const arg = ( + separator < 0 ? node.value : node.value.slice(0, separator) + ).trim(); + const value = updates.get(arg); + if (value === undefined) return; + result.foundArgs.add(arg); + + const entry = `${arg}=${value}`; + const edit = getScalarEdit(text, node, entry); + if (edit) { + result.edits.push(edit); + result.changes.push({ path: [...argsPath, index], value: entry }); + } + }); +} + +/** + * Returns the edit that sets a scalar node to `value` in its current quote + * style, or null when it already has that value. + */ +function getScalarEdit( text: string, node: unknown, value: string, @@ -48,130 +209,57 @@ function scalarEdit( if (!isScalar(node) || !node.range) throw Error("Expected a scalar at the edit location"); if (node.value === value) return null; - let replacement: string; - if (json || node.type === Scalar.QUOTE_DOUBLE) - replacement = JSON.stringify(value); - else if (node.type === Scalar.QUOTE_SINGLE) - replacement = `'${value.replace(/'/g, "''")}'`; - else if (node.type === Scalar.PLAIN && yaml.load(value) === value) - replacement = value; - else if (node.type === Scalar.PLAIN) replacement = JSON.stringify(value); - else throw Error("Block scalars cannot be safely edited as upstream pins"); + const [start, end] = node.range; if (!text.slice(start, end)) throw Error("Empty scalar edit location"); - return { start, end, value: replacement }; + + return { start, end, value: formatScalar(node, value, json) }; } -function replace(text: string, edits: (TextEdit | null)[]): string { - let previous = text.length; - for (const edit of edits - .filter((edit): edit is TextEdit => edit !== null) - .sort((a, b) => b.start - a.start)) { - if (edit.end > previous) throw Error("Overlapping upstream edits"); - text = text.slice(0, edit.start) + edit.value + text.slice(edit.end); - previous = edit.start; +function formatScalar(node: Scalar, value: string, json: boolean): string { + if (json || node.type === Scalar.QUOTE_DOUBLE) return JSON.stringify(value); + if (node.type === Scalar.QUOTE_SINGLE) + return `'${value.replace(/'/g, "''")}'`; + if (node.type === Scalar.PLAIN) { + // Quote values that YAML would not read back as the same string, e.g. 1.10 + return yaml.load(value) === value ? value : JSON.stringify(value); } - return text; + throw Error("Block scalars cannot be safely edited as upstream pins"); } -/** Reparse independently, allowing only the explicitly requested semantic changes. */ -export function validateEdits( - before: string, - after: string, - changes: ValueChange[], - json = false -): void { - const parse = (text: string): unknown => - json ? JSON.parse(text) : yaml.load(text); - const expected = parse(before); - for (const change of changes) { - let parent = expected as Record; - for (const key of change.path.slice(0, -1)) { - if (!parent || typeof parent !== "object" || !(key in parent)) - throw Error("Missing edit path"); - parent = parent[key] as Record; - } - const key = change.path[change.path.length - 1]; - if (!parent || typeof parent !== "object" || !(key in parent)) - throw Error("Missing edit value"); - parent[key] = change.value; +/** + * Applies the edits from the end of the text backwards, so that the offsets of + * the remaining edits stay valid. + */ +function applyTextEdits(text: string, edits: TextEdit[]): string { + const sortedEdits = [...edits].sort((a, b) => b.start - a.start); + + let edited = text; + let previousStart = text.length; + for (const edit of sortedEdits) { + if (edit.end > previousStart) throw Error("Overlapping upstream edits"); + edited = edited.slice(0, edit.start) + edit.value + edited.slice(edit.end); + previousStart = edit.start; } - if (!isDeepStrictEqual(expected, parse(after))) - throw Error( - "Edit changed values outside the intended upstream/package updates" - ); -} -export function editManifest(text: string, changes: ValueChange[]): string { - JSON.parse(text); - const doc = document(text); - const result = replace( - text, - changes.map(change => - scalarEdit(text, doc.getIn(change.path, true), change.value, true) - ) - ); - validateEdits(text, result, changes, true); - return result; + return edited; } -export function editCompose( - text: string, - updates: Map -): string { - const doc = document(text); - const compose = yaml.load(text) as BumpCompose; - const edits: (TextEdit | null)[] = []; - const changes: ValueChange[] = []; - const found = new Set(); - for (const service of Object.keys(compose.services || {})) { - const base = ["services", service, "build", "args"]; - const args = doc.getIn(base, true); - if (isMap(args)) { - for (const [arg, value] of updates) { - if (!args.has(arg)) continue; - found.add(arg); - const node = args.get(arg, true); - edits.push(scalarEdit(text, node, value)); - if (!isScalar(node) || node.value !== value) - changes.push({ path: [...base, arg], value }); - } - } else if (isSeq(args)) { - args.items.forEach((node, index) => { - if (!isScalar(node) || typeof node.value !== "string") - throw Error("List build args must be strings"); - const split = node.value.indexOf("="); - const arg = ( - split < 0 ? node.value : node.value.slice(0, split) - ).trim(); - const value = updates.get(arg); - if (value === undefined) return; - found.add(arg); - const replacement = `${arg}=${value}`; - edits.push(scalarEdit(text, node, replacement)); - if (node.value !== replacement) - changes.push({ path: [...base, index], value: replacement }); - }); - } +function setValueAtPath(root: unknown, { path, value }: ValueChange): void { + let parent = root; + for (const key of path.slice(0, -1)) { + if (!hasKey(parent, key)) throw Error("Missing edit path"); + parent = parent[key]; } - for (const arg of updates.keys()) - if (!found.has(arg)) - throw Error(`Required Compose build argument '${arg}' is absent`); - const result = replace(text, edits); - validateEdits(text, result, changes); - return result; -} -export interface PlannedFile { - path: string; - before: string; - after: string; + const key = path[path.length - 1]; + if (!hasKey(parent, key)) throw Error("Missing edit value"); + parent[key] = value; } -export function writePlan(plan: PlannedFile[]): void { - for (const file of plan) - if (fs.readFileSync(file.path, "utf8") !== file.before) - throw Error(`File changed during bump planning: ${file.path}`); - for (const file of plan) - if (file.before !== file.after) fs.writeFileSync(file.path, file.after); +function hasKey( + value: unknown, + key: string | number +): value is Record { + return typeof value === "object" && value !== null && key in value; } diff --git a/src/bump/metadata.ts b/src/bump/metadata.ts index 4616db28..6750d3d8 100644 --- a/src/bump/metadata.ts +++ b/src/bump/metadata.ts @@ -1,72 +1,74 @@ import { UpstreamItem } from "@dappnode/types"; -/** Optional metadata supported before it is published in shared types. */ +/** + * Manifest upstream entry with the optional bump metadata. Drop it for + * `UpstreamItem` once the published @dappnode/types includes these fields. + */ export type BumpUpstreamItem = UpstreamItem & { provider?: string; tagPattern?: string; }; -export type Provider = "docker" | "github" | "docker_tags"; -export function parseProvider(value: unknown): Provider { - if (value === undefined) return "docker"; - if (typeof value === "string") { - switch (value.toLowerCase()) { - case "docker": - return "docker"; - case "github": - return "github"; - case "docker_tags": - case "docker-tags": - case "dockerhub": - return "docker_tags"; - } - } - throw Error(`Unknown upstream provider: ${String(value)}`); +export type UpstreamProvider = "docker" | "github" | "docker_tags"; + +const providersByName = new Map([ + ["docker", "docker"], + ["github", "github"], + ["docker_tags", "docker_tags"], + ["docker-tags", "docker_tags"], + ["dockerhub", "docker_tags"] +]); + +/** + * Parses the upstream `provider` field, case-insensitively. An unset or empty + * provider means `docker`. + */ +export function parseProvider(provider: unknown): UpstreamProvider { + if (provider === undefined) return "docker"; + if (typeof provider !== "string") + throw Error(`Unknown upstream provider: ${String(provider)}`); + + const name = provider.trim().toLowerCase(); + if (name === "") return "docker"; + + const parsed = providersByName.get(name); + if (!parsed) throw Error(`Unknown upstream provider: ${provider}`); + return parsed; } -/** Restrict patterns to constructs shared by JavaScript and Rust regex. */ -export function compileTagPattern(value: unknown): RegExp | undefined { - if (value === undefined) return undefined; - if (typeof value !== "string") throw Error("tagPattern must be a string"); - let inClass = false; - for (let i = 0; i < value.length; i++) { - const char = value[i]; - if (char === "\\") { - const escaped = value[++i]; - if (/[1-9]/.test(escaped || "") || escaped === "k") - throw Error("tagPattern must not use backreferences"); - if ( - escaped && - /[a-zA-Z0-9]/.test(escaped) && - !/[dDsSwWbBnrtfvxu]/.test(escaped) - ) - throw Error( - "tagPattern uses an escape outside the common regex subset" - ); - if (inClass && escaped === "b") - throw Error( - "tagPattern uses a class escape outside the common regex subset" - ); - continue; - } - if (char === "[") inClass = true; - if (char === "]") inClass = false; - if ( - !inClass && - char === "(" && - value[i + 1] === "?" && - value[i + 2] !== ":" - ) - throw Error( - "tagPattern must use the common regex subset (no lookaround or named groups)" - ); - } - return new RegExp(value); +/** + * Compiles the upstream `tagPattern`. The org bump engine evaluates the same + * pattern with Rust's `regex`, which has no lookaround or backreferences, so + * those are rejected here as well. + */ +export function compileTagPattern(tagPattern: unknown): RegExp | undefined { + if (tagPattern === undefined) return undefined; + if (typeof tagPattern !== "string") + throw Error("tagPattern must be a string"); + + // Drop escaped backslashes so `\\1` is not mistaken for a backreference + const pattern = tagPattern.replace(/\\\\/g, ""); + if (/\(\? { const manifest = JSON.parse( - fs.readFileSync(path.join(dir, "dappnode_package.json"), "utf8") - ) as Manifest; + fs.readFileSync(path.join(dir, defaultManifestFileName), "utf8") + ) as BumpManifest; const compose = yaml.load( - fs.readFileSync(path.join(dir, "docker-compose.yml"), "utf8") + fs.readFileSync(path.join(dir, defaultComposeFileName), "utf8") ) as BumpCompose; + const upstreams = discoverUpstreams(manifest, compose); if (explicitVersion !== undefined && upstreams.length !== 1) throw Error("An explicit version requires exactly one upstream"); - const results: ResolvedUpstream[] = []; + + const resolvedUpstreams: ResolvedUpstream[] = []; for (const upstream of upstreams) { - const result = await resolveUpstream( + const resolved = await resolveUpstream( dir, compose, upstream, dependencies, explicitVersion ); - if (result) results.push(result); + if (resolved) resolvedUpstreams.push(resolved); } - return results; + + return resolvedUpstreams; } +/** + * Plans the file edits of a bump: the upstream versions in the root manifest, + * the next package version in each package manifest (root or variants), and + * the compose build args. Returns no edits when every upstream is current. + */ export async function planPackageEdits({ dir, upstreams, @@ -58,53 +71,86 @@ export async function planPackageEdits({ nextPackageVersion: (dir: string) => Promise; variantsDir?: string; }): Promise { - const changed = upstreams.filter(item => item.targetVersion !== item.version); - if (!changed.length) return []; - const manifestFile = path.join(dir, "dappnode_package.json"); - const manifestText = fs.readFileSync(manifestFile, "utf8"); - const manifest = JSON.parse(manifestText) as Manifest; - const changes: ValueChange[] = changed.map(item => { - const index = manifest.upstream?.findIndex( - upstream => upstream.repo === item.repo - ); - if (manifest.upstream && (index === undefined || index < 0)) - throw Error(`Upstream '${item.repo}' is absent from root manifest`); - return { - path: manifest.upstream - ? ["upstream", index!, "version"] - : ["upstreamVersion"], - value: item.targetVersion - }; - }); + const changedUpstreams = upstreams.filter( + upstream => upstream.targetVersion !== upstream.version + ); + if (changedUpstreams.length === 0) return []; + + const manifestPath = path.join(dir, defaultManifestFileName); + const manifestText = fs.readFileSync(manifestPath, "utf8"); + const rootChanges = getUpstreamVersionChanges( + JSON.parse(manifestText), + changedUpstreams + ); const plan: PlannedFile[] = []; - for (const packageDir of packageManifestDirs(dir, variantsDir)) { - const file = path.join(packageDir, "dappnode_package.json"); - const value = await nextPackageVersion(packageDir); - if (path.resolve(file) === path.resolve(manifestFile)) - changes.push({ path: ["version"], value }); - else { - const before = fs.readFileSync(file, "utf8"); - plan.push({ - path: file, - before, - after: editManifest(before, [{ path: ["version"], value }]) - }); + + for (const packageDir of getPackageManifestDirs(dir, variantsDir)) { + const versionChange = { + path: ["version"], + value: await nextPackageVersion(packageDir) + }; + if (path.resolve(packageDir) === path.resolve(dir)) { + rootChanges.push(versionChange); + } else { + const variantManifestPath = path.join( + packageDir, + defaultManifestFileName + ); + plan.push( + planFileEdit(variantManifestPath, text => + editManifest(text, [versionChange]) + ) + ); } } + plan.push({ - path: manifestFile, + path: manifestPath, before: manifestText, - after: editManifest(manifestText, changes) + after: editManifest(manifestText, rootChanges) }); - const composeFile = path.join(dir, "docker-compose.yml"); - const before = fs.readFileSync(composeFile, "utf8"); - plan.push({ - path: composeFile, - before, - after: editCompose( - before, - new Map(changed.map(item => [item.arg, item.composeValue])) + + const composeUpdates = new Map( + changedUpstreams.map(upstream => [upstream.arg, upstream.composeValue]) + ); + plan.push( + planFileEdit(path.join(dir, defaultComposeFileName), text => + editCompose(text, composeUpdates) ) - }); + ); + return plan; } + +/** + * Manifest changes that set each upstream's new version, in `upstream[]` or in + * the legacy `upstreamVersion` field. + */ +function getUpstreamVersionChanges( + manifest: BumpManifest, + upstreams: ResolvedUpstream[] +): ValueChange[] { + return upstreams.map(upstream => { + if (!manifest.upstream) + return { path: ["upstreamVersion"], value: upstream.targetVersion }; + + const index = manifest.upstream.findIndex( + ({ repo }) => repo === upstream.repo + ); + if (index < 0) + throw Error(`Upstream '${upstream.repo}' is absent from root manifest`); + + return { + path: ["upstream", index, "version"], + value: upstream.targetVersion + }; + }); +} + +function planFileEdit( + filePath: string, + edit: (text: string) => string +): PlannedFile { + const before = fs.readFileSync(filePath, "utf8"); + return { path: filePath, before, after: edit(before) }; +} diff --git a/src/bump/registry.ts b/src/bump/registry.ts index dde3b004..ae24e91f 100644 --- a/src/bump/registry.ts +++ b/src/bump/registry.ts @@ -1,112 +1,57 @@ -import { createHash } from "node:crypto"; +import { createHash } from "crypto"; import { highestSemverTag } from "./versions.js"; export const manifestAccept = "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json"; export const digestPattern = /^sha256:[a-f0-9]{64}$/; -export function registryImage(image: string): { +/** Exact host names that address Docker Hub. */ +const dockerHubHosts = new Set([ + "docker.io", + "index.docker.io", + "registry-1.docker.io" +]); +const dockerHubRegistryHost = "registry-1.docker.io"; +const requestTimeoutMs = 30000; +const maxTagPages = 50; + +export interface RegistryImage { + /** Registry host, `registry-1.docker.io` for Docker Hub. */ host: string; + /** Repository name, with `library/` for official Docker Hub images. */ name: string; - hub: boolean; -} { + isDockerHub: boolean; +} + +/** + * Splits an image reference without tag into its registry host and name. + * Images without a registry host are on Docker Hub. + */ +export function parseRegistryImage(image: string): RegistryImage { const parts = image.split("/"); - const explicit = + const hasHost = parts.length > 1 && (/[.:]/.test(parts[0]) || parts[0] === "localhost"); - const host = explicit ? parts.shift()! : "docker.io"; - const hub = ["docker.io", "index.docker.io", "registry-1.docker.io"].includes( - host - ); + const host = hasHost ? parts.shift()! : "docker.io"; const name = parts.join("/"); + + if (!dockerHubHosts.has(host)) return { host, name, isDockerHub: false }; + return { - host: hub ? "registry-1.docker.io" : host, - name: hub && !name.includes("/") ? `library/${name}` : name, - hub + host: dockerHubRegistryHost, + name: name.includes("/") ? name : `library/${name}`, + isDockerHub: true }; } -function checkUrl(value: string): URL { - const url = new URL(value); - const loopback = - url.hostname === "localhost" || - url.hostname === "[::1]" || - /^127\.\d+\.\d+\.\d+$/.test(url.hostname); - if (url.protocol !== "https:" && !(url.protocol === "http:" && loopback)) - throw Error(`Registry URL must use HTTPS: ${url.origin}`); - return url; -} - +/** + * Anonymous client for the Docker registry HTTP API, used to check image tags, + * read manifest digests and list tags. Answers bearer token challenges. + */ export class RegistryClient { constructor(private readonly request: typeof fetch = fetch) {} - private async send( - url: string, - method: string, - name: string, - accept = "application/json" - ): Promise { - checkUrl(url); - const headers: Record = { Accept: accept }; - const send = () => - this.request(url, { - method, - headers, - signal: AbortSignal.timeout(30000) - }); - let response = await send(); - if (response.status !== 401) return response; - const challenge = response.headers.get("www-authenticate") || ""; - if (!/^Bearer\s/i.test(challenge)) - throw Error(`Registry authentication failed (${response.status})`); - const fields = new Map( - [...challenge.matchAll(/([a-z_]+)=(?:"([^"]*)"|([^,\s]+))/gi)].map( - match => [match[1].toLowerCase(), match[2] ?? match[3]] - ) - ); - const realm = fields.get("realm"); - if (!realm) throw Error("Registry bearer challenge has no realm"); - const tokenUrl = checkUrl(realm); - if (fields.has("service")) - tokenUrl.searchParams.set("service", fields.get("service")!); - tokenUrl.searchParams.set("scope", `repository:${name}:pull`); - const tokenResponse = await this.request(tokenUrl, { - signal: AbortSignal.timeout(30000) - }); - if (tokenResponse.status !== 200) - throw Error(`Registry token request failed (${tokenResponse.status})`); - const body = (await tokenResponse.json()) as { - token?: string; - access_token?: string; - }; - const token = body.token || body.access_token; - if (!token) throw Error("Registry token response has no token"); - headers.Authorization = `Bearer ${token}`; - response = await send(); - return response; - } - - private manifestUrl( - image: string, - tag: string - ): { url: string; name: string } { - const { host, name } = registryImage(image); - const scheme = - host === "localhost" || - host.startsWith("localhost:") || - host.startsWith("127.") || - host.startsWith("[::1]") - ? "http" - : "https"; - return { - url: `${scheme}://${host}/v2/${name}/manifests/${encodeURIComponent( - tag - )}`, - name - }; - } - async tagExists(image: string, tag: string): Promise { - const { url, name } = this.manifestUrl(image, tag); + const { url, name } = getManifestUrl(image, tag); const response = await this.send(url, "HEAD", name, manifestAccept); if (response.status === 200) return true; if (response.status === 404) return false; @@ -115,74 +60,205 @@ export class RegistryClient { ); } + /** + * Returns the digest of an image tag, from the `Docker-Content-Digest` + * header or, when the registry does not send it, by hashing the manifest. + */ async manifestDigest(image: string, tag: string): Promise { - const { url, name } = this.manifestUrl(image, tag); - let response = await this.send(url, "HEAD", name, manifestAccept); - if (response.status !== 200) - throw Error( - `Registry returned ${response.status} pinning ${image}:${tag}` - ); - let digest = response.headers.get("docker-content-digest")?.trim(); - if (digest === undefined) { - response = await this.send(url, "GET", name, manifestAccept); - if (response.status !== 200) - throw Error( - `Registry returned ${response.status} pinning ${image}:${tag}` - ); - digest = `sha256:${createHash("sha256") - .update(Buffer.from(await response.arrayBuffer())) - .digest("hex")}`; - } + const { url, name } = getManifestUrl(image, tag); + const head = await this.send(url, "HEAD", name, manifestAccept); + if (head.status !== 200) + throw Error(`Registry returned ${head.status} pinning ${image}:${tag}`); + + const digest = + head.headers.get("docker-content-digest")?.trim() ?? + (await this.hashManifest(url, name, `${image}:${tag}`)); if (!digestPattern.test(digest)) throw Error( `Registry returned malformed digest for ${image}:${tag}: ${digest}` ); + return digest; } + /** + * Lists the tags of an image. Docker Hub tags come from its own API, other + * registries' from the OCI tag list. + */ async listTags(image: string): Promise { - const { host, name, hub } = registryImage(image); - const scheme = - host === "localhost" || - host.startsWith("localhost:") || - host.startsWith("127.") || - host.startsWith("[::1]") - ? "http" - : "https"; - let url: string | null = hub + const { host, name, isDockerHub } = parseRegistryImage(image); + let url: string | null = isDockerHub ? `https://registry.hub.docker.com/v2/repositories/${name}/tags?page_size=100&ordering=last_updated` - : `${scheme}://${host}/v2/${name}/tags/list`; + : `${getRegistryScheme(host)}://${host}/v2/${name}/tags/list`; + const tags: string[] = []; - const seen = new Set(); - for (let page = 0; url && page < 50; page++) { - if (seen.has(url)) throw Error(`Registry pagination loop for ${image}`); - seen.add(url); + const visitedUrls = new Set(); + for (let page = 0; url && page < maxTagPages; page++) { + if (visitedUrls.has(url)) + throw Error(`Registry pagination loop for ${image}`); + visitedUrls.add(url); + const response = await this.send(url, "GET", name); if (response.status !== 200) throw Error(`Registry returned ${response.status} listing ${image}`); - if (hub) { - const body = (await response.json()) as { - results: { name: string }[]; - next?: string | null; - }; - tags.push(...body.results.map(tag => tag.name)); - url = body.next ? new URL(body.next, url).href : null; - } else { - const body = (await response.json()) as { tags?: string[] | null }; - tags.push(...(body.tags || [])); - const link = response.headers.get("link"); - const next: string | undefined = link - ?.split(",") - .map(entry => /<([^>]+)>\s*;\s*rel\s*=\s*"?next"?/i.exec(entry)?.[1]) - .find(Boolean); - url = next ? new URL(next, url).href : null; - } + + const tagPage = isDockerHub + ? await readDockerHubTagPage(response) + : await readOciTagPage(response); + tags.push(...tagPage.tags); + url = tagPage.next ? new URL(tagPage.next, url).href : null; } - if (url) throw Error(`Tag listing exceeded 50 pages for ${image}`); + + if (url) + throw Error(`Tag listing exceeded ${maxTagPages} pages for ${image}`); return tags; } async latestTag(image: string): Promise { return highestSemverTag(await this.listTags(image)); } + + private async hashManifest( + url: string, + name: string, + imageRef: string + ): Promise { + const response = await this.send(url, "GET", name, manifestAccept); + if (response.status !== 200) + throw Error(`Registry returned ${response.status} pinning ${imageRef}`); + + const manifest = Buffer.from(await response.arrayBuffer()); + return `sha256:${createHash("sha256").update(manifest).digest("hex")}`; + } + + /** + * Sends a registry request. On a 401 bearer challenge, fetches an anonymous + * pull token for `name` and retries once with it. + */ + private async send( + url: string, + method: string, + name: string, + accept = "application/json" + ): Promise { + assertSecureUrl(url); + const headers: Record = { Accept: accept }; + + const response = await this.fetchWithTimeout(url, { method, headers }); + if (response.status !== 401) return response; + + const token = await this.getBearerToken( + response.headers.get("www-authenticate") || "", + name + ); + headers.Authorization = `Bearer ${token}`; + return this.fetchWithTimeout(url, { method, headers }); + } + + private async getBearerToken( + challenge: string, + name: string + ): Promise { + if (!/^Bearer\s/i.test(challenge)) + throw Error("Registry authentication failed (401)"); + + const params = parseChallengeParams(challenge); + const realm = params.get("realm"); + if (!realm) throw Error("Registry bearer challenge has no realm"); + + const tokenUrl = assertSecureUrl(realm); + const service = params.get("service"); + if (service) tokenUrl.searchParams.set("service", service); + tokenUrl.searchParams.set("scope", `repository:${name}:pull`); + + const response = await this.fetchWithTimeout(tokenUrl); + if (response.status !== 200) + throw Error(`Registry token request failed (${response.status})`); + + const body = (await response.json()) as { + token?: string; + access_token?: string; + }; + const token = body.token || body.access_token; + if (!token) throw Error("Registry token response has no token"); + return token; + } + + private fetchWithTimeout( + url: string | URL, + init: RequestInit = {} + ): Promise { + return this.request(url, { + ...init, + signal: AbortSignal.timeout(requestTimeoutMs) + }); + } +} + +function getManifestUrl( + image: string, + tag: string +): { url: string; name: string } { + const { host, name } = parseRegistryImage(image); + const scheme = getRegistryScheme(host); + const url = `${scheme}://${host}/v2/${name}/manifests/${encodeURIComponent( + tag + )}`; + return { url, name }; +} + +/** Local registries, as used in tests, are served over plain HTTP. */ +function getRegistryScheme(host: string): "http" | "https" { + const hostname = host.startsWith("[") ? host : host.split(":")[0]; + return isLoopback(hostname) ? "http" : "https"; +} + +function assertSecureUrl(value: string): URL { + const url = new URL(value); + const isLocalHttp = url.protocol === "http:" && isLoopback(url.hostname); + if (url.protocol !== "https:" && !isLocalHttp) + throw Error(`Registry URL must use HTTPS: ${url.origin}`); + return url; +} + +function isLoopback(hostname: string): boolean { + return ( + hostname === "localhost" || + hostname.startsWith("[::1]") || + /^127\.\d+\.\d+\.\d+$/.test(hostname) + ); +} + +/** Parses `key="value"` and `key=value` pairs of a WWW-Authenticate header. */ +function parseChallengeParams(challenge: string): Map { + const params = new Map(); + for (const match of challenge.matchAll( + /([a-z_]+)=(?:"([^"]*)"|([^,\s]+))/gi + )) { + params.set(match[1].toLowerCase(), match[2] ?? match[3]); + } + return params; +} + +async function readDockerHubTagPage( + response: Response +): Promise<{ tags: string[]; next?: string | null }> { + const body = (await response.json()) as { + results: { name: string }[]; + next?: string | null; + }; + return { tags: body.results.map(tag => tag.name), next: body.next }; +} + +/** The next page of an OCI tag list is in the `Link: ; rel="next"` header. */ +async function readOciTagPage( + response: Response +): Promise<{ tags: string[]; next?: string }> { + const body = (await response.json()) as { tags?: string[] | null }; + const next = response.headers + .get("link") + ?.split(",") + .map(link => /<([^>]+)>\s*;\s*rel\s*=\s*"?next"?/i.exec(link)?.[1]) + .find(Boolean); + return { tags: body.tags || [], next }; } diff --git a/src/bump/resolve.ts b/src/bump/resolve.ts index 1472645b..b16b57ec 100644 --- a/src/bump/resolve.ts +++ b/src/bump/resolve.ts @@ -1,64 +1,95 @@ -import { BumpCompose, buildArgs, Upstream } from "./discovery.js"; +import { BumpCompose, getBuildArgs, Upstream } from "./discovery.js"; import { findImageSource, ImageSource, renderDockerTag } from "./dockerfile.js"; import { extractVersion } from "./metadata.js"; import { digestPattern, RegistryClient } from "./registry.js"; import { isOlderVersion, versionCandidates } from "./versions.js"; -export function splitPin(value: string): { version: string; digest?: string } { - const at = value.lastIndexOf("@"); - if (at < 0 || !value.slice(at + 1).startsWith("sha256:")) +/** A compose build arg value: a version, optionally pinned by digest. */ +export interface Pin { + version: string; + digest?: string; +} + +export interface ResolvedUpstream extends Upstream { + /** Version to write to the manifest; `version` when already current. */ + targetVersion: string; + /** Value to write to the compose build arg, keeping a digest pin. */ + composeValue: string; +} + +export interface ResolutionDependencies { + latestRelease: (repo: string, tagPattern?: string) => Promise; + registry: Pick; + imageSource?: ( + dir: string, + compose: BumpCompose, + arg: string + ) => ImageSource | null; +} + +interface ResolutionContext { + upstream: Upstream; + dependencies: ResolutionDependencies; + explicitVersion?: string; + /** Reads the Dockerfile image lazily: most bumps never need it. */ + getImageSource: () => ImageSource | null; +} + +/** + * Splits a build arg value into its version and its `@sha256:` pin. + */ +export function parsePin(value: string): Pin { + const separator = value.lastIndexOf("@"); + if (separator < 0 || !value.slice(separator + 1).startsWith("sha256:")) return { version: value }; - const version = value.slice(0, at); - const digest = value.slice(at + 1); + + const version = value.slice(0, separator); + const digest = value.slice(separator + 1); if (!version || !digestPattern.test(digest)) throw Error( `Invalid digest pin: ${value}; expected 64 lowercase hex characters` ); + return { version, digest }; } -export function currentPin( - compose: BumpCompose, - upstream: Upstream -): { version: string; digest?: string } { - const entries = Object.entries(compose.services).flatMap(([service, value]) => - buildArgs(value) - .filter(([key]) => key === upstream.arg) - .map(([, value]) => ({ service, value })) +/** + * Returns the pin of the upstream arg. Every service declaring the arg must use + * the manifest version and the same digest. + */ +export function getCurrentPin(compose: BumpCompose, upstream: Upstream): Pin { + const values = Object.entries(compose.services).flatMap( + ([service, definition]) => + getBuildArgs(definition) + .filter(([name]) => name === upstream.arg) + .map(([, value]) => ({ service, value })) ); - if (!entries.length) + if (values.length === 0) throw Error(`Required Compose build argument '${upstream.arg}' is absent`); - const pins = entries.map(({ service, value }) => { + + const pins = values.map(({ service, value }) => { if (typeof value !== "string" && typeof value !== "number") throw Error(`Inherited or invalid arg ${upstream.arg} in ${service}`); - const pin = splitPin(String(value)); + + const pin = parsePin(String(value)); if (pin.version !== upstream.version) throw Error( - `Inconsistent Compose pin for ${upstream.arg} in ${service}: ${String( - value - )}; manifest has ${upstream.version}` + `Inconsistent Compose pin for ${upstream.arg} in ${service}: ${value}; manifest has ${upstream.version}` ); return pin; }); + if (pins.some(pin => pin.digest !== pins[0].digest)) throw Error(`Inconsistent digest pins across services for ${upstream.arg}`); - return pins[0]; -} -export interface ResolvedUpstream extends Upstream { - targetVersion: string; - composeValue: string; -} -export interface ResolutionDependencies { - latestRelease: (repo: string, tagPattern?: string) => Promise; - registry: Pick; - imageSource?: ( - dir: string, - compose: BumpCompose, - arg: string - ) => ImageSource | null; + return pins[0]; } +/** + * Resolves the version to bump an upstream to, and the compose value for it. + * A resolved release older than the current version is ignored unless it was + * requested explicitly. Returns null when there is no qualifying release. + */ export async function resolveUpstream( dir: string, compose: BumpCompose, @@ -66,80 +97,123 @@ export async function resolveUpstream( dependencies: ResolutionDependencies, explicitVersion?: string ): Promise { - const pin = currentPin(compose, upstream); - let source: ImageSource | null | undefined; - const image = () => - source === undefined - ? (source = (dependencies.imageSource || findImageSource)( - dir, - compose, - upstream.arg - )) - : source; - let tag: string | null; - if (explicitVersion !== undefined) - tag = - extractVersion(explicitVersion, upstream.tagPattern) ?? explicitVersion; - else if (upstream.provider === "docker_tags") { - const found = image(); - if (!found) throw Error(`No FROM image uses ${upstream.arg}`); - const chosen = await dependencies.registry.latestTag(found.image); - tag = chosen === null ? null : extractVersion(chosen, upstream.tagPattern); - if (chosen !== null && tag === null) - throw Error(`Latest image tag '${chosen}' does not match tagPattern`); - } else - tag = await dependencies.latestRelease(upstream.repo, upstream.tagPattern); + const pin = getCurrentPin(compose, upstream); + const context: ResolutionContext = { + upstream, + dependencies, + explicitVersion, + getImageSource: () => + (dependencies.imageSource || findImageSource)(dir, compose, upstream.arg) + }; + + const tag = await getTargetTag(context); if (tag === null) return null; - const unchanged = (): ResolvedUpstream => ({ + + const current: ResolvedUpstream = { ...upstream, targetVersion: upstream.version, - composeValue: upstream.version + (pin.digest ? `@${pin.digest}` : "") - }); - if (versionCandidates(tag).includes(upstream.version)) return unchanged(); - if (explicitVersion === undefined && isOlderVersion(tag, upstream.version)) - return unchanged(); - let targetVersion = tag; - if ( - upstream.provider === "docker" || - (upstream.provider === "docker_tags" && explicitVersion !== undefined) - ) { - const found = image(); - if (found) { - let candidate: string | undefined; - for (const version of versionCandidates(tag)) { - if ( - await dependencies.registry.tagExists( - found.image, - renderDockerTag(found, version) - ) - ) { - candidate = version; - break; - } - } - if (!candidate) - throw Error( - `No Docker tag candidate exists on ${found.image} for ${tag}` - ); - targetVersion = candidate; - } - } - if ( - targetVersion === upstream.version || - (explicitVersion === undefined && - isOlderVersion(targetVersion, upstream.version)) - ) - return unchanged(); - let composeValue = targetVersion; - if (pin.digest) { - const found = image(); - if (!found) - throw Error(`Pinned argument ${upstream.arg} needs a FROM image`); - const digest = await dependencies.registry.manifestDigest( - found.image, - renderDockerTag(found, targetVersion) + composeValue: pin.digest + ? `${upstream.version}@${pin.digest}` + : upstream.version + }; + if (versionCandidates(tag).includes(upstream.version)) return current; + if (isUnwantedDowngrade(context, tag)) return current; + + const targetVersion = await getImageTagVersion(context, tag); + if (targetVersion === upstream.version) return current; + if (isUnwantedDowngrade(context, targetVersion)) return current; + + const composeValue = pin.digest + ? `${targetVersion}@${await getImageDigest(context, targetVersion)}` + : targetVersion; + + return { ...upstream, targetVersion, composeValue }; +} + +/** + * The tag to bump to: the explicit version, the highest image tag for the + * `docker_tags` provider, or the latest GitHub release otherwise. + */ +async function getTargetTag( + context: ResolutionContext +): Promise { + const { upstream, dependencies, explicitVersion } = context; + + if (explicitVersion !== undefined) + return ( + extractVersion(explicitVersion, upstream.tagPattern) ?? explicitVersion ); - composeValue += `@${digest}`; + + if (upstream.provider !== "docker_tags") + return dependencies.latestRelease(upstream.repo, upstream.tagPattern); + + const imageSource = context.getImageSource(); + if (!imageSource) throw Error(`No FROM image uses ${upstream.arg}`); + + const latestTag = await dependencies.registry.latestTag(imageSource.image); + if (latestTag === null) return null; + + const version = extractVersion(latestTag, upstream.tagPattern); + if (version === null) + throw Error(`Latest image tag '${latestTag}' does not match tagPattern`); + return version; +} + +/** + * For the `docker` provider, returns the form of `tag` (the tag itself or its + * bare version) that the package image is published under. An explicit + * `docker_tags` version is checked too, since it does not come from the tag + * list. Packages whose arg is not used in a `FROM` image keep the tag. + */ +async function getImageTagVersion( + context: ResolutionContext, + tag: string +): Promise { + const { upstream, dependencies, explicitVersion } = context; + const checksImageTag = + upstream.provider === "docker" || + (upstream.provider === "docker_tags" && explicitVersion !== undefined); + if (!checksImageTag) return tag; + + const imageSource = context.getImageSource(); + if (!imageSource) return tag; + + for (const version of versionCandidates(tag)) { + const imageTag = renderDockerTag(imageSource, version); + if (await dependencies.registry.tagExists(imageSource.image, imageTag)) + return version; } - return { ...upstream, targetVersion, composeValue }; + + throw Error( + `No Docker tag candidate exists on ${imageSource.image} for ${tag}` + ); +} + +async function getImageDigest( + context: ResolutionContext, + version: string +): Promise { + const imageSource = context.getImageSource(); + if (!imageSource) + throw Error(`Pinned argument ${context.upstream.arg} needs a FROM image`); + + return context.dependencies.registry.manifestDigest( + imageSource.image, + renderDockerTag(imageSource, version) + ); +} + +/** + * Some upstreams publish images ahead of GitHub releases, so the latest + * release can be older than the packaged version. Only an explicit request + * may go back. + */ +function isUnwantedDowngrade( + context: ResolutionContext, + version: string +): boolean { + return ( + context.explicitVersion === undefined && + isOlderVersion(version, context.upstream.version) + ); } diff --git a/src/bump/versions.ts b/src/bump/versions.ts index b9d2fd88..ff149e9d 100644 --- a/src/bump/versions.ts +++ b/src/bump/versions.ts @@ -1,36 +1,58 @@ import semver from "semver"; +/** + * Returns the version strings to try for a release tag: the tag itself, then + * its bare semver when the tag has a prefix, e.g. `v1.2.3` or `n8n@1.2.3` + * also try `1.2.3`. + */ export function versionCandidates(tag: string): string[] { - const bare = /^\d/.test(tag) - ? undefined - : tag.match(/\d+\.\d+\.\d+(?:[-+].*)?/)?.[0]; - return bare && bare !== tag ? [tag, bare] : [tag]; + if (/^\d/.test(tag)) return [tag]; + + const bareVersion = tag.match(/\d+\.\d+\.\d+(?:[-+].*)?/)?.[0]; + if (!bareVersion) return [tag]; + return [tag, bareVersion]; } -export function comparableVersion(value: string): string | null { - return ( - versionCandidates(value) - .map(candidate => semver.valid(candidate)) - .find(Boolean) ?? null - ); +/** + * Returns the semver form of a version or tag, or null when it has none. + */ +export function comparableVersion(version: string): string | null { + for (const candidate of versionCandidates(version)) { + const valid = semver.valid(candidate); + if (valid) return valid; + } + return null; } +/** + * True when both versions are semver and `latest` is older than `current`. + */ export function isOlderVersion(latest: string, current: string): boolean { - const a = comparableVersion(latest); - const b = comparableVersion(current); - return Boolean(a && b && semver.lt(a, b)); + const latestSemver = comparableVersion(latest); + const currentSemver = comparableVersion(current); + if (!latestSemver || !currentSemver) return false; + return semver.lt(latestSemver, currentSemver); } +/** + * Returns the highest `MAJOR.MINOR.PATCH` tag, allowing a leading `v` and a + * pre-release or build suffix. Stable tags win over pre-releases, which are + * only used when there is no stable tag. On equal versions the tag without `v` + * wins. + */ export function highestSemverTag(tags: string[]): string | null { - const valid = tags.filter( + const semverTags = tags.filter( tag => /^v?\d+\.\d+\.\d+(?:[-+].*)?$/.test(tag) && semver.valid(tag) ); - const stable = valid.filter(tag => semver.prerelease(tag) === null); - return ( - (stable.length ? stable : valid).sort( - (a, b) => - semver.rcompare(a, b) || - Number(a.startsWith("v")) - Number(b.startsWith("v")) - )[0] ?? null - ); + const stableTags = semverTags.filter(tag => semver.prerelease(tag) === null); + const candidates = stableTags.length > 0 ? stableTags : semverTags; + + const [highest] = [...candidates].sort(compareTagsDescending); + return highest ?? null; +} + +function compareTagsDescending(a: string, b: string): number { + const byVersion = semver.rcompare(a, b); + if (byVersion !== 0) return byVersion; + return Number(a.startsWith("v")) - Number(b.startsWith("v")); } diff --git a/src/commands/githubActions/bumpUpstream/index.ts b/src/commands/githubActions/bumpUpstream/index.ts index 1c8ea40e..5205ffc7 100644 --- a/src/commands/githubActions/bumpUpstream/index.ts +++ b/src/commands/githubActions/bumpUpstream/index.ts @@ -5,8 +5,7 @@ import { shell } from "../../../utils/shell.js"; import { getGitHead } from "../../../utils/git.js"; import { buildAndComment } from "../build/index.js"; import { readManifest } from "../../../files/index.js"; -import { parseProvider } from "../../../bump/metadata.js"; -import { packageManifestDirs } from "../../../bump/discovery.js"; +import { getPackageManifestDirs } from "../../../bump/discovery.js"; import { planPackageEdits } from "../../../bump/plan.js"; import { writePlan } from "../../../bump/edits.js"; import { printBumpDeprecation } from "./deprecation.js"; @@ -21,7 +20,6 @@ import { isBranchNew, recoverBumpBranch } from "./github/index.js"; -import path from "path"; interface CliCommandOptions extends CliGlobalOptions { eth_provider: string; @@ -93,11 +91,10 @@ async function gaBumpUpstreamHandler({ console.log( "--use_variants is ignored; variants are detected automatically" ); - const manifestDirs = packageManifestDirs(dir, variantsDir); - const allVariants = manifestDirs.some( - packageDir => path.resolve(packageDir) !== path.resolve(dir) - ); - const { upstreamSettings, gitSettings, ethProvider } = + // Variants are detected from the manifests in variantsDir + const allVariants = !getPackageManifestDirs(dir, variantsDir).includes(dir); + + const { upstreamSettings, resolvedUpstreams, gitSettings, ethProvider } = await getInitialSettings({ dir, userEthProvider, @@ -146,15 +143,7 @@ async function gaBumpUpstreamHandler({ const plan = await planPackageEdits({ dir, variantsDir, - upstreams: upstreamSettings.map(item => ({ - repo: item.repo, - arg: item.arg, - provider: parseProvider(item.provider), - tagPattern: item.tagPattern, - version: item.manifestVersion, - targetVersion: item.githubVersion, - composeValue: item.composeValue || item.githubVersion - })), + upstreams: resolvedUpstreams, nextPackageVersion: packageDir => getNewManifestVersion({ dir: packageDir, ethProvider }) }); @@ -185,9 +174,7 @@ async function gaBumpUpstreamHandler({ commitSha: gitHead.commit, branch: branchName, all_variants: allVariants, - variants_dir_name: allVariants - ? path.relative(dir, path.dirname(manifestDirs[0])) - : variantsDir + variants_dir_name: variantsDir }); } diff --git a/src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts b/src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts index 6e2cdc0a..acb66558 100644 --- a/src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts +++ b/src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts @@ -1,8 +1,9 @@ import { readManifest, readCompose } from "../../../../files/index.js"; import { getFirstAvailableEthProvider } from "../../../../utils/tryEthProviders.js"; -import { InitialSetupData, GitSettings } from "../types.js"; +import { InitialSetupData, GitSettings, UpstreamSettings } from "../types.js"; import { resolvePackage } from "../../../../bump/plan.js"; import { RegistryClient } from "../../../../bump/registry.js"; +import { ResolvedUpstream } from "../../../../bump/resolve.js"; import { fetchGithubUpstreamVersion } from "../github/fetchGithubUpstreamVersion.js"; export async function getInitialSettings({ @@ -19,7 +20,7 @@ export async function getInitialSettings({ const { manifest, format } = readManifest([{ dir }]); const compose = readCompose([{ dir }]); - const resolved = await resolvePackage( + const resolvedUpstreams = await resolvePackage( dir, { latestRelease: fetchGithubUpstreamVersion, @@ -27,17 +28,10 @@ export async function getInitialSettings({ }, explicitVersion ); - const upstreamSettings = resolved.length - ? resolved.map(item => ({ - repo: item.repo, - arg: item.arg, - provider: item.provider, - tagPattern: item.tagPattern, - manifestVersion: item.version, - githubVersion: item.targetVersion, - composeValue: item.composeValue - })) - : null; + const upstreamSettings = + resolvedUpstreams.length > 0 + ? resolvedUpstreams.map(toUpstreamSettings) + : null; const gitSettings = getGitSettings(); @@ -52,6 +46,7 @@ export async function getInitialSettings({ return { upstreamSettings, + resolvedUpstreams, manifestData: { manifest, format }, compose, gitSettings, @@ -59,6 +54,18 @@ export async function getInitialSettings({ }; } +function toUpstreamSettings(upstream: ResolvedUpstream): UpstreamSettings { + return { + repo: upstream.repo, + arg: upstream.arg, + provider: upstream.provider, + tagPattern: upstream.tagPattern, + manifestVersion: upstream.version, + githubVersion: upstream.targetVersion, + composeValue: upstream.composeValue + }; +} + function getEthProviders( useFallback: boolean, userEthProvider: string diff --git a/src/commands/githubActions/bumpUpstream/types.ts b/src/commands/githubActions/bumpUpstream/types.ts index 62796da0..e2220468 100644 --- a/src/commands/githubActions/bumpUpstream/types.ts +++ b/src/commands/githubActions/bumpUpstream/types.ts @@ -1,5 +1,6 @@ import { Manifest, Compose } from "@dappnode/types"; import { BumpUpstreamItem } from "../../../bump/metadata.js"; +import { ResolvedUpstream } from "../../../bump/resolve.js"; import { ManifestFormat } from "../../../files/manifest/types.js"; import { Github } from "../../../providers/github/Github.js"; @@ -28,6 +29,7 @@ export interface GitBranch { export interface InitialSetupData { upstreamSettings: UpstreamSettings[] | null; + resolvedUpstreams: ResolvedUpstream[]; manifestData: { manifest: Manifest; format: ManifestFormat; diff --git a/src/spec/checkPackageSpec.ts b/src/spec/checkPackageSpec.ts index 8cb9fec2..8404f59b 100644 --- a/src/spec/checkPackageSpec.ts +++ b/src/spec/checkPackageSpec.ts @@ -1,8 +1,3 @@ -import { - BumpUpstreamItem, - compileTagPattern, - parseProvider -} from "../bump/metadata.js"; import fs from "fs"; import path from "path"; import yaml from "js-yaml"; @@ -12,6 +7,12 @@ import { PackageToBuildProps } from "../types.js"; import { defaultComposeFileName, defaultVariantsDirName } from "../params.js"; import { parseGithubRepoSlug } from "../files/manifest/getRepoSlugFromManifest.js"; import { shell } from "../utils/shell.js"; +import { BumpManifest } from "../bump/discovery.js"; +import { + BumpUpstreamItem, + compileTagPattern, + parseProvider +} from "../bump/metadata.js"; import { SpecFinding, SpecRuleId } from "./rules.js"; import { checkManifestReferences, @@ -348,23 +349,18 @@ export function checkPackageSpec({ const sourceManifest = mergedSourceManifest(pkg) || manifest; const buildArgs = collectBuildArgs(compose); const usesUpstreamArray = Array.isArray(sourceManifest.upstream); - const upstreams: (Partial & { repo: string })[] = - usesUpstreamArray - ? sourceManifest.upstream || [] - : sourceManifest.upstreamRepo - ? [ - { - repo: sourceManifest.upstreamRepo, - version: sourceManifest.upstreamVersion, - arg: sourceManifest.upstreamArg, - provider: ( - sourceManifest as Partial & { - upstreamProvider?: string; - } - ).upstreamProvider - } - ] - : []; + const upstreams: Partial[] = usesUpstreamArray + ? sourceManifest.upstream || [] + : sourceManifest.upstreamRepo + ? [ + { + repo: sourceManifest.upstreamRepo, + version: sourceManifest.upstreamVersion, + arg: sourceManifest.upstreamArg, + provider: (sourceManifest as BumpManifest).upstreamProvider + } + ] + : []; const upstreamDir = manifestOrigin(m => usesUpstreamArray ? "upstream" in m @@ -372,20 +368,12 @@ export function checkPackageSpec({ ); for (const item of upstreams) { if (!item?.repo) continue; - for (const validate of [ - () => parseProvider(item.provider), - () => compileTagPattern(item.tagPattern) - ]) { - try { - validate(); - } catch (error) { - add( - "SPEC-M12", - `Upstream ${item.repo}${where(upstreamDir)}: ${error.message}`, - manifestFileIn(upstreamDir) - ); - } - } + for (const error of getBumpMetadataErrors(item)) + add( + "SPEC-M12", + `Upstream ${item.repo}${where(upstreamDir)}: ${error}`, + manifestFileIn(upstreamDir) + ); // The legacy fields default the arg to UPSTREAM_VERSION; check the version against it too. const upstream = { ...item, arg: item.arg || "UPSTREAM_VERSION" }; if (!item.arg) { @@ -646,6 +634,24 @@ function mergeByPosition(parts: T[]): T { } /** Build arg name -> service -> value; list-style "KEY=value" args included. */ +/** + * Errors in the optional upstream bump metadata (`provider`, `tagPattern`). + */ +function getBumpMetadataErrors(item: Partial): string[] { + const errors: string[] = []; + try { + parseProvider(item.provider); + } catch (e) { + errors.push(e.message); + } + try { + compileTagPattern(item.tagPattern); + } catch (e) { + errors.push(e.message); + } + return errors; +} + function collectBuildArgs(compose: Compose): Map> { const args = new Map>(); const put = (service: string, key: string, value: unknown) => { diff --git a/test/bump/compareEngines.ts b/test/bump/compareEngines.ts index c74c4967..ce4e4765 100644 --- a/test/bump/compareEngines.ts +++ b/test/bump/compareEngines.ts @@ -7,7 +7,7 @@ import { execFileSync } from "node:child_process"; import { parseArgs } from "node:util"; import assert from "node:assert/strict"; import semver from "semver"; -import { packageManifestDirs } from "../../src/bump/discovery.js"; +import { getPackageManifestDirs } from "../../src/bump/discovery.js"; import { editManifest, writePlan } from "../../src/bump/edits.js"; import { planPackageEdits, resolvePackage } from "../../src/bump/plan.js"; import { RegistryClient } from "../../src/bump/registry.js"; @@ -91,7 +91,7 @@ const referenceOutput = execFileSync("docker", dockerArgs, { timeout: 180000 }); fs.writeFileSync(path.join(scratch, "reference.json"), referenceOutput); -const packageManifests = packageManifestDirs(sdk).map(dir => +const packageManifests = getPackageManifestDirs(sdk).map(dir => path.relative(sdk, path.join(dir, "dappnode_package.json")) ); const files = [ diff --git a/test/bump/deprecation.test.ts b/test/bump/deprecation.test.ts deleted file mode 100644 index 157fc20c..00000000 --- a/test/bump/deprecation.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { expect } from "chai"; -import { printBumpDeprecation } from "../../src/commands/githubActions/bumpUpstream/deprecation.js"; -import { getBumpPrBody } from "../../src/commands/githubActions/bumpUpstream/github/getBumpPrBody.js"; - -describe("bump migration and PR metadata", () => { - it("logs the deprecation and an Actions warning with both migration paths", () => { - const lines: string[] = []; - printBumpDeprecation(line => lines.push(line)); - expect(lines).to.have.length(2); - expect(lines[1]).to.equal( - `::warning title=dappnodesdk bump-upstream is deprecated::${lines[0]}` - ); - expect(lines[0]) - .to.include( - "dappnode/workflows/.github/workflows/bump-upstream.yml@master" - ) - .and.to.include("dappnode/workflows/actions/bump-upstream@master"); - }); - - it("includes Explorer metadata and the exact Compose pin", () => { - const value = `v2.0.0@sha256:${"a".repeat(64)}`; - const body = getBumpPrBody([ - { - repo: "acme/app", - arg: "VERSION", - manifestVersion: "v1.0.0", - githubVersion: "v2.0.0", - composeValue: value - } - ]); - expect(body).to.include(`Compose: \`${value}\``); - expect(body.split("\n")).to.include( - '' - ); - }); -}); diff --git a/test/bump/discovery.test.ts b/test/bump/discovery.test.ts index 0e1058cb..48ea1dd7 100644 --- a/test/bump/discovery.test.ts +++ b/test/bump/discovery.test.ts @@ -2,133 +2,159 @@ import { expect } from "chai"; import fs from "node:fs"; import path from "node:path"; import { + BumpCompose, discoverUpstreams, - packageManifestDirs, - BumpCompose + getPackageManifestDirs } from "../../src/bump/discovery.js"; -import { - compileTagPattern, - parseProvider, - extractVersion -} from "../../src/bump/metadata.js"; import { temporaryDirectory } from "../helpers/fixtures.js"; const compose: BumpCompose = { services: { app: { build: { args: { APP_TAG: "1.0.0" } } } } }; -describe("bump discovery and metadata", () => { - it("discovers array and legacy fields, inferring a single arg", () => { - expect( - discoverUpstreams( +describe("bump / discovery", () => { + describe("discoverUpstreams", () => { + it("reads upstream[] and infers the arg of a single upstream", () => { + const upstreams = discoverUpstreams( { upstream: [{ repo: "acme/app", version: "1.0.0" }] }, compose - ) - ).to.deep.equal([ - { repo: "acme/app", version: "1.0.0", arg: "APP_TAG", provider: "docker" } - ]); - expect( - discoverUpstreams( + ); + + expect(upstreams).to.deep.equal([ + { + repo: "acme/app", + version: "1.0.0", + arg: "APP_TAG", + provider: "docker" + } + ]); + }); + + it("reads the legacy fields, including upstreamProvider", () => { + const [upstream] = discoverUpstreams( { upstreamRepo: "acme/app", upstreamVersion: "1.0.0", upstreamProvider: "GITHUB" }, compose - )[0] - ).to.include({ arg: "APP_TAG", provider: "github" }); - expect( - discoverUpstreams( + ); + + expect(upstream).to.include({ arg: "APP_TAG", provider: "github" }); + }); + + it("defaults the arg to UPSTREAM_VERSION", () => { + const [upstream] = discoverUpstreams( { upstreamRepo: "acme/app", upstreamVersion: "1.0.0" }, { services: {} } - )[0].arg - ).to.equal("UPSTREAM_VERSION"); - expect(discoverUpstreams({}, compose)).to.deep.equal([]); - }); + ); - it("requires explicit multi-upstream args, and unique repos and args", () => { - const upstream = [ - { repo: "acme/a", version: "1", arg: "A" }, - { repo: "acme/b", version: "2", arg: "B" } - ]; - expect(discoverUpstreams({ upstream }, compose)).to.have.length(2); - expect(() => - discoverUpstreams( - { upstream: [upstream[0], { ...upstream[1], arg: undefined }] }, - compose - ) - ).to.throw("multi-upstream"); - expect(() => - discoverUpstreams( - { upstream: [upstream[0], { ...upstream[1], repo: "acme/a" }] }, - compose - ) - ).to.throw("repositories must be unique"); - expect(() => - discoverUpstreams( - { upstream: [upstream[0], { ...upstream[1], arg: "A" }] }, - compose - ) - ).to.throw("args must be unique"); - expect(() => - discoverUpstreams( - { upstreamRepo: "acme/app", upstreamVersion: "1" }, - { services: { app: { build: { args: ["A_VERSION=1", "B_TAG=2"] } } } } - ) - ).to.throw("uniquely detect"); - }); + expect(upstream.arg).to.equal("UPSTREAM_VERSION"); + }); + + it("returns no upstreams for a manifest without any", () => { + expect(discoverUpstreams({}, compose)).to.deep.equal([]); + }); + + describe("with several upstreams", () => { + const upstream = [ + { repo: "acme/a", version: "1", arg: "A" }, + { repo: "acme/b", version: "2", arg: "B" } + ]; + + it("reads every entry", () => { + expect(discoverUpstreams({ upstream }, compose)).to.have.length(2); + }); + + it("requires each entry to name its arg", () => { + const manifest = { + upstream: [upstream[0], { ...upstream[1], arg: undefined }] + }; + expect(() => discoverUpstreams(manifest, compose)).to.throw( + "multi-upstream" + ); + }); + + it("requires unique repositories", () => { + const manifest = { + upstream: [upstream[0], { ...upstream[1], repo: "acme/a" }] + }; + expect(() => discoverUpstreams(manifest, compose)).to.throw( + "repositories must be unique" + ); + }); - it("normalizes providers and rejects invalid regex constructs", () => { - for (const name of [ - "docker_tags", - "DOCKER_TAGS", - "docker-tags", - "dockerhub" - ]) - expect(parseProvider(name)).to.equal("docker_tags"); - expect(parseProvider(undefined)).to.equal("docker"); - expect(() => parseProvider("npm")).to.throw("Unknown upstream provider"); - for (const pattern of [ - "[", - "(?=a)", - "(?<=a)b", - "(a)\\1", - "(?a)\\k" - ]) - expect(() => compileTagPattern(pattern), pattern).to.throw(); - expect(extractVersion("op-node/v1.2.3", "^op-node/(v.*)$")).to.equal( - "v1.2.3" - ); - expect(extractVersion("v1.2.3", "v.*")).to.equal("v1.2.3"); - expect(extractVersion("other/v1", "^node/(.*)$")).to.equal(null); - expect(compileTagPattern(String.raw`^a\\1$`)).to.be.instanceOf(RegExp); + it("requires unique args", () => { + const manifest = { + upstream: [upstream[0], { ...upstream[1], arg: "A" }] + }; + expect(() => discoverUpstreams(manifest, compose)).to.throw( + "args must be unique" + ); + }); + }); + + it("fails when several version args could be the upstream arg", () => { + const manifest = { upstreamRepo: "acme/app", upstreamVersion: "1" }; + const listArgsCompose = { + services: { app: { build: { args: ["A_VERSION=1", "B_TAG=2"] } } } + }; + + expect(() => discoverUpstreams(manifest, listArgsCompose)).to.throw( + "uniquely detect" + ); + }); }); - it("detects variant manifests automatically and supports legacy directories", () => { - const dir = temporaryDirectory(); - try { - expect(packageManifestDirs(dir)).to.deep.equal([dir]); + describe("getPackageManifestDirs", () => { + let dir: string; + + beforeEach(() => { + dir = temporaryDirectory(); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function addVariants(variantsDirName: string): void { for (const variant of ["mainnet", "testnet"]) { - const base = path.join(dir, "package_variants", variant); - fs.mkdirSync(base, { recursive: true }); - fs.writeFileSync(path.join(base, "dappnode_package.json"), "{}"); + const variantDir = path.join(dir, variantsDirName, variant); + fs.mkdirSync(variantDir, { recursive: true }); + fs.writeFileSync(path.join(variantDir, "dappnode_package.json"), "{}"); } - expect( - packageManifestDirs(dir).map(file => path.basename(file)) - ).to.deep.equal(["mainnet", "testnet"]); - fs.renameSync( - path.join(dir, "package_variants"), - path.join(dir, "legacy") - ); - expect(packageManifestDirs(dir, "legacy")).to.have.length(2); + } + + it("returns the root for a package without variants", () => { + expect(getPackageManifestDirs(dir)).to.deep.equal([dir]); + }); + + it("returns each variant in package_variants", () => { + addVariants("package_variants"); + + const dirs = getPackageManifestDirs(dir); + + expect(dirs.map(variantDir => path.basename(variantDir))).to.deep.equal([ + "mainnet", + "testnet" + ]); + }); + + it("supports a custom variants directory", () => { + addVariants("legacy"); + + expect(getPackageManifestDirs(dir, "legacy")).to.have.length(2); + }); + + it("fails for a variant without manifest", () => { + addVariants("package_variants"); fs.unlinkSync( - path.join(dir, "legacy", "mainnet", "dappnode_package.json") + path.join(dir, "package_variants", "mainnet", "dappnode_package.json") ); - expect(() => packageManifestDirs(dir, "legacy")).to.throw( + + expect(() => getPackageManifestDirs(dir)).to.throw( "Variant has no manifest" ); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } + }); }); }); diff --git a/test/bump/dockerfile.test.ts b/test/bump/dockerfile.test.ts new file mode 100644 index 00000000..c6005de4 --- /dev/null +++ b/test/bump/dockerfile.test.ts @@ -0,0 +1,94 @@ +import { expect } from "chai"; +import fs from "node:fs"; +import path from "node:path"; +import { + extractImageSource, + findImageSource, + renderDockerTag +} from "../../src/bump/dockerfile.js"; +import { temporaryDirectory } from "../helpers/fixtures.js"; + +describe("bump / dockerfile", () => { + describe("extractImageSource", () => { + it("expands indirect args from their static defaults", () => { + const dockerfile = [ + "ARG BASE_IMAGE=statusim/nimbus-eth2", + "ARG ARCH=multiarch", + "ARG BASE_TAG=${ARCH}-${VERSION}", + "FROM --platform=$TARGETPLATFORM ${BASE_IMAGE}:${BASE_TAG} AS base" + ].join("\n"); + + const source = extractImageSource(dockerfile, "VERSION"); + + expect(source).to.deep.equal({ + image: "statusim/nimbus-eth2", + template: "multiarch-{version}" + }); + expect(renderDockerTag(source!, "v26.10.0")).to.equal( + "multiarch-v26.10.0" + ); + }); + + it("keeps a registry port in the image name", () => { + expect( + extractImageSource("FROM host:5000/org/app:$VERSION", "VERSION") + ).to.deep.equal({ image: "host:5000/org/app", template: "{version}" }); + }); + + it("supports prefix removal", () => { + const source = extractImageSource("FROM app:${VERSION#v}", "VERSION"); + + expect(source).to.deep.equal({ image: "app", template: "{version#v}" }); + expect(renderDockerTag(source!, "v1.2.3")).to.equal("1.2.3"); + }); + + it("returns null when no FROM tag uses the arg", () => { + expect( + extractImageSource( + "FROM node:22\nRUN npm install app@${VERSION}", + "VERSION" + ) + ).to.equal(null); + expect(extractImageSource("FROM app:$VERSION_EXTRA", "VERSION")).to.equal( + null + ); + }); + }); + + describe("findImageSource", () => { + let dir: string; + + beforeEach(() => { + dir = temporaryDirectory(); + fs.mkdirSync(path.join(dir, "build")); + fs.writeFileSync( + path.join(dir, "build", "custom.dockerfile"), + "FROM org/app:$VERSION\n" + ); + }); + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + }); + + for (const args of [{ VERSION: "1" }, ["VERSION=1"]]) { + it(`reads the Dockerfile of the service declaring the arg (${JSON.stringify( + args + )})`, () => { + const compose = { + services: { + unrelated: { build: "." }, + app: { + build: { context: "build", dockerfile: "custom.dockerfile", args } + } + } + }; + + expect(findImageSource(dir, compose, "VERSION")).to.deep.equal({ + image: "org/app", + template: "{version}" + }); + }); + } + }); +}); diff --git a/test/bump/edits.test.ts b/test/bump/edits.test.ts index d7825a55..040f1235 100644 --- a/test/bump/edits.test.ts +++ b/test/bump/edits.test.ts @@ -25,7 +25,7 @@ const upstream: ResolvedUpstream = { const compose = "# header\nservices:\n app:\n build:\n args: {VERSION: \"v1.0.0\", STATIC: keep} # comment\n worker:\n build:\n args:\n - 'VERSION=v1.0.0' # worker\n - OTHER=keep\n"; -describe("bump targeted edits and planning", () => { +describe("bump / edits", () => { it("preserves mapping/list args, quotes, comments, flow styles, CRLF and unrelated bytes", () => { for (const before of [compose, compose.replace(/\n/g, "\r\n")]) { const after = editCompose(before, new Map([["VERSION", "v2.0.0"]])); diff --git a/test/bump/metadata.test.ts b/test/bump/metadata.test.ts new file mode 100644 index 00000000..d17898b6 --- /dev/null +++ b/test/bump/metadata.test.ts @@ -0,0 +1,66 @@ +import { expect } from "chai"; +import { + compileTagPattern, + extractVersion, + parseProvider +} from "../../src/bump/metadata.js"; + +describe("bump / metadata", () => { + describe("parseProvider", () => { + it("defaults to docker", () => { + expect(parseProvider(undefined)).to.equal("docker"); + expect(parseProvider(" ")).to.equal("docker"); + }); + + it("accepts docker_tags aliases case-insensitively", () => { + for (const name of [ + "docker_tags", + "DOCKER_TAGS", + "docker-tags", + "dockerhub" + ]) + expect(parseProvider(name)).to.equal("docker_tags"); + }); + + it("rejects unknown providers", () => { + expect(() => parseProvider("npm")).to.throw("Unknown upstream provider"); + expect(() => parseProvider("constructor")).to.throw( + "Unknown upstream provider" + ); + }); + }); + + describe("compileTagPattern", () => { + for (const pattern of [ + "[", + "(?=a)", + "(?<=a)b", + "(a)\\1", + "(?a)\\k" + ]) { + it(`rejects ${pattern}`, () => { + expect(() => compileTagPattern(pattern)).to.throw(); + }); + } + + it("accepts an escaped backslash followed by a digit", () => { + expect(compileTagPattern(String.raw`^a\\1$`)).to.be.instanceOf(RegExp); + }); + }); + + describe("extractVersion", () => { + it("returns capture group 1", () => { + expect(extractVersion("op-node/v1.2.3", "^op-node/(v.*)$")).to.equal( + "v1.2.3" + ); + }); + + it("returns the whole match when the pattern has no group", () => { + expect(extractVersion("v1.2.3", "v.*")).to.equal("v1.2.3"); + }); + + it("returns null when the tag does not match", () => { + expect(extractVersion("other/v1", "^node/(.*)$")).to.equal(null); + }); + }); +}); diff --git a/test/bump/registry.test.ts b/test/bump/registry.test.ts index 8ff0d75a..5c2e0046 100644 --- a/test/bump/registry.test.ts +++ b/test/bump/registry.test.ts @@ -10,12 +10,12 @@ import { createHash } from "node:crypto"; import { manifestAccept, RegistryClient, - registryImage + parseRegistryImage } from "../../src/bump/registry.js"; const digest = `sha256:${"b".repeat(64)}`; -describe("bump anonymous registry client", () => { +describe("bump / registry", () => { let server: Server; let host: string; let handler: (request: IncomingMessage, response: ServerResponse) => void; @@ -54,17 +54,23 @@ describe("bump anonymous registry client", () => { "docker.io/alpine", "index.docker.io/library/alpine" ]) - expect(registryImage(image)).to.deep.equal({ + expect(parseRegistryImage(image)).to.deep.equal({ host: "registry-1.docker.io", name: "library/alpine", - hub: true + isDockerHub: true }); - expect(registryImage("org/app").name).to.equal("org/app"); - expect(registryImage("ghcr.io/org/app")).to.deep.equal({ + expect(parseRegistryImage("org/app").name).to.equal("org/app"); + expect(parseRegistryImage("ghcr.io/org/app")).to.deep.equal({ host: "ghcr.io", name: "org/app", - hub: false + isDockerHub: false }); + for (const host of ["evil-docker.io", "docker.io.evil.com"]) + expect(parseRegistryImage(`${host}/org/app`)).to.deep.equal({ + host, + name: "org/app", + isDockerHub: false + }); }); it("answers a bearer challenge, using token or access_token and pull scope", async () => { diff --git a/test/bump/resolve.test.ts b/test/bump/resolve.test.ts index d1e2bc56..d43e277d 100644 --- a/test/bump/resolve.test.ts +++ b/test/bump/resolve.test.ts @@ -1,10 +1,10 @@ import { expect } from "chai"; import { BumpCompose, Upstream } from "../../src/bump/discovery.js"; import { - currentPin, + getCurrentPin, ResolutionDependencies, resolveUpstream, - splitPin + parsePin } from "../../src/bump/resolve.js"; const digest = `sha256:${"a".repeat(64)}`; @@ -48,19 +48,19 @@ function dependencies( }; } -describe("bump resolution and pinning", () => { +describe("bump / resolve", () => { it("splits only the last @sha256 suffix and enforces lowercase digests", () => { - expect(splitPin(`n8n@2.10.3@${digest}`)).to.deep.equal({ + expect(parsePin(`n8n@2.10.3@${digest}`)).to.deep.equal({ version: "n8n@2.10.3", digest }); - expect(splitPin("n8n@2.10.3")).to.deep.equal({ version: "n8n@2.10.3" }); + expect(parsePin("n8n@2.10.3")).to.deep.equal({ version: "n8n@2.10.3" }); for (const value of [ `1@sha256:${"A".repeat(64)}`, "1@sha256:short", `@${digest}` ]) - expect(() => splitPin(value)).to.throw("Invalid digest pin"); + expect(() => parsePin(value)).to.throw("Invalid digest pin"); }); it("maps candidate tags and writes the candidate rather than its rendered template", async () => { @@ -167,19 +167,21 @@ describe("bump resolution and pinning", () => { b: { build: { args: [`VERSION=${b}`] } } } }); - expect(() => currentPin(mixed("1.0.0", "0.9.0"), upstream)).to.throw( + expect(() => getCurrentPin(mixed("1.0.0", "0.9.0"), upstream)).to.throw( "Inconsistent Compose pin" ); expect(() => - currentPin(mixed("1.0.0", `1.0.0@${digest}`), upstream) + getCurrentPin(mixed("1.0.0", `1.0.0@${digest}`), upstream) ).to.throw("Inconsistent digest pins"); expect(() => - currentPin( + getCurrentPin( mixed(`1.0.0@${digest}`, `1.0.0@sha256:${"b".repeat(64)}`), upstream ) ).to.throw("Inconsistent digest pins"); - expect(() => currentPin({ services: {} }, upstream)).to.throw("is absent"); + expect(() => getCurrentPin({ services: {} }, upstream)).to.throw( + "is absent" + ); }); it("fails on absent candidates and registry errors", async () => { diff --git a/test/bump/versions.test.ts b/test/bump/versions.test.ts index 627596ab..69e8a410 100644 --- a/test/bump/versions.test.ts +++ b/test/bump/versions.test.ts @@ -1,143 +1,54 @@ import { expect } from "chai"; -import fs from "node:fs"; -import path from "node:path"; -import { - extractImageSource, - findImageSource, - renderDockerTag -} from "../../src/bump/dockerfile.js"; import { highestSemverTag, isOlderVersion, versionCandidates } from "../../src/bump/versions.js"; -import { pickLatestRelease } from "../../src/commands/githubActions/bumpUpstream/github/pickLatestRelease.js"; -import { temporaryDirectory } from "../helpers/fixtures.js"; -const release = (tag_name: string, draft = false, prerelease = false) => ({ - tag_name, - draft, - prerelease -}); +describe("bump / versions", () => { + describe("versionCandidates", () => { + it("tries the tag before its bare semver", () => { + expect(versionCandidates("v1.2.3")).to.deep.equal(["v1.2.3", "1.2.3"]); + expect(versionCandidates("n8n@2.10.3")).to.deep.equal([ + "n8n@2.10.3", + "2.10.3" + ]); + }); -describe("bump version selection and Dockerfile templates", () => { - it("selects the first eligible release with optional tag extraction", () => { - const releases = [ - release("v9.0.0", true), - release("v8.0.0", false, true), - release("v7.0.0-NiGhTlY"), - release("n8n@6.0.0-rc.1"), - release("op-deployer/v0.9.0-allowlist"), - release("op-node/v1.2.3"), - release("op-node/v1.1.0") - ]; - expect(pickLatestRelease(releases, "^op-node/(v.*)$")).to.equal("v1.2.3"); - // Without a pattern another tag family wins: `-allowlist` is not a pre-release word. - expect(pickLatestRelease(releases)).to.equal( - "op-deployer/v0.9.0-allowlist" - ); - const warn = console.warn; - const warnings: string[] = []; - try { - console.warn = message => warnings.push(message); - expect(pickLatestRelease([release("release_2026")])).to.equal( - "release_2026" - ); - expect( - pickLatestRelease([release("bad tag"), release("stable")]) - ).to.equal("stable"); - } finally { - console.warn = warn; - } - expect(warnings).to.have.length(2); - expect(pickLatestRelease([release("v2.0.0-rc.1")])).to.equal(null); - expect(pickLatestRelease([release("nightly/v1.0.0")], "(v.*)$")).to.equal( - null - ); + it("returns bare and non-semver tags as they are", () => { + expect(versionCandidates("1.2.3")).to.deep.equal(["1.2.3"]); + expect(versionCandidates("stable")).to.deep.equal(["stable"]); + }); }); - it("tries the original tag before a bare semver suffix", () => { - expect(versionCandidates("v1.2.3")).to.deep.equal(["v1.2.3", "1.2.3"]); - expect(versionCandidates("n8n@2.10.3")).to.deep.equal([ - "n8n@2.10.3", - "2.10.3" - ]); - expect(versionCandidates("1.2.3")).to.deep.equal(["1.2.3"]); - expect(versionCandidates("stable")).to.deep.equal(["stable"]); - expect(isOlderVersion("v1.2.3", "n8n@2.10.3")).to.equal(true); - expect(isOlderVersion("stable", "v2.0.0")).to.equal(false); - }); + describe("isOlderVersion", () => { + it("compares versions after removing their prefix", () => { + expect(isOlderVersion("v1.2.3", "n8n@2.10.3")).to.equal(true); + }); - it("prefers stable image tags, highest semver, and bare tags on ties", () => { - expect( - highestSemverTag(["latest", "3.0.0-rc.1", "v2.0.0", "1.9.0", "2.0.0"]) - ).to.equal("2.0.0"); - expect(highestSemverTag(["v1.0.0-rc.1", "1.0.0-rc.2"])).to.equal( - "1.0.0-rc.2" - ); - expect(highestSemverTag(["latest", "1.0", "n8n@1.2.3"])).to.equal(null); - expect(highestSemverTag(["1.0.0+build", "v0.9.0"])).to.equal("1.0.0+build"); + it("never reports a non-semver version as older", () => { + expect(isOlderVersion("stable", "v2.0.0")).to.equal(false); + }); }); - it("expands direct, indirect and static Dockerfile arguments", () => { - const source = extractImageSource( - "ARG BASE_IMAGE=statusim/nimbus-eth2\nARG ARCH=multiarch\nARG BASE_TAG=${ARCH}-${VERSION}\nFROM --platform=$TARGETPLATFORM ${BASE_IMAGE}:${BASE_TAG} AS base\n", - "VERSION" - ); - expect(source).to.deep.equal({ - image: "statusim/nimbus-eth2", - template: "multiarch-{version}" + describe("highestSemverTag", () => { + it("prefers stable tags, then the highest version, then the tag without v", () => { + expect( + highestSemverTag(["latest", "3.0.0-rc.1", "v2.0.0", "1.9.0", "2.0.0"]) + ).to.equal("2.0.0"); }); - expect(renderDockerTag(source!, "v26.10.0")).to.equal("multiarch-v26.10.0"); - expect( - extractImageSource("FROM host:5000/org/app:$VERSION", "VERSION") - ).to.deep.equal({ image: "host:5000/org/app", template: "{version}" }); - expect( - extractImageSource("FROM app:${VERSION#v}", "VERSION") - ).to.deep.equal({ image: "app", template: "{version#v}" }); - expect( - renderDockerTag({ image: "app", template: "{version#v}" }, "v1.2.3") - ).to.equal("1.2.3"); - expect( - extractImageSource( - "FROM node:22\nRUN npm install app@${VERSION}", - "VERSION" - ) - ).to.equal(null); - expect(extractImageSource("FROM app:$VERSION_EXTRA", "VERSION")).to.equal( - null - ); - }); - it("finds the service context and Dockerfile with mapping or list args", () => { - const dir = temporaryDirectory(); - try { - fs.mkdirSync(path.join(dir, "build")); - fs.writeFileSync( - path.join(dir, "build", "custom.dockerfile"), - "FROM org/app:$VERSION\n" + it("uses pre-releases only when there is no stable tag", () => { + expect(highestSemverTag(["v1.0.0-rc.1", "1.0.0-rc.2"])).to.equal( + "1.0.0-rc.2" ); - for (const args of [{ VERSION: "1" }, ["VERSION=1"]]) - expect( - findImageSource( - dir, - { - services: { - unrelated: { build: "." }, - app: { - build: { - context: "build", - dockerfile: "custom.dockerfile", - args - } - } - } - }, - "VERSION" - ) - ).to.deep.equal({ image: "org/app", template: "{version}" }); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } + }); + + it("ignores tags that are not MAJOR.MINOR.PATCH", () => { + expect(highestSemverTag(["latest", "1.0", "n8n@1.2.3"])).to.equal(null); + expect(highestSemverTag(["1.0.0+build", "v0.9.0"])).to.equal( + "1.0.0+build" + ); + }); }); }); diff --git a/test/commands/gaBumpUpstream/deprecation.test.ts b/test/commands/gaBumpUpstream/deprecation.test.ts new file mode 100644 index 00000000..b098e0b4 --- /dev/null +++ b/test/commands/gaBumpUpstream/deprecation.test.ts @@ -0,0 +1,19 @@ +import { expect } from "chai"; +import { printBumpDeprecation } from "../../../src/commands/githubActions/bumpUpstream/deprecation.js"; + +describe("command / gaBumpUpstream / deprecation", () => { + it("logs the notice and an Actions warning with both migration paths", () => { + const lines: string[] = []; + printBumpDeprecation(line => lines.push(line)); + + expect(lines).to.have.length(2); + expect(lines[0]) + .to.include( + "dappnode/workflows/.github/workflows/bump-upstream.yml@master" + ) + .and.to.include("dappnode/workflows/actions/bump-upstream@master"); + expect(lines[1]).to.equal( + `::warning title=dappnodesdk bump-upstream is deprecated::${lines[0]}` + ); + }); +}); diff --git a/test/commands/gaBumpUpstream/format.test.ts b/test/commands/gaBumpUpstream/format.test.ts index 5bc0dafb..2da4edb3 100644 --- a/test/commands/gaBumpUpstream/format.test.ts +++ b/test/commands/gaBumpUpstream/format.test.ts @@ -56,6 +56,22 @@ describe("command / gaBumpUpstream / format", () => { }); }); + describe("digest-pinned version", () => { + it("getPrBody shows the pinned Compose value", () => { + const composeValue = `v2.0.0@sha256:${"a".repeat(64)}`; + const upstreamVersion = getBumpPrBody([ + { + repo: "acme/app", + arg: "VERSION", + manifestVersion: "v1.0.0", + githubVersion: "v2.0.0", + composeValue + } + ]); + expect(upstreamVersion).to.include(`(Compose: \`${composeValue}\`)`); + }); + }); + describe("Check valid releases", () => { const correctVersionsToUpdate: ComposeVersionsToUpdate = { "sigp/lighthouse": { diff --git a/test/commands/gaBumpUpstream/pickLatestRelease.test.ts b/test/commands/gaBumpUpstream/pickLatestRelease.test.ts index dd4b491e..d8e0fef0 100644 --- a/test/commands/gaBumpUpstream/pickLatestRelease.test.ts +++ b/test/commands/gaBumpUpstream/pickLatestRelease.test.ts @@ -58,12 +58,37 @@ describe("command / gaBumpUpstream / pickLatestRelease", () => { }); it("filters tag families with tagPattern", () => { - expect( - pickLatestRelease( - [release("op-deployer/v0.9.0-allowlist"), release("op-node/v1.20.1")], - "^op-node/v(.+)$" - ) - ).to.equal("1.20.1"); + const releases = [ + release("op-deployer/v0.9.0-allowlist"), + release("op-node/v1.20.1") + ]; + expect(pickLatestRelease(releases, "^op-node/v(.+)$")).to.equal("1.20.1"); + // `-allowlist` is not a pre-release word, so without a pattern the other + // tag family wins + expect(pickLatestRelease(releases)).to.equal( + "op-deployer/v0.9.0-allowlist" + ); + }); + + it("skips nightly tags even when tagPattern matches", () => { + expect(pickLatestRelease([release("nightly/v1.0.0")], "(v.*)$")).to.equal( + null + ); + }); + + it("accepts non-semver tags and skips tags with invalid characters", () => { + const warn = console.warn; + console.warn = () => undefined; + try { + expect(pickLatestRelease([release("release_2026")])).to.equal( + "release_2026" + ); + expect( + pickLatestRelease([release("bad tag"), release("stable")]) + ).to.equal("stable"); + } finally { + console.warn = warn; + } }); describe("selectLatestRelease", () => {