fix cache failures caused due to outdated buildah.

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 Notable changes.
 
+## Unreleased
+
+- Use COPY-based Feature installation for Podman with bundled Buildah older than 1.41.1, or when its version cannot be confirmed, to avoid named-context Feature cache misses. (https://github.com/devcontainers/cli/issues/1262)
+
 ## October 2026
 
 ### [0.90.0]

README.md

@@ -24,6 +24,10 @@ This CLI is in active development. Current status:
 
 Lockfiles (`.devcontainer-lock.json`) are generated by default when running `build` or `up` to pin feature versions for reproducible builds. Use `--no-lockfile` to opt out, or `--frozen-lockfile` to enforce an existing lockfile.
 
+### Podman Feature Build Caching
+
+Dev Containers CLI 0.90.0 and earlier may rerun unchanged Feature installation steps with Podman because of an upstream Buildah cache issue. Upgrade to a more recent CLI version and rebuild to get the compatibility fix.
+
 ## Try it out
 
 We'd love for you to try out the dev container CLI and let us know what you think. You can quickly try it out in just a few simple steps, either by using the install script, installing its npm package, or building the CLI repo from sources (see "[Build from sources](#build-from-sources)").

src/spec-node/containerFeatures.ts

@@ -241,7 +241,8 @@ async function getFeaturesBuildOptions(params: DockerResolverParameters, devCont
 	// TODO generate an image name that is specific to this dev container?
 	const buildKitVersionParsed = params.buildKitVersion?.versionMatch ? parseVersion(params.buildKitVersion.versionMatch) : undefined;
 	const minRequiredVersion = [0, 8, 0];
-	const useBuildKitBuildContexts = buildKitVersionParsed ? !isEarlierVersion(buildKitVersionParsed, minRequiredVersion) : false;
+	// Keep BuildKit enabled globally while using COPY for Features on affected Podman builders.
+	const useBuildKitBuildContexts = !params.useFeatureCopyFallback && (buildKitVersionParsed ? !isEarlierVersion(buildKitVersionParsed, minRequiredVersion) : false);
 	const buildContentImageName = 'dev_container_feature_content_temp';
 	const disableSELinuxLabels = useBuildKitBuildContexts && await isUsingSELinuxLabels(params);
     // Access Docker engine version

src/spec-node/devContainers.ts

@@ -18,7 +18,7 @@ import { LogLevel, LogDimensions, toErrorText, createCombinedLog, createTerminal
 import { dockerComposeCLIConfig } from './dockerCompose';
 import { Mount } from '../spec-configuration/containerFeaturesConfiguration';
 import { getPackageConfig, PackageConfiguration } from '../spec-utils/product';
-import { dockerBuildKitVersion, dockerEngineVersion, lookupCLIVariant, CLIVariant } from '../spec-shutdown/dockerUtils';
+import { dockerBuildKitVersion, dockerEngineVersion, lookupCLIVariant, CLIVariant, useFeatureCopyFallback } from '../spec-shutdown/dockerUtils';
 import { Event } from '../spec-utils/event';
 
 
@@ -224,6 +224,8 @@ export async function createDockerParams(options: ProvisionOptions, disposables:
 	}));
 
 	const cliVariant = await lookupCLIVariant({ exec: cliHost.exec, cmd: dockerPath, args: options.dockerArgs, env: cliHost.env, output });
+	// The explicit non-BuildKit path already uses COPY, so no compatibility probe is needed.
+	const featureCopyFallback = options.useBuildKit !== 'never' && await useFeatureCopyFallback({ exec: cliHost.exec, cmd: dockerPath, args: options.dockerArgs, env: cliHost.env, output }, cliVariant);
 
 	const dockerEngineVer = await dockerEngineVersion({
 		cliHost,
@@ -259,6 +261,7 @@ export async function createDockerParams(options: ProvisionOptions, disposables:
 		updateRemoteUserUIDDefault,
 		additionalCacheFroms: options.additionalCacheFroms,
 		buildKitVersion,
+		useFeatureCopyFallback: featureCopyFallback,
 		dockerEngineVersion: dockerEngineVer,
 		isTTY: process.stdout.isTTY || options.logFormat === 'json',
 		noLockfile,

src/spec-node/utils.ts

@@ -136,6 +136,8 @@ export interface DockerResolverParameters {
 	updateRemoteUserUIDDefault: UpdateRemoteUserUIDDefault;
 	additionalCacheFroms: string[];
 	buildKitVersion: { versionString: string; versionMatch?: string } | undefined;
+	// Limit the Podman compatibility fallback to Feature content transport.
+	useFeatureCopyFallback?: boolean;
 	dockerEngineVersion: { versionString: string; versionMatch?: string } | undefined;
 	isTTY: boolean;
 	noLockfile?: boolean;

src/spec-shutdown/dockerUtils.ts

@@ -6,6 +6,7 @@
 import { CLIHost, runCommand, runCommandNoPty, ExecFunction, ExecParameters, Exec, PtyExecFunction, PtyExec, PtyExecParameters, plainExecAsPtyExec, PlatformInfo } from '../spec-common/commonUtils';
 import { toErrorText } from '../spec-common/errors';
 import * as ptyType from 'node-pty';
+import * as semver from 'semver';
 import { Log, makeLog } from '../spec-utils/log';
 import { Event } from '../spec-utils/event';
 import { escapeRegExCharacters } from '../spec-utils/strings';
@@ -306,6 +307,33 @@ export enum CLIVariant {
 	Wslc = 'wslc',
 }
 
+// Older bundled Buildah versions can miss cache for Feature named-context mounts.
+export async function useFeatureCopyFallback(params: DockerCLIParameters | PartialExecParameters | DockerResolverParameters, cliVariant: CLIVariant): Promise<boolean> {
+	if (cliVariant !== CLIVariant.Podman) {
+		return false;
+	}
+	const { output } = toExecParameters(params);
+	let reason: string;
+	try {
+		const result = await dockerCLI(params, 'info', '--format', '{{.Host.BuildahVersion}}');
+		const versionString = result.stdout.toString().trim();
+		const version = semver.parse(versionString);
+		const distributionSuffix = /^\d*\.?(?:ubuntu|deb|fc|el)[0-9.]*$/i;
+		if (version && (!version.prerelease.length || distributionSuffix.test(version.prerelease.join('.')))) {
+			if (semver.gte(`${version.major}.${version.minor}.${version.patch}`, '1.41.1')) {
+				return false;
+			}
+			reason = `Buildah ${versionString} lacks the named-context build cache fixes`;
+		} else {
+			reason = `Buildah version detection returned an unrecognized or prerelease version: ${JSON.stringify(versionString)}`;
+		}
+	} catch (err) {
+		reason = `Buildah version detection failed: ${err instanceof Error ? err.message : String(err)}`;
+	}
+	output.write(`Using COPY-based Feature installation for Podman: ${reason}. Feature source files may increase image size.`);
+	return true;
+}
+
 export async function lookupCLIVariant(params: PartialExecParameters): Promise<CLIVariant> {
 	try {
 		const { stdout } = await dockerCLI(params, '-v');

src/test/container-features/generateFeaturesConfig.test.ts

@@ -1,6 +1,6 @@
 import { assert } from 'chai';
 import { generateFeaturesConfig, getFeatureLayers, getContainerFeaturesBaseDockerFile, FeatureSet } from '../../spec-configuration/containerFeaturesConfiguration';
-import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log';
+import { createPlainLog, LogLevel, makeLog, nullLog } from '../../spec-utils/log';
 import * as path from 'path';
 import * as process from 'process';
 import * as os from 'os';
@@ -11,9 +11,106 @@ import { URI } from 'vscode-uri';
 import { getLocalCacheFolder } from '../../spec-node/utils';
 import { createTestCommonParams, findFromArgsWithoutDefault, shellExec } from '../testUtils';
 import { getEntPasswdShellCommand } from '../../spec-common/commonUtils';
+import { createDockerParams } from '../../spec-node/devContainers';
+import { staticProvisionParams } from '../../spec-node/featuresCLI/utils';
+import { getExtendImageBuildInfo } from '../../spec-node/containerFeatures';
+import { CLIVariant } from '../../spec-shutdown/dockerUtils';
+import { PassThrough } from 'stream';
+import * as fs from 'fs/promises';
 
 export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace));
 
+describe('Feature content transport', function () {
+    this.timeout(20000);
+
+    for (const mode of ['copy', 'mount', 'no-buildkit'] as const) {
+        it(`keeps content images, contexts and installer paths consistent in ${mode} mode`, async () => {
+            const disposables: (() => Promise<unknown> | undefined)[] = [];
+            let workspaceFolder: string | undefined;
+            try {
+                workspaceFolder = await fs.mkdtemp(path.join(os.tmpdir(), 'feature-transport-'));
+                await fs.cp(path.resolve(__dirname, 'example-v2-features-sets/simple/src'), path.join(workspaceFolder, '.devcontainer'), { recursive: true });
+                const params = await createDockerParams({
+                    ...staticProvisionParams,
+                    dockerPath: process.execPath,
+                    dockerArgs: ['-e', 'process.stdout.write("Docker 28.0.0")', '--'],
+                    workspaceFolder, persistedFolder: os.tmpdir(),
+                    mountWorkspaceGitRoot: false, mountGitWorktreeCommonDir: false,
+                    log: () => { }, logLevel: LogLevel.Error,
+                    remoteEnv: {}, additionalLabels: [], dotfiles: {},
+                    skipFeatureAutoMapping: false, skipPersistingCustomizationsFromFeatures: false,
+                }, disposables);
+                params.isTTY = false;
+                const calls: string[][] = [];
+                const hostExec = params.common.cliHost.exec;
+                params.common.cliHost = {
+                    ...params.common.cliHost,
+                    exec: async options => {
+                        if (options.cmd !== 'selected-engine' && options.cmd !== 'getenforce') {
+                            return hostExec(options);
+                        }
+                        calls.push([options.cmd, ...options.args || []]);
+                        const stdout = new PassThrough();
+                        const stderr = new PassThrough();
+                        return {
+                            stdin: new PassThrough(), stdout, stderr,
+                            exit: new Promise(resolve => setImmediate(() => {
+                                stdout.end('Disabled');
+                                stderr.end();
+                                resolve({ code: 0, signal: null });
+                            })),
+                            terminate: async () => { },
+                        };
+                    },
+                };
+                params.common.output = nullLog;
+                params.noLockfile = true;
+                params.dockerCLI = 'selected-engine';
+                params.dockerArgs = ['--connection', 'test'];
+                params.cliVariant = CLIVariant.Podman;
+                params.useFeatureCopyFallback = mode === 'copy';
+                params.buildKitVersion = mode === 'no-buildkit' ? undefined : { versionString: '0.12.0', versionMatch: '0.12.0' };
+                const config: DevContainerConfig = {
+                    configFilePath: URI.file(path.join(workspaceFolder, '.devcontainer/devcontainer.json')),
+                    image: 'debian:bookworm-slim',
+                    features: { './color': { favorite: 'green' }, './hello': { greeting: 'hello' } },
+                };
+                const result = await getExtendImageBuildInfo(params, { config, raw: config, substitute: value => value }, 'debian:bookworm-slim', {
+                    user: 'root', metadata: { config: [], raw: [], substitute: value => value },
+                }, undefined, {}, false);
+                const build = result?.featureBuildInfo;
+                assert.exists(build);
+                const contentBuilds = calls.filter(call => call.includes('build'));
+                if (mode === 'mount') {
+                    assert.deepEqual(build!.buildKitContexts, { dev_containers_feature_content_source: build!.dstFolder });
+                    assert.lengthOf(contentBuilds, 0);
+                    assert.include(build!.dockerfileContent, 'RUN --mount=type=bind,from=dev_containers_feature_content_source,source=color_0,');
+                    assert.notInclude(build!.dockerfileContent, 'FROM dev_container_feature_content_temp');
+                } else {
+                    assert.deepEqual(build!.buildKitContexts, {});
+                    assert.lengthOf(contentBuilds, 1);
+                    assert.deepEqual(contentBuilds[0], ['selected-engine', '--connection', 'test', 'build', '-t', 'dev_container_feature_content_temp', '-f', path.join(build!.dstFolder, 'Dockerfile.buildContent'), build!.dstFolder]);
+                    assert.include(build!.dockerfileContent, 'FROM dev_container_feature_content_temp as dev_containers_feature_content_source');
+                    assert.include(build!.dockerfileContent, 'COPY --chown=root:root --from=dev_containers_feature_content_source /tmp/build-features/color_0');
+                    assert.notInclude(build!.dockerfileContent, 'RUN --mount');
+                }
+                assert.include(build!.dockerfileContent, 'hello_1');
+                assert.strictEqual((build!.dockerfileContent.match(/&&? \.\/devcontainer-features-install.sh/g) || []).length, 2);
+                assert.isBelow(build!.dockerfileContent.indexOf('color_0'), build!.dockerfileContent.indexOf('hello_1'));
+                assert.strictEqual(params.buildKitVersion?.versionMatch, mode === 'no-buildkit' ? undefined : '0.12.0');
+            } finally {
+                try {
+                    await Promise.all(disposables.map(dispose => dispose()));
+                } finally {
+                    if (workspaceFolder) {
+                        await fs.rm(workspaceFolder, { recursive: true, force: true });
+                    }
+                }
+            }
+        });
+    }
+});
+
 // Testing fetching/generating the devcontainer-features.json config
 describe('validate generateFeaturesConfig()', function () {
 

src/test/dockerUtils.test.ts

@@ -6,7 +6,7 @@
 import { createPlainLog, LogLevel, makeLog, nullLog } from '../spec-utils/log';
 import { inspectImageInRegistry, qualifyImageName } from '../spec-node/utils';
 import assert from 'assert';
-import { CLIVariant, DockerCLIParameters, dockerBuildKitVersion, dockerCLI, dockerEngineVersion, dockerExecFunction, dockerPtyCLI, dockerPtyExecFunction, getEvents, inspectContainer, inspectImage, listContainers, lookupCLIVariant, PartialExecParameters, removeContainer, toExecParameters, toPtyExecParameters } from '../spec-shutdown/dockerUtils';
+import { CLIVariant, DockerCLIParameters, dockerBuildKitVersion, dockerCLI, dockerEngineVersion, dockerExecFunction, dockerPtyCLI, dockerPtyExecFunction, getEvents, inspectContainer, inspectImage, listContainers, lookupCLIVariant, PartialExecParameters, removeContainer, toExecParameters, toPtyExecParameters, useFeatureCopyFallback } from '../spec-shutdown/dockerUtils';
 import { CLIHost, ExecFunction, ExecParameters, PtyExecFunction } from '../spec-common/commonUtils';
 import { PassThrough } from 'stream';
 import { dockerComposeCLIConfig } from '../spec-node/dockerCompose';
@@ -19,6 +19,74 @@ import { createCLIParams } from './testUtils';
 
 export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace));
 
+describe('Podman Feature cache compatibility', () => {
+	function probe(response: string, fail = false) {
+		const calls: ExecParameters[] = [];
+		const lines: string[] = [];
+		const params: PartialExecParameters = {
+			cmd: '/selected/podman', args: ['--connection', 'remote'], env: { CONTAINER_HOST: 'test' },
+			output: makeLog(createPlainLog(text => lines.push(text), () => LogLevel.Trace)),
+			exec: async params => {
+				calls.push(params);
+				if (fail) {
+					throw new Error('builder unavailable');
+				}
+				const stdout = new PassThrough();
+				const stderr = new PassThrough();
+				return {
+					stdin: new PassThrough(), stdout, stderr,
+					exit: new Promise(resolve => setImmediate(() => {
+						stdout.end(response);
+						stderr.end();
+						resolve({ code: 0, signal: null });
+					})),
+					terminate: async () => { },
+				};
+			},
+		};
+		return { params, calls, lines };
+	}
+
+	for (const version of ['1.33.7', '1.39.0', '1.41.0', '1.41.0-1ubuntu0.2', '1.41.1-rc.1', '1.42.0-dev', '', 'unknown']) {
+		it(`uses COPY with affected or unknown Buildah ${JSON.stringify(version)}`, async () => {
+			const { params, lines } = probe(version);
+			assert.strictEqual(await useFeatureCopyFallback(params, CLIVariant.Podman), true);
+			assert.match(lines.join(''), /Using COPY-based Feature installation for Podman/);
+		});
+	}
+
+	for (const version of ['1.41.1', '1.42.0', '1.43.5', '1.41.1-1ubuntu0.2', '1.43.5+vendor.1', '1.43.5-2.fc44', ' 1.43.5\n']) {
+		it(`retains mounts with fixed Buildah ${JSON.stringify(version)}`, async () => {
+			const { params, lines } = probe(version);
+			assert.strictEqual(await useFeatureCopyFallback(params, CLIVariant.Podman), false);
+			assert.doesNotMatch(lines.join(''), /Using COPY-based/);
+		});
+	}
+
+	it('preserves selected executable, command prefix, environment and output', async () => {
+		const { params, calls } = probe('1.39.0');
+		await useFeatureCopyFallback(params, CLIVariant.Podman);
+		assert.strictEqual(calls[0].cmd, params.cmd);
+		assert.deepStrictEqual(calls[0].args, [...params.args!, 'info', '--format', '{{.Host.BuildahVersion}}']);
+		assert.strictEqual(calls[0].env, params.env);
+		assert.strictEqual(calls[0].output, params.output);
+	});
+
+	it('reports query failures and conservatively falls back', async () => {
+		const { params, lines } = probe('', true);
+		assert.strictEqual(await useFeatureCopyFallback(params, CLIVariant.Podman), true);
+		assert.match(lines.join(''), /Buildah version detection failed:.*builder unavailable/s);
+	});
+
+	it('does not query Docker or other CLI variants', async () => {
+		const { params, calls } = probe('');
+		for (const variant of [CLIVariant.Docker, CLIVariant.Wslc]) {
+			assert.strictEqual(await useFeatureCopyFallback(params, variant), false);
+		}
+		assert.strictEqual(calls.length, 0);
+	});
+});
+
 describe('Docker utils', function () {
 	this.timeout(20 * 1000);