diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 073f5e5..500f09b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,13 @@ permissions: jobs: dart: - name: Dart package + name: Dart package (code_assets ${{ matrix.code-assets }}) runs-on: ubuntu-24.04 timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + code-assets: [lowest, latest] permissions: contents: read steps: @@ -30,10 +34,53 @@ jobs: with: sdk: '3.13.0' - run: dart pub get + - name: Resolve the oldest supported code_assets + if: matrix.code-assets == 'lowest' + run: dart pub downgrade code_assets - run: dart format --output=none --set-exit-if-changed . - run: dart analyze --fatal-infos - run: dart test + windows-wsl: + name: Native Windows Dart to WSL2 + runs-on: windows-2025 + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dart-lang/setup-dart@6afc89df92d6eb3834022f73cd65adc8cdfcb92d # v1.8.1 + with: + sdk: '3.13.0' + - uses: Vampire/setup-wsl@4c48163d9f981914dd8caa0fc335fe19cca5bff4 # v7.2.0 + with: + distribution: Ubuntu-24.04 + wsl-version: 2 + set-as-default: 'true' + wsl-conf: | + [boot] + systemd=true + additional-packages: curl xz-utils git ca-certificates + - name: Install Nix inside WSL2 + shell: wsl-bash {0} + run: | + # Keep this installer pin in sync with native.yml. + curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix/tag/v3.11.2 \ + | sh -s -- install linux --no-confirm + - name: Resolve dependencies with native Windows Dart + shell: pwsh + run: dart pub get + # Run Dart in Windows, so the smoke tests must call wsl.exe themselves. + # The tiny store fixture needs no compiler or native-library rebuild. + - name: Test the native Windows to WSL process and filesystem boundary + shell: pwsh + env: + DNIX_TEST_NIX: '1' + DNIX_TEST_WSL_DISTRIBUTION: Ubuntu-24.04 + run: dart test --reporter expanded + # The fixture crate goes through the same reusable workflow consumers use. fixture: name: Fixture diff --git a/CHANGELOG.md b/CHANGELOG.md index 39ee267..1232c46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ ## Unreleased +* Native Windows source and development builds invoke Nix through WSL2. + The hook translates paths, builds the requested non-Apple target, copies + the library back, and retains source mode's pinned hash verification. + The optional `wsl_distribution` user define selects a distribution; + otherwise the default is used. `verify_release`, and `update_lock` for + packages without Apple targets, also work from Windows through WSL2. + This changes only the Dart build runner; Nix recipes, native sources, + and expected library hashes are unchanged. + +* Source builds resolve the flake from the package directory, supporting + parent paths containing `#`. Windows source dependencies with non-ASCII + filenames are tracked correctly. + * Three more Windows targets beside `x86_64-pc-windows-msvc`: `x86_64-pc-windows-gnullvm` and `aarch64-pc-windows-gnullvm` build with zig's MinGW-w64 toolchain, for C and C++ dependencies that need GCC-style flags, diff --git a/README.md b/README.md index e668403..520b013 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,13 @@ dart tool/verify_release.dart linux | Linux aarch64 | yes | Linux and Windows targets | | macOS on Apple Silicon | yes | every target; Apple ones need Xcode 27.0 | | macOS on Intel | yes | no | -| Windows | yes | no; inside WSL2 `verify_release` covers Linux, Android, and Windows targets, and locks come from CI | +| Windows | yes | non-Apple targets through WSL2; the Linux host limits above apply | + +On Windows, install Nix inside a WSL2 Linux distribution. Native Windows +Dart and Flutter invoke it through `wsl.exe`, then copy the built library +back and verify its hash. The default WSL distribution is used unless the +package's hook user defines specify `wsl_distribution`; see +[Windows source builds](doc/configuration.md#windows-source-builds). ## More diff --git a/doc/configuration.md b/doc/configuration.md index 056a5db..8ca3148 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -190,12 +190,11 @@ hooks: | Mode | What happens | Needs | |---|---|---| | `prebuilt` | Downloads the library of the release `dnix.lock` names and verifies its hash. | Dart only. | -| `source` | Builds with Nix exactly as the release did and verifies the same hash. | Nix on Linux or an Apple Silicon Mac; the pinned Xcode for Apple targets. | -| `dev` | Builds with Nix, skipping the Xcode pin and the hash check. | Nix on Linux or an Apple Silicon Mac. | +| `source` | Builds with Nix exactly as the release did and verifies the same hash. | Nix on Linux, an Apple Silicon Mac, or WSL2 on Windows; the pinned Xcode for Apple targets. | +| `dev` | Builds with Nix, skipping the Xcode pin and the hash check. | Nix on Linux, an Apple Silicon Mac, or WSL2 on Windows. | -On Windows and on Intel Macs the hook can only use prebuilt libraries. Nix -runs on Windows inside WSL2, where the tools work but the hook of a Windows -Dart process does not. +Intel Macs can only use prebuilt libraries. On Windows the hook invokes +Nix inside WSL2 for non-Apple targets, as described below. In `source` and `dev` mode the hook declares the flake files and the workspace's files (the git-tracked ones in a checkout, which is what Nix @@ -207,3 +206,45 @@ The hook refuses static linking and refuses builds whose deployment target is below the floors, since those could not match the release. Flutter asks for iOS 13 and macOS 13 whatever the application's own deployment target, so floors above those make every Apple build fail. + +### Windows source builds + +Install WSL2 with a Linux distribution and install Nix inside it. Flutter +and Dart stay on Windows. In `source` and `dev` mode, dnix starts +`wsl.exe`, translates the package and output paths with `wslpath`, and +builds the requested target in Nix's Linux store. A Windows target still +builds a Windows DLL. The library is copied back to the Windows hook +output directory; `source` mode then checks its hash against `dnix.lock` +before Flutter bundles it. A build or hash failure stops the hook. + +The build reads the checked-out native files, including local edits. Keep +native text in LF form through the package's `.gitattributes`, and preserve +any required symlinks and executable permissions. CI builds a fresh clone +on WSL's Linux filesystem, so check for checkout differences when diagnosing +a Windows-only hash mismatch. The expected hash remains the release hash. + +By default, dnix uses the default WSL distribution. Inspect installed +distributions with `wsl.exe --list --verbose`, or select one per package: + +```yaml +hooks: + user_defines: + frostdart: + build: source + wsl_distribution: Ubuntu-24.04 +``` + +The selected distribution must be set up for WSL2, and must be able to +access the package and output directories. Nix is found on its `PATH` or +in `~/.nix-profile/bin`, `/nix/var/nix/profiles/default/bin`, or NixOS's +`/run/current-system/sw/bin`, without loading login-shell profiles. The +distribution's existing Nix configuration is retained, and flakes are +enabled for the build. Configure proxies and other Nix environment settings +inside the distribution; Windows environment variables are not forwarded +automatically. The first build downloads Nix's pinned inputs; later builds +reuse its store as usual. + +No separate application `dnix.lock` is needed. The package supplies the +same flake, native sources, and expected hashes used on other hosts. +Apple targets still require a Mac; Android targets need an x86_64 Linux +distribution because the NDK is not available on Linux arm64. diff --git a/doc/tools.md b/doc/tools.md index fd63a68..8b5e2ae 100644 --- a/doc/tools.md +++ b/doc/tools.md @@ -28,8 +28,14 @@ void main(List args) async => exit(await verifyRelease(args)); ``` Rebuilds one platform and compares with the lock: `linux`, `android`, -`windows`, `macos`, `ios`, or a target triple. Inside WSL2 it covers the -non-Apple platforms. +`windows`, `macos`, `ios`, or a target triple. Native Windows Dart invokes +Nix in the default WSL2 distribution for non-Apple platforms. Set that +distribution with `wsl.exe --set-default `; the `wsl_distribution` +user define applies to build hooks, not these standalone tools. + +`update_lock` also uses WSL2 when called from Windows for a package with +only non-Apple targets. A package shipping Apple targets still needs a +Mac to write its full lock, or the lock printed by CI. `writeLock`, also in `package:dnix/tools.dart`, writes a lock from `SHA256SUMS` files. CI uses it; a package rarely needs a shim for it. @@ -37,3 +43,34 @@ non-Apple platforms. Both tools exit 1 with a one-line reason on any problem dnix can name: a mistake in `dnix.json`, a missing or inconsistent lock, a platform the package does not ship, or Nix missing from PATH. + +## Testing the build runner + +`dart test` checks WSL argument handling and failures, and exercises the +shell script with stand-in tools on POSIX hosts. A separate opt-in smoke +test exercises the real Nix process and file copying with a tiny store +fixture. It also runs the build hook to check Windows target selection, +hash verification, and rejection of mismatched bytes before an asset is +registered. It requires no compiler toolchains or native-library rebuilds: + +```sh +DNIX_TEST_NIX=1 dart test test/nix_build_test.dart +``` + +CI runs the smoke tests from native Windows Dart in PowerShell, with Nix +installed in WSL2. They cover both the default and an explicitly selected +distribution, paths containing spaces, Unicode and literal percent escapes, +and replacing an existing output. The Dart analysis and unit tests also run +with the oldest and latest supported `code_assets` versions. + +Run it from native Windows Dart to exercise the WSL handoff. In PowerShell, +with Nix installed in the default WSL2 distribution: + +```powershell +$env:DNIX_TEST_NIX = '1' +# Optional: also exercise explicit distribution selection. +$env:DNIX_TEST_WSL_DISTRIBUTION = 'Ubuntu-24.04' +dart test test/nix_build_test.dart +Remove-Item Env:DNIX_TEST_NIX +Remove-Item Env:DNIX_TEST_WSL_DISTRIBUTION +``` diff --git a/doc/workflow.md b/doc/workflow.md index 6245826..98df548 100644 --- a/doc/workflow.md +++ b/doc/workflow.md @@ -102,8 +102,9 @@ else fi ``` -Windows runners can only use `prebuilt`, so skip the tests there until the -release exists. +Windows runners without WSL2 and Nix can only use `prebuilt`, so skip the +tests there until the release exists. Configured runners can use `source` +through the native Windows hook's WSL handoff. ## Before publishing to pub.dev @@ -124,8 +125,8 @@ dart tool/verify_release.dart linux # or android, windows, macos, ios, or a t Rebuilds that platform with Nix and compares each library with the lock. The Apple targets use the Xcode bundle under `/Applications` that reports -the pinned version, whatever `xcode-select` points at. Inside WSL2 this -covers the Linux, Android, and Windows targets. +the pinned version, whatever `xcode-select` points at. On Windows the tool +invokes Nix in the default WSL2 distribution for non-Apple targets. ## Bumping dnix, Rust, or nixpkgs diff --git a/lib/src/hook.dart b/lib/src/hook.dart index 1c45470..f748b08 100644 --- a/lib/src/hook.dart +++ b/lib/src/hook.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:code_assets/code_assets.dart'; @@ -110,6 +111,7 @@ Future _build(BuildInput input, BuildOutputBuilder output) async { fileName: fileName, outputDirectory: input.outputDirectory, anyXcode: false, + wslDistribution: input.userDefines['wsl_distribution'], ); verifyHash( bytes: file.readAsBytesSync(), @@ -124,6 +126,7 @@ Future _build(BuildInput input, BuildOutputBuilder output) async { fileName: fileName, outputDirectory: input.outputDirectory, anyXcode: true, + wslDistribution: input.userDefines['wsl_distribution'], ); output.dependencies.addAll(sourceFiles(root, config).where(declared.add)); } @@ -144,9 +147,8 @@ String pinnedHash(DnixLock lock, String target, String fileName) { throw DnixException( 'dnix.lock has no $fileName for $target: the package was locked ' 'without that target, and its maintainer adds it with update_lock. ' - 'Build mode "dev" builds it locally without a pinned hash, on Linux ' - 'or on an Apple Silicon Mac; a Windows process can only use a ' - 'prebuilt library.', + 'Build mode "dev" builds it locally without a pinned hash, using ' + 'Nix on Linux, an Apple Silicon Mac, or WSL2 on Windows.', ); } return expected; @@ -155,20 +157,26 @@ String pinnedHash(DnixLock lock, String target, String fileName) { /// Fails when the consumer's deployment target is below the floor the /// libraries are built for, since such a build could not match the release. void checkFloors(CodeConfig code, DnixConfig config) { + // code_assets 1.2.1 has no deploymentVersion API. Keep its integer API + // while supporting both 1.x and 2.x; dnix's floors are integer majors. final (String, num, int, String)? check = switch (code.targetOS) { OS.macOS => ( 'macOS', + // ignore: deprecated_member_use code.macOS.targetVersion, config.macOSFloor, 'This build asks every native asset for macOS ' + // ignore: deprecated_member_use '${code.macOS.targetVersion}, whatever the application targets, so ' '${config.name} must lower "floors.macos" in ${DnixConfig.fileName}.', ), OS.iOS => ( 'iOS', + // ignore: deprecated_member_use code.iOS.targetVersion, config.iOSFloor, 'This build asks every native asset for iOS ' + // ignore: deprecated_member_use '${code.iOS.targetVersion}, whatever the application targets, so ' '${config.name} must lower "floors.ios" in ${DnixConfig.fileName}.', ), @@ -202,11 +210,10 @@ void checkFloors(CodeConfig code, DnixConfig config) { /// as modified. Iterable sourceFiles(Uri packageRoot, DnixConfig config) sync* { yield* existingFiles(packageRoot, ['flake.nix', 'flake.lock']); - final rootPath = Directory.fromUri(packageRoot).path - .replaceAll(RegExp(r'/+$'), ''); for (final directory in config.sourceDirectories) { - final base = Directory( - directory == '.' ? rootPath : '$rootPath/$directory', + final base = Directory.fromUri( + // The config holds a filesystem path, not an already-encoded URI. + packageRoot.resolveUri(Uri(pathSegments: [...directory.split('/'), ''])), ); if (!base.existsSync()) continue; yield* gitTrackedFiles(base) ?? _files(base, root: base); @@ -221,7 +228,9 @@ List? gitTrackedFiles(Directory directory) { result = Process.runSync( 'git', ['-C', directory.path, 'ls-files', '-z', '--cached', '--', '.'], - stdoutEncoding: systemEncoding, + // Git emits UTF-8 paths with -z, including on Windows where the + // system encoding may otherwise turn them into nonexistent names. + stdoutEncoding: utf8, stderrEncoding: systemEncoding, ); } on ProcessException { @@ -230,13 +239,15 @@ List? gitTrackedFiles(Directory directory) { if (result.exitCode != 0) return null; // Dart's own tool directory is written during a build, so even if it is // tracked it must not be an input. - return [ - for (final name in (result.stdout as String).split('\u0000')) - if (name.isNotEmpty && - !name.startsWith('.dart_tool/') && - File('${directory.path}/$name').existsSync()) - File('${directory.path}/$name').uri, - ]; + final files = []; + for (final name in (result.stdout as String).split('\u0000')) { + if (name.isEmpty || name.startsWith('.dart_tool/')) continue; + final file = File.fromUri( + directory.uri.resolveUri(Uri(pathSegments: name.split('/'))), + ); + if (file.existsSync()) files.add(file.uri); + } + return files; } Iterable _files(Directory directory, {required Directory root}) sync* { diff --git a/lib/src/nix_build.dart b/lib/src/nix_build.dart index 236a5dd..48b251c 100644 --- a/lib/src/nix_build.dart +++ b/lib/src/nix_build.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'package:dnix/src/exception.dart'; import 'package:dnix/src/targets.dart'; +import 'package:dnix/src/wsl_build.dart'; /// Builds [fileName] for [target] with the package's Nix flake and returns /// a writable copy at `//`. @@ -10,17 +11,26 @@ import 'package:dnix/src/targets.dart'; /// Non-Apple targets are pure derivations of the flake. Apple targets need /// the host's Xcode, so they are built by the `dnix-apple-build` command of /// the flake's `apple` shell; [anyXcode] skips its pinned-version check. +/// Windows invokes Nix in WSL2, optionally selecting [wslDistribution]. +/// Only Windows validates and consumes that raw hook configuration value. Future buildWithNix({ required Uri packageRoot, required String target, required String fileName, required Uri outputDirectory, required bool anyXcode, + Object? wslDistribution, }) async { + final destination = File.fromUri( + outputDirectory.resolve('$target/$fileName'), + ); if (Platform.isWindows) { - throw DnixException( - 'Building from source needs Nix, which runs on Windows only inside ' - 'WSL2; a Windows Dart process can only use the prebuilt library.', + return buildWithWsl( + packagePath: packageRoot.toFilePath(), + target: target, + fileName: fileName, + destination: destination, + distribution: wslDistribution, ); } if (Platform.isMacOS && Abi.current() == Abi.macosX64) { @@ -29,14 +39,10 @@ Future buildWithNix({ 'longer supports Intel Macs as hosts. Use the prebuilt library.', ); } - // The package directory is the flake. Nix reads a git checkout as a git - // flake (tracked files, including local modifications, and shallow clones - // handled) and anything else as a plain directory; a bare path needs no - // URL encoding. - final flake = packageRoot.toFilePath().replaceAll(RegExp(r'/+$'), ''); - final destination = File.fromUri( - outputDirectory.resolve('$target/$fileName'), - ); + // Run inside the package so characters such as '#' in its parent path + // cannot be interpreted as part of the flake reference. Nix still sees + // tracked files in a git checkout, or all files in a plain directory. + final packageDirectory = packageRoot.toFilePath(); destination.parent.createSync(recursive: true); if (isAppleTarget(target)) { @@ -44,23 +50,24 @@ Future buildWithNix({ await runNix( [ 'develop', - '$flake#apple', + '.#apple', '--command', 'dnix-apple-build', built.path, target, ], environment: {if (anyXcode) 'DNIX_ANY_XCODE': '1'}, + workingDirectory: packageDirectory, ); return copyOut(File('${built.path}/$target/$fileName'), destination); } final storePath = await runNix([ 'build', - '$flake#$target', + '.#$target', '--no-link', '--print-out-paths', - ]); + ], workingDirectory: packageDirectory); return copyOut(File('${storePath.trim()}/lib/$fileName'), destination); } diff --git a/lib/src/update_lock.dart b/lib/src/update_lock.dart index d46a556..d4399d0 100644 --- a/lib/src/update_lock.dart +++ b/lib/src/update_lock.dart @@ -29,11 +29,6 @@ Future updateLock(List args) async { } Future _update() async { - if (Platform.isWindows) { - throw DnixException( - 'This needs Nix, which runs on Windows inside WSL2. Run it there.', - ); - } final root = DnixConfig.findRoot(); final config = DnixConfig.load(root); if (config.targets.any(isAppleTarget) && !Platform.isMacOS) { diff --git a/lib/src/verify_release.dart b/lib/src/verify_release.dart index bb5b41e..f56bdde 100644 --- a/lib/src/verify_release.dart +++ b/lib/src/verify_release.dart @@ -30,11 +30,6 @@ Future verifyRelease(List args) async { const _platforms = ['linux', 'android', 'windows', 'macos', 'ios']; Future _verify(String platform) async { - if (Platform.isWindows) { - throw DnixException( - 'This needs Nix, which runs on Windows inside WSL2. Run it there.', - ); - } final root = DnixConfig.findRoot(); final config = DnixConfig.load(root); final lock = DnixLock.load(root); diff --git a/lib/src/wsl_build.dart b/lib/src/wsl_build.dart new file mode 100644 index 0000000..0eca5ec --- /dev/null +++ b/lib/src/wsl_build.dart @@ -0,0 +1,145 @@ +import 'dart:io'; + +import 'package:dnix/src/exception.dart'; +import 'package:dnix/src/targets.dart'; + +/// Runs a command, forwarding its output, and returns its exit code. +typedef WslCommandRunner = Future Function( + String executable, + List arguments, +); + +/// The optional distribution chosen by an application's hook configuration. +String? parseWslDistribution(Object? value) { + if (value == null) return null; + if (value is! String || value.trim().isEmpty || value.contains('\u0000')) { + throw DnixException('wsl_distribution must be a non-empty string.'); + } + return value.trim(); +} + +/// Builds a non-Apple target through WSL and copies its library back to a +/// regular, writable Windows file. The caller still verifies its pinned hash. +/// +/// [distribution] defaults to the user's default WSL distribution. The +/// package stays on the Windows filesystem; Nix builds in its Linux store. +Future buildWithWsl({ + required String packagePath, + required String target, + required String fileName, + required File destination, + Object? distribution, + WslCommandRunner runCommand = _runCommand, +}) async { + if (isAppleTarget(target)) { + throw DnixException('Apple libraries can only be built on macOS.'); + } + final selectedDistribution = parseWslDistribution(distribution); + destination.parent.createSync(recursive: true); + + final arguments = [ + if (selectedDistribution != null) ...[ + '--distribution', + selectedDistribution, + ], + // Do not inherit a Windows working directory that this distribution + // might not mount. The script changes to the translated package path. + '--cd', + '/', + '--exec', + '/bin/sh', + '-c', + _buildScript, + 'dnix-wsl', + // These are positional arguments, never interpolated into shell code. + packagePath, + destination.absolute.path, + target, + fileName, + ]; + final int code; + try { + code = await runCommand('wsl.exe', arguments); + } on ProcessException { + throw DnixException.infrastructure( + 'Building from source on Windows needs WSL2 and Nix in a Linux ' + 'distribution. Could not start wsl.exe. Install WSL2, or use ' + 'build: prebuilt.', + ); + } + if (code != 0) { + final where = selectedDistribution == null + ? 'the default distribution' + : 'distribution "$selectedDistribution"'; + final message = + 'WSL build for $target in $where failed with exit code $code; ' + 'its output is above.'; + // A Linux shell status is 0..255. WSL startup errors use a Windows + // exit code (commonly -1 / 0xffffffff); 127 means a missing tool. + if (code == 127 || code < 0 || code > 255) { + throw DnixException.infrastructure( + '$message Check that WSL2 and Nix are installed in $where. ' + 'Use wsl.exe --list --verbose to inspect distributions and ' + 'wsl.exe --set-default to select the default. ' + 'Build hooks also accept the wsl_distribution user define.', + ); + } + throw DnixException(message); + } + if (!destination.existsSync()) { + throw DnixException( + 'WSL finished without writing ${destination.path}. ' + 'Check that the output directory is accessible inside WSL.', + ); + } + return destination; +} + +Future _runCommand(String executable, List arguments) async { + final process = await Process.start( + executable, + arguments, + // WSL's own diagnostics otherwise use UTF-16 when redirected, while + // the Linux commands' output is UTF-8. Keep both streams readable. + environment: const {'WSL_UTF8': '1'}, + ); + await Future.wait([ + process.stdout.forEach(stderr.add), + process.stderr.forEach(stderr.add), + ]); + return process.exitCode; +} + +// A non-login shell avoids profile output and arbitrary shell setup. Nix's +// usual profile locations cover installations whose PATH is set only at login. +// Preserve the distribution's Nix configuration and enable flakes explicitly. +const _buildScript = r''' +set -eu +if ! command -v nix >/dev/null 2>&1; then + for profile in "$HOME/.nix-profile/bin" /nix/var/nix/profiles/default/bin /run/current-system/sw/bin; do + if [ -x "$profile/nix" ]; then + PATH="$profile:$PATH" + export PATH + break + fi + done +fi +if ! command -v nix >/dev/null 2>&1; then + echo 'dnix: install Nix inside this WSL2 distribution before building from source.' >&2 + exit 127 +fi +if ! command -v wslpath >/dev/null 2>&1; then + echo 'dnix: wslpath is missing; select a WSL2 Linux distribution.' >&2 + exit 127 +fi +package=$(wslpath -a -u "$1") +destination=$(wslpath -a -u "$2") +cd "$package" +store=$(nix --extra-experimental-features 'nix-command flakes' build ".#$3" --no-link --print-out-paths) +# Copy to a writable sibling first, then replace the output only on success. +# A failed read or write must not leave an empty or partial library behind. +temporary=$(mktemp "${destination}.dnix-XXXXXX") +trap 'rm -f -- "$temporary"' 0 +cat -- "$store/lib/$4" > "$temporary" +mv -f -- "$temporary" "$destination" +'''; diff --git a/test/hook_test.dart b/test/hook_test.dart index 2c0d725..9542840 100644 --- a/test/hook_test.dart +++ b/test/hook_test.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:code_assets/code_assets.dart'; @@ -61,6 +62,26 @@ void main() { expect(files, {'flake.nix', 'flake.lock', 'Cargo.toml', 'dnix.json'}); }, ); + for (final workspace in [ + 'native sources', + 'native%20sources', + 'native%25sources', + 'native%2Fsources', + 'native%2E%2E/sources', + ]) { + test('treats workspace $workspace as a literal filesystem path', () async { + final root = await Directory.systemTemp.createTemp('dnix %20 café '); + addTearDown(() => root.delete(recursive: true)); + File('${root.path}/dnix.json').writeAsStringSync( + jsonEncode({ + 'rust': {'workspace': workspace, 'crate': 'x'}, + }), + ); + final source = File('${root.path}/$workspace/src/lib.rs') + ..createSync(recursive: true); + expect(sourceFiles(root.uri, DnixConfig.load(root.uri)), [source.uri]); + }); + } gitTests(); } @@ -75,6 +96,10 @@ void gitTests() { for (final path in [ 'native/Cargo.toml', 'native/src/lib.rs', + 'native/src/café_λ.rs', + 'native/src/100% done.rs', + 'native/src/encoded%20%25%2F.rs', + if (!Platform.isWindows) r'native/src/back\slash.rs', 'native/src/gone.rs', 'native/untracked.rs', 'native/target/debug/libx.so', @@ -89,11 +114,18 @@ void gitTests() { await git(['init', '-q']); await git(['add', 'native/Cargo.toml', 'native/src', 'dnix.json']); File('${root.path}/native/src/gone.rs').deleteSync(); - final files = sourceFiles( - root.uri, - DnixConfig.load(root.uri), - ).map((uri) => uri.path.substring(root.uri.path.length)).toSet(); - expect(files, {'native/Cargo.toml', 'native/src/lib.rs'}); + final files = sourceFiles(root.uri, DnixConfig.load(root.uri)).toSet(); + expect(files, { + for (final path in [ + 'native/Cargo.toml', + 'native/src/lib.rs', + 'native/src/café_λ.rs', + 'native/src/100% done.rs', + 'native/src/encoded%20%25%2F.rs', + if (!Platform.isWindows) r'native/src/back\slash.rs', + ]) + File('${root.path}/$path').uri, + }); }, ); @@ -128,11 +160,17 @@ void gitTests() { iOS: os == OS.iOS ? IOSCodeConfig( targetSdk: IOSSdk.iPhoneOS, + // Exercise the API shared with code_assets 1.2.1. + // ignore: deprecated_member_use targetVersion: version, ) : null, macOS: os == OS.macOS - ? MacOSCodeConfig(targetVersion: version) + ? MacOSCodeConfig( + // Exercise the API shared with code_assets 1.2.1. + // ignore: deprecated_member_use + targetVersion: version, + ) : null, android: os == OS.android ? AndroidCodeConfig(targetNdkApi: version) diff --git a/test/nix_build_test.dart b/test/nix_build_test.dart new file mode 100644 index 0000000..a0cc70f --- /dev/null +++ b/test/nix_build_test.dart @@ -0,0 +1,218 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:code_assets/code_assets.dart'; +import 'package:dnix/src/lock.dart'; +import 'package:dnix/src/nix_build.dart'; +import 'package:hooks/hooks.dart'; +import 'package:test/test.dart'; + +void main() { + group( + 'Nix runner (WSL on Windows)', + () { + late Directory root; + const bytes = [0, 10, 13, 127, 128, 255]; + const gnuTarget = 'x86_64-pc-windows-gnullvm'; + + setUp(() async { + // The file URIs must be decoded exactly once before passing native + // paths to Nix or wsl.exe, preserving literal percent escapes. + root = await Directory.systemTemp.createTemp( + "dnix runner's # & café %20 %25 %2F ", + ); + final library = File('${root.path}/native/lib/wallet.dll'); + library.parent.createSync(recursive: true); + library.writeAsBytesSync(bytes); + // A store path is a valid nix build output. This exercises the real + // process and filesystem boundary without compiling native libraries + // or downloading nixpkgs and a compiler toolchain. + File('${root.path}/flake.nix').writeAsStringSync(''' +{ + outputs = { self }: let + library = builtins.path { path = ./native; name = "dnix-runner-smoke"; }; + in { + packages = builtins.listToAttrs (map (system: { + name = system; + value.x86_64-pc-windows-msvc = library; + value.$gnuTarget = library; + }) [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]); + }; +} +'''); + }); + tearDown(() => root.delete(recursive: true)); + + test('copies exact bytes and keeps the output writable', () async { + final built = await buildWithNix( + packageRoot: root.uri, + target: 'x86_64-pc-windows-msvc', + fileName: 'wallet.dll', + outputDirectory: root.uri.resolve('output/'), + anyXcode: false, + ); + expect(built.readAsBytesSync(), bytes); + verifyHash( + bytes: built.readAsBytesSync(), + expected: sha256Hex(bytes), + what: 'Nix runner output', + ); + built.writeAsBytesSync([1]); // The copied output remains writable. + final rebuilt = await buildWithNix( + packageRoot: root.uri, + target: 'x86_64-pc-windows-msvc', + fileName: 'wallet.dll', + outputDirectory: root.uri.resolve('output/'), + anyXcode: false, + ); + expect(rebuilt.readAsBytesSync(), bytes); + }); + + test( + 'selects the requested WSL distribution from native Windows', + () async { + final built = await buildWithNix( + packageRoot: root.uri, + target: gnuTarget, + fileName: 'wallet.dll', + outputDirectory: root.uri.resolve('output/'), + anyXcode: false, + wslDistribution: Platform.environment['DNIX_TEST_WSL_DISTRIBUTION'], + ); + expect(built.readAsBytesSync(), bytes); + }, + skip: + !Platform.isWindows || + Platform.environment['DNIX_TEST_WSL_DISTRIBUTION'] == null + ? 'Set DNIX_TEST_WSL_DISTRIBUTION on native Windows.' + : false, + ); + + test( + 'git sources include local edits and exclude untracked files', + () async { + for (final arguments in [ + ['init', '-q'], + ['add', 'flake.nix', 'native'], + ]) { + final result = await Process.run( + 'git', + arguments, + workingDirectory: root.path, + ); + expect(result.exitCode, 0, reason: '${result.stderr}'); + } + final untracked = File('${root.path}/native/lib/untracked.dll') + ..writeAsBytesSync([99]); + // Nix must see the current worktree, including an unstaged edit. + File('${root.path}/native/lib/wallet.dll').writeAsBytesSync([42]); + Future build(String name) => buildWithNix( + packageRoot: root.uri, + target: gnuTarget, + fileName: name, + outputDirectory: root.uri.resolve('output/'), + anyXcode: false, + ); + expect((await build('wallet.dll')).readAsBytesSync(), [42]); + await expectLater(build('untracked.dll'), throwsA(isA())); + expect(untracked.existsSync(), isTrue); + }, + ); + + test('the hook verifies source output before registering an asset', () async { + File('${root.path}/dnix.json').writeAsStringSync( + jsonEncode({ + 'name': 'demo', + 'repository': 'https://example.invalid/demo', + 'library': 'wallet', + 'asset': 'wallet.dart', + 'rust': {'workspace': 'native', 'crate': 'demo'}, + // The hook must keep the package's GNU target, even though MSVC is + // the default for Windows and the flake offers both variants. + 'targets': [gnuTarget], + }), + ); + final hook = File('${root.path}/build.dart') + ..writeAsStringSync(''' +import 'package:dnix/dnix.dart'; +Future main(List args) => buildNativeLibrary(args); +'''); + final output = File('${root.path}/output/build-output.json'); + output.parent.createSync(recursive: true); + final packageConfig = (await Isolate.packageConfig)!.toFilePath(); + + Future runHook(String mode) async { + final input = BuildInputBuilder() + ..setupShared( + packageRoot: root.uri, + packageName: 'demo', + outputDirectoryShared: root.uri.resolve('shared/'), + outputFile: output.uri, + userDefines: PackageUserDefines( + workspacePubspec: PackageUserDefinesSource( + defines: { + 'build': mode, + // Only Windows consumes this define. Other hosts must + // not validate a setting they do not use. + if (!Platform.isWindows) 'wsl_distribution': 24.04, + if (Platform.isWindows && + Platform.environment['DNIX_TEST_WSL_DISTRIBUTION'] != + null) + 'wsl_distribution': + Platform.environment['DNIX_TEST_WSL_DISTRIBUTION'], + }, + basePath: root.uri, + ), + ), + ) + ..config.setupBuild(linkingEnabled: false) + ..addExtension( + CodeAssetExtension( + targetArchitecture: Architecture.x64, + targetOS: OS.windows, + linkModePreference: LinkModePreference.dynamic, + ), + ); + final inputFile = File('${root.path}/input.json') + ..writeAsStringSync(jsonEncode(input.json)); + return Process.run(Platform.resolvedExecutable, [ + '--packages=$packageConfig', + hook.uri.toString(), + '--config', + inputFile.path, + ]); + } + + Map readOutput() => + jsonDecode(output.readAsStringSync()) as Map; + + DnixLock.write(root.uri, {'$gnuTarget/wallet.dll': sha256Hex(bytes)}); + final verified = await runHook('source'); + expect(verified.exitCode, 0, reason: '${verified.stderr}'); + final asset = BuildOutput(readOutput()).assets.code.single; + expect(asset.file!.path, contains('/$gnuTarget/wallet.dll')); + expect(File.fromUri(asset.file!).readAsBytesSync(), bytes); + + // A valid lock for different bytes must fail even when a previous + // successful build left a library at the destination. + DnixLock.write(root.uri, { + '$gnuTarget/wallet.dll': sha256Hex([99]), + }); + final mismatch = await runHook('source'); + expect(mismatch.exitCode, isNot(0)); + expect(mismatch.stderr, contains('Hash mismatch')); + expect(readOutput()['status'], 'failure'); + expect(readOutput()['assets'] ?? [], isEmpty); + + final development = await runHook('dev'); + expect(development.exitCode, 0, reason: '${development.stderr}'); + expect(BuildOutput(readOutput()).assets.code, hasLength(1)); + }); + }, + skip: Platform.environment['DNIX_TEST_NIX'] != '1' + ? 'Set DNIX_TEST_NIX=1 with Nix installed (inside WSL2 on Windows).' + : false, + timeout: const Timeout(Duration(minutes: 2)), + ); +} diff --git a/test/wsl_build_test.dart b/test/wsl_build_test.dart new file mode 100644 index 0000000..072dbce --- /dev/null +++ b/test/wsl_build_test.dart @@ -0,0 +1,350 @@ +import 'dart:io'; + +import 'package:dnix/src/exception.dart'; +import 'package:dnix/src/lock.dart'; +import 'package:dnix/src/wsl_build.dart'; +import 'package:test/test.dart'; + +void main() { + late Directory root; + late File destination; + const target = 'x86_64-pc-windows-gnullvm'; + const library = 'wallet.dll'; + const packagePath = r"C:\Users\Alice Smith\dart's cache & $(touch injected)"; + const bytes = [0, 10, 13, 127, 128, 255]; + + setUp(() async { + root = await Directory.systemTemp.createTemp('dnix wsl '); + destination = File('${root.path}/output # & cache/$target/$library'); + }); + tearDown(() => root.delete(recursive: true)); + + Future build({ + String buildTarget = target, + Object? distribution, + required WslCommandRunner run, + }) => buildWithWsl( + packagePath: packagePath, + target: buildTarget, + fileName: library, + destination: destination, + distribution: distribution, + runCommand: run, + ); + + test('uses the default distribution and passes paths as arguments', () async { + final built = await build( + run: (executable, arguments) async { + expect(executable, 'wsl.exe'); + expect(arguments.take(5), ['--cd', '/', '--exec', '/bin/sh', '-c']); + expect(arguments[5], isNot(contains(packagePath))); + expect(arguments.skip(6), [ + 'dnix-wsl', + packagePath, + destination.absolute.path, + target, + library, + ]); + destination.writeAsBytesSync(bytes); + return 0; + }, + ); + expect(built.path, destination.path); + verifyHash( + bytes: built.readAsBytesSync(), + expected: sha256Hex(bytes), + what: 'WSL output', + ); + }); + + test('selects a distribution without shell interpolation', () async { + await build( + distribution: ' Ubuntu Development ', + run: (executable, arguments) async { + expect(arguments.take(2), ['--distribution', 'Ubuntu Development']); + destination.writeAsBytesSync(bytes); + return 0; + }, + ); + }); + + test('rejects malformed distribution settings before starting WSL', () async { + expect(parseWslDistribution(null), isNull); + for (final value in [true, 42, '', ' ', 'Ubuntu\u0000']) { + await expectLater( + build( + distribution: value, + run: (_, _) async => + fail('WSL must not start for an invalid setting'), + ), + throwsA(isA()), + ); + } + }); + + test('does not try to build Apple targets through WSL', () async { + await expectLater( + build( + buildTarget: 'aarch64-apple-ios', + run: (_, _) async => fail('WSL must not start for Apple targets'), + ), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('only be built on macOS'), + ), + ), + ); + }); + + test('explains how to proceed when WSL cannot start', () async { + await expectLater( + build( + run: (executable, arguments) async => + throw ProcessException(executable, arguments, 'not found'), + ), + throwsA( + isA() + .having((error) => error.infrastructure, 'infrastructure', isTrue) + .having( + (error) => error.message, + 'message', + contains('Install WSL2'), + ), + ), + ); + }); + + test('a failed build never returns a previous output', () async { + destination.parent.createSync(recursive: true); + destination.writeAsBytesSync(bytes); + await expectLater( + build(run: (_, _) async => 1), + throwsA( + isA() + .having((error) => error.infrastructure, 'infrastructure', isFalse) + .having( + (error) => error.message, + 'message', + contains('exit code 1'), + ), + ), + ); + }); + + test('a missing WSL tool is an infrastructure failure', () async { + await expectLater( + build(distribution: 'Ubuntu', run: (_, _) async => 127), + throwsA( + isA() + .having((error) => error.infrastructure, 'infrastructure', isTrue) + .having( + (error) => error.message, + 'message', + contains('distribution "Ubuntu"'), + ), + ), + ); + }); + + for (final code in [-1, 0xffffffff]) { + test( + 'a WSL startup failure ($code) is an infrastructure failure', + () async { + await expectLater( + build(run: (_, _) async => code), + throwsA( + isA() + .having( + (error) => error.infrastructure, + 'infrastructure', + isTrue, + ) + .having( + (error) => error.message, + 'message', + contains('wsl.exe --set-default'), + ), + ), + ); + }, + ); + } + + test('reports a missing output even if WSL exits successfully', () async { + await expectLater( + build(run: (_, _) async => 0), + throwsA( + isA().having( + (error) => error.message, + 'message', + contains('without writing'), + ), + ), + ); + }); + + group( + 'WSL shell script', + () { + const windowsOutputPath = r'D:\Build Output # &\wallet.dll'; + late Directory bin; + late Directory package; + late Directory store; + late Map environment; + late ProcessResult result; + + void executable(String name, String script) { + final file = File('${bin.path}/$name')..writeAsStringSync(script); + final chmod = Process.runSync('chmod', ['+x', file.path]); + expect(chmod.exitCode, 0, reason: chmod.stderr.toString()); + } + + setUp(() { + bin = Directory('${root.path}/bin')..createSync(); + package = Directory("${root.path}/package's # & directory") + ..createSync(); + store = Directory('${root.path}/store')..createSync(); + final built = File('${store.path}/lib/$library'); + built.parent.createSync(); + built.writeAsBytesSync(bytes); + environment = { + // Keep a real /usr/bin/wslpath from masking the missing-tool + // case when these tests themselves run inside WSL. + 'PATH': bin.path, + 'DNIX_TEST_PACKAGE_WINDOWS': packagePath, + 'DNIX_TEST_PACKAGE_UNIX': package.path, + 'DNIX_TEST_OUTPUT_WINDOWS': windowsOutputPath, + 'DNIX_TEST_OUTPUT_UNIX': destination.absolute.path, + 'DNIX_TEST_STORE': store.path, + 'DNIX_TEST_ARGS': '${root.path}/nix-arguments', + 'DNIX_TEST_CWD': '${root.path}/nix-working-directory', + 'DNIX_TEST_NIX_EXIT': '0', + }; + // Expose only the host commands the fixture needs. Resolve them + // through PATH so this also works with NixOS's filesystem layout. + for (final name in ['cat', 'touch', 'mktemp', 'mv', 'rm']) { + final command = Platform.environment['PATH']! + .split(':') + .map((path) => File('$path/$name')) + .firstWhere((file) => file.existsSync()); + Link('${bin.path}/$name').createSync(command.absolute.path); + } + executable('wslpath', r'''#!/bin/sh +set -eu +test "$1" = -a +test "$2" = -u +case "$3" in + "$DNIX_TEST_PACKAGE_WINDOWS") printf '%s\n' "$DNIX_TEST_PACKAGE_UNIX" ;; + "$DNIX_TEST_OUTPUT_WINDOWS") printf '%s\n' "$DNIX_TEST_OUTPUT_UNIX" ;; + *) exit 3 ;; +esac +'''); + executable('nix', r'''#!/bin/sh +set -eu +printf '%s\n' "$@" > "$DNIX_TEST_ARGS" +pwd > "$DNIX_TEST_CWD" +test "$DNIX_TEST_NIX_EXIT" = 0 || exit "$DNIX_TEST_NIX_EXIT" +printf '%s\n' "$DNIX_TEST_STORE" +'''); + }); + + Future runScript(String executable, List arguments) async { + expect(executable, 'wsl.exe'); + final shellIndex = arguments.indexOf('/bin/sh'); + final shellArguments = arguments.sublist(shellIndex + 1); + // Emulate the Windows-side output spelling on this POSIX host. + // Writing to the untranslated argument must not reach destination. + shellArguments[shellArguments.indexOf(destination.absolute.path)] = + windowsOutputPath; + result = await Process.run( + '/bin/sh', + shellArguments, + environment: environment, + workingDirectory: root.path, + ); + return result.exitCode; + } + + test( + 'translates paths, keeps the Windows target, and copies bytes', + () async { + final built = await build(run: runScript); + expect(built.readAsBytesSync(), bytes); + expect(File('${root.path}/nix-arguments').readAsLinesSync(), [ + '--extra-experimental-features', + 'nix-command flakes', + 'build', + '.#$target', + '--no-link', + '--print-out-paths', + ]); + final cwd = File('${root.path}/nix-working-directory') + .readAsStringSync() + .trim(); + expect( + Directory(cwd).resolveSymbolicLinksSync(), + package.resolveSymbolicLinksSync(), + ); + expect(File('${root.path}/injected').existsSync(), isFalse); + expect(File('${package.path}/injected').existsSync(), isFalse); + // Repeated builds can overwrite the copied file; it is not read-only. + built.writeAsBytesSync([1]); + await build(run: runScript); + expect(built.readAsBytesSync(), bytes); + }, + ); + + test('a Nix failure stops before copying any library', () async { + environment['DNIX_TEST_NIX_EXIT'] = '42'; + await expectLater(build(run: runScript), throwsA(isA())); + expect(result.exitCode, 42); + expect(destination.existsSync(), isFalse); + }); + + test('a path conversion failure stops before Nix', () async { + environment['DNIX_TEST_PACKAGE_WINDOWS'] = 'another path'; + await expectLater(build(run: runScript), throwsA(isA())); + expect(result.exitCode, 3); + expect(File('${root.path}/nix-arguments').existsSync(), isFalse); + }); + + test('reports a missing wslpath before trying a build', () async { + File('${bin.path}/wslpath').deleteSync(); + await expectLater(build(run: runScript), throwsA(isA())); + expect(result.exitCode, 127); + expect(result.stderr, contains('wslpath is missing')); + expect(File('${root.path}/nix-arguments').existsSync(), isFalse); + }); + + test( + 'a missing library fails instead of returning an empty file', + () async { + File('${store.path}/lib/$library').deleteSync(); + await expectLater( + build(run: runScript), + throwsA(isA()), + ); + expect(result.exitCode, isNot(0)); + expect(destination.existsSync(), isFalse); + expect(destination.parent.listSync(), isEmpty); + }, + ); + + test('a failed copy preserves the previous library', () async { + destination.parent.createSync(recursive: true); + destination.writeAsBytesSync([42]); + File('${store.path}/lib/$library').deleteSync(); + await expectLater(build(run: runScript), throwsA(isA())); + expect(destination.readAsBytesSync(), [42]); + expect(destination.parent.listSync().map((file) => file.path), [ + destination.path, + ]); + }); + }, + skip: Platform.isWindows + ? 'Runs the WSL shell with POSIX test tools.' + : false, + ); +}